From e6f2f248f2387ab4296b6df397b1735feaa10489 Mon Sep 17 00:00:00 2001 From: ruckus7382sfgsgdf <355121112+ruckus7382sfgsgdf@users.noreply.github.com> Date: Sun, 22 Nov 2026 11:59:06 -0600 Subject: [PATCH] Triforce v0.7.0 "Sage Of Courage" The 0.6 crashes after warps and area changes fixed (no write-back into memory the capture leaves out), invisible objects fixed (occlusion gate), DLSS frame generation flicker fixed, achievements, the renderer built with a play profile, faster translated game code, helper threads judged on settled scenes, memory, shader preparation, audio effects, launcher, controls and Linux fixes. --- builder/HookshotBuilder/Runner.cs | 7 +- builder/py/hookshot_builder/core.py | 11 +- docs/design.md | 2 +- docs/development.md | 177 +- gpu/render/rx_texture.cpp | 3 +- gpu/vk/achievement_rows.h | 145 ++ gpu/vk/astc_bc_policy.h | 8 + gpu/vk/astc_opaque.h | 47 +- gpu/vk/astc_reuse.h | 133 ++ gpu/vk/barrier_rules.h | 54 + gpu/vk/bc_scratch.h | 66 + gpu/vk/built_check.h | 164 ++ gpu/vk/cap_spare.h | 64 + gpu/vk/capture_tables.h | 110 + gpu/vk/checkpoint_kinds.h | 4 + gpu/vk/commit_room.h | 120 ++ gpu/vk/compute_pin.h | 125 ++ gpu/vk/cpu_count.h | 41 + gpu/vk/dead_count.h | 39 + gpu/vk/desc_counts.h | 67 + gpu/vk/desc_pool_grow.h | 97 + gpu/vk/dlss_checks.h | 69 + gpu/vk/dlss_feature.h | 58 + gpu/vk/dlss_math.h | 30 + gpu/vk/dmabuf_gate.h | 38 + gpu/vk/drawn_page_map.h | 41 + gpu/vk/driver_faults.h | 140 ++ gpu/vk/evict_reason.h | 65 + gpu/vk/fallback_size_policy.h | 73 + gpu/vk/fg_motion.h | 38 + gpu/vk/finalized_obsolete.h | 42 + gpu/vk/first_start.h | 74 + gpu/vk/frame_trace.h | 8 +- gpu/vk/framegen_idle.h | 49 + gpu/vk/framegen_refresh.h | 40 + gpu/vk/framegen_rows.h | 20 + gpu/vk/gpu_choice.h | 20 + gpu/vk/gpu_clock.h | 57 + gpu/vk/gpu_wait_kinds.h | 86 + gpu/vk/guest_holes.h | 112 ++ gpu/vk/helper_spin.h | 5 +- gpu/vk/import_budget.h | 53 + gpu/vk/index_rebase.h | 30 + gpu/vk/lab_ab.h | 68 + gpu/vk/late_inline.h | 49 + gpu/vk/linux_pressure.h | 153 ++ gpu/vk/lock_census.h | 213 ++ gpu/vk/mem_parts.h | 26 + gpu/vk/memory_priority.h | 73 + gpu/vk/mesa_spilling.h | 38 + gpu/vk/mirror_budget.h | 27 + gpu/vk/mirror_spare.h | 42 + gpu/vk/mods_entry_anim.h | 22 +- gpu/vk/mods_page_anim.h | 48 + gpu/vk/msync_pass.h | 95 + gpu/vk/new_wait.h | 49 + gpu/vk/occ_answer.h | 160 +- gpu/vk/occ_lean.h | 82 + gpu/vk/page_marks.h | 100 + gpu/vk/pcache_bound.h | 23 +- gpu/vk/pcache_libs.h | 107 + gpu/vk/pcache_play.h | 8 + gpu/vk/piece_idle.h | 62 + gpu/vk/pipe_ahead.h | 48 + gpu/vk/pipe_order.h | 41 +- gpu/vk/pipe_wait.h | 26 +- gpu/vk/precompile_save.h | 92 + gpu/vk/predict_check.h | 211 ++ gpu/vk/predict_stream.h | 74 + gpu/vk/predict_types.h | 96 + gpu/vk/prepare_again.h | 79 + gpu/vk/prepare_threads.h | 10 + gpu/vk/program_order.h | 7 +- gpu/vk/range_walk.h | 37 + gpu/vk/readback_late.h | 123 ++ gpu/vk/recorder_auto.h | 56 + gpu/vk/redraw_state.h | 57 + gpu/vk/release_queue.h | 64 + gpu/vk/replay_pin.h | 77 + gpu/vk/replay_threads.h | 91 +- gpu/vk/retired_give_back.h | 85 + gpu/vk/settle_gate.h | 58 + gpu/vk/shaders/sim_gpu_burn.comp | 16 + gpu/vk/shaders/sim_gpu_burn_comp_spv.h | 43 + gpu/vk/shadow_prev.h | 7 +- gpu/vk/sim_gpu.h | 56 + gpu/vk/slot_retire.h | 49 + gpu/vk/staging_keep.h | 41 + gpu/vk/stall_watch.h | 7 + gpu/vk/stub_same_size.h | 53 + gpu/vk/surface_lost.h | 39 + gpu/vk/table_stream.h | 97 + gpu/vk/target_overlap.h | 28 + gpu/vk/target_pool.h | 58 + gpu/vk/tex_bc1.h | 4 +- gpu/vk/tex_count.h | 40 + gpu/vk/tex_warm.h | 108 + gpu/vk/translate_wait.h | 71 + gpu/vk/ub_batch.h | 52 + gpu/vk/upscale_off.h | 34 + gpu/vk/upscaler_rows.h | 19 +- gpu/vk/vk_api.h | 21 + gpu/vk/vk_dlss.cpp | 127 +- gpu/vk/vk_dlss.h | 5 + gpu/vk/vk_dlssg.cpp | 18 +- gpu/vk/vk_draw.cpp | 1785 ++++++++++++++--- gpu/vk/vk_draw.h | 25 +- gpu/vk/vk_dump.cpp | 20 +- gpu/vk/vk_dump.h | 8 +- gpu/vk/vk_framegen.cpp | 66 +- gpu/vk/vk_framegen.h | 3 + gpu/vk/vk_fsr3.cpp | 2 + gpu/vk/vk_fsr3_inputs.cpp | 5 + gpu/vk/vk_game_tex.h | 14 + gpu/vk/vk_gpu_clock.cpp | 124 ++ gpu/vk/vk_gpu_clock.h | 10 + gpu/vk/vk_imgui.cpp | 5 +- gpu/vk/vk_imgui.h | 1 + gpu/vk/vk_internal.h | 40 + gpu/vk/vk_jobs.cpp | 3 +- gpu/vk/vk_main.cpp | 221 +- gpu/vk/vk_memory.cpp | 1171 +++++++++-- gpu/vk/vk_menu.cpp | 592 +++++- gpu/vk/vk_menu.h | 6 + gpu/vk/vk_predict.cpp | 89 +- gpu/vk/vk_recorder.h | 17 +- gpu/vk/vk_reflex.cpp | 6 +- gpu/vk/vk_replay.cpp | 1453 +++++++++++++- gpu/vk/vk_salesman.cpp | 3 +- gpu/vk/vk_shader.cpp | 108 +- gpu/vk/vk_shader.h | 23 + gpu/vk/vk_temporal.cpp | 39 +- gpu/vk/vk_texture.cpp | 1091 +++++++++- gpu/vk/vk_texture.h | 16 + gpu/vk/vk_ui.h | 39 + gpu/vk/vk_vendor.h | 11 + gpu/vk/vk_vma.cpp | 66 +- gpu/vk/vk_work.h | 40 +- gpu/vk/vram_compact.h | 21 + gpu/vk/wait_hold.h | 81 + gpu/vk/warm_computes.h | 39 + gpu/vk/wb_excl_clip.h | 53 + gpu/vk/wb_lazy_state.h | 23 +- gpu/vk/wb_rec_wait.h | 104 + gpu/vk/wrote_regions.h | 50 + host/achievements.c | 176 ++ host/achievements.h | 300 +++ host/display.c | 11 + host/heap_commit.h | 52 + host/hos.h | 19 + host/hos_main.c | 148 +- host/hos_mem.c | 115 +- host/hos_sim_cpu.h | 75 + host/hos_svc.c | 7 +- host/hos_thread.c | 316 ++- host/hos_x3d.h | 25 + host/hos_zstd.c | 24 +- host/host_alloc.h | 48 + host/host_zstd.h | 102 +- host/menu_sound.c | 29 +- host/menu_sound.h | 6 + host/mkw_glue.c | 216 +- host/mods_music.c | 111 + host/mods_music.h | 66 + host/movie_pace.h | 54 + host/nn_applet.c | 6 + host/nn_audio.c | 511 ++++- host/nn_fs.c | 49 +- host/nn_hid.c | 11 +- host/nn_os_sync.c | 187 +- host/nn_vi.c | 60 +- host/nn_video.c | 210 +- host/nvn_native.c | 97 +- host/oot_menu.c | 180 +- host/oot_shadow_step.c | 25 + host/oot_shadow_step.h | 41 + host/oot_tex_compact.c | 37 +- host/oot_tex_compact.h | 39 + host/pgo_rt.c | 7 +- host/prompts.c | 8 +- host/rx_backend.c | 27 + host/shadow_slot.h | 52 + host/short_sleep.h | 115 ++ host/stall_host.h | 143 ++ host/video_av1.h | 19 + host/video_catchup.h | 46 + host/video_input.h | 33 + host/video_movie.h | 45 + host/video_nv12.h | 42 + launcher/App/Backend.Bench.cs | 2 +- launcher/App/Backend.Core.cs | 30 +- launcher/App/Backend.GameOpts.cs | 13 +- launcher/App/Host/WinHost.cs | 12 +- launcher/App/Triforce.Launcher.csproj | 3 + launcher/Logic/Benchmark.cs | 10 +- launcher/Logic/Build/BuildProtocol.cs | 25 +- launcher/Logic/Build/BuildRunner.cs | 10 +- launcher/Logic/Build/BuildTimes.cs | 96 +- launcher/Logic/Game.cs | 3 + launcher/Logic/GameOptions/GameOptionRow.cs | 46 +- launcher/Logic/GameOptions/GameOptionsFile.cs | 152 +- .../Logic/GameOptions/GameOptionsSchema.cs | 538 ++--- launcher/Logic/Install.cs | 82 +- launcher/Logic/Tools.cs | 13 + launcher/LogicTests/Tests.Build.cs | 110 + launcher/LogicTests/Tests.Core.cs | 58 + launcher/LogicTests/Tests.GameOptions.cs | 331 ++- launcher/LogicTests/Tests.Release.cs | 18 + nx2/build.py | 129 +- nx2/chunks.py | 5 +- nx2/cold_parts.py | 164 ++ nx2/fastvec.py | 167 ++ nx2/fma_soft.py | 147 ++ nx2/fma_steps.py | 62 + nx2/insn.py | 16 + nx2/lift.py | 8 + nx2/lifted_c.py | 50 + nx2/make_game.py | 459 ++++- nx2/nup.py | 127 +- nx2/oot_patches.py | 84 + nx2/pipeline_states.py | 172 +- nx2/prompt_assets.py | 38 +- nx2/prompt_faces/README.md | 4 +- nx2/rtgen.py | 13 +- nx2/split_hot.py | 218 +- nx2/test_nup_in_place.py | 286 +++ nx2/test_nup_out_of_space.py | 9 +- nx2/test_nup_paths.py | 54 +- nx2/vecfp.py | 308 +++ nx2/win_manifest.py | 36 +- readme.md | 34 + scripts/linux/appimage.sh | 2 +- scripts/linux/check_player_build.sh | 0 setup/Setup.cs | 26 +- setup/pack_tools.ps1 | 2 +- tests/achievement_rows_test.cpp | 99 + tests/achievements_host_test.c | 174 ++ tests/achievements_test.c | 190 ++ tests/audio_mix_effects_test.c | 297 +++ tests/audio_wave_buffer_release_test.c | 2 + tests/desc_sets_test.cpp | 1 + tests/dmb_light_test.c | 96 + tests/draw_stream_visibility_test.cpp | 11 +- tests/dump_readback_test.cpp | 2 +- tests/fallback_buffer_cleanup_test.cpp | 42 + tests/fma_soft_test.c | 339 ++++ tests/fma_steps_side.c | 26 + tests/fma_steps_test.c | 211 ++ tests/gpu_waits_test.cpp | 160 ++ tests/helper_spin_test.cpp | 4 +- tests/input_background_test.c | 7 +- tests/invalidate_vcore_test.cpp | 25 +- tests/mirror_bookkeeping_test.cpp | 119 ++ tests/occ_answer_test.cpp | 85 + tests/page_table_stream_test.cpp | 162 ++ tests/parallel_init_order_test.cpp | 4 + tests/parallel_table_order_test.cpp | 3 + tests/pipeline_cache_save_test.cpp | 1 + tests/present_source_crop_test.cpp | 7 +- tests/program_cache_cleanup_test.cpp | 3 + tests/recording_admission_test.cpp | 40 +- tests/replay_threads_tuner_test.cpp | 136 +- tests/sampler_admission_test.cpp | 1 + tests/test_achievements.py | 213 ++ tests/test_alt_menu.py | 104 + tests/test_astc_bc_policy.cpp | 15 +- tests/test_astc_bc_policy.py | 16 +- tests/test_astc_reuse.py | 197 ++ tests/test_audio_mix_effects.py | 82 + tests/test_barrier_rules.py | 29 +- tests/test_bc_scratch_layers.py | 114 ++ tests/test_bc_small_layers.py | 148 ++ tests/test_bg_import_held.py | 6 + tests/test_build_priority.py | 125 ++ tests/test_built_check.py | 413 ++++ tests/test_cap_free_async.py | 241 +++ tests/test_capture_behind.py | 3 +- tests/test_capture_copy_helpers.py | 1 + tests/test_capture_spare_reuse.py | 440 ++++ tests/test_capture_tables.py | 203 ++ tests/test_chunk_default.py | 135 ++ tests/test_chunk_parts.py | 323 +++ tests/test_commit_limit.py | 376 ++++ tests/test_commit_room.py | 498 +++++ tests/test_compact_start.py | 147 ++ tests/test_compile_stops_on_exception.py | 166 ++ tests/test_compute_pin.py | 214 ++ tests/test_dead_count.py | 91 + tests/test_dead_pool_now.py | 55 + tests/test_desc_counts.py | 217 ++ tests/test_desc_keep.py | 2 +- tests/test_desc_pool_grow.py | 168 ++ tests/test_desc_sets.py | 2 +- tests/test_dispatch_quiet.py | 188 ++ tests/test_dlss_checks.cpp | 83 + tests/test_dlss_checks.py | 28 + tests/test_dlss_feature.py | 149 ++ tests/test_dlss_prewarm.py | 160 ++ tests/test_dlss_sdk_cache.py | 3 +- tests/test_dlssg_depth_clip.py | 186 ++ tests/test_dmabuf_driver_gate.py | 208 ++ tests/test_dmb_light.py | 266 +++ tests/test_draw_binds.py | 5 + tests/test_draw_side_effects.py | 21 +- tests/test_draw_stream_visibility.py | 2 +- tests/test_drawn_page_map.py | 188 ++ tests/test_driver_cache_folder.py | 215 ++ tests/test_driver_cache_room.py | 176 ++ tests/test_driver_faults.py | 243 +++ tests/test_evict_excess.py | 121 ++ tests/test_evict_reason.cpp | 77 + tests/test_evict_reason.py | 221 ++ tests/test_failed_compile_keeps_objects.py | 141 +- tests/test_fallback_buffer_cleanup.py | 4 +- tests/test_fallback_loaded.py | 199 ++ tests/test_fallback_size_policy.py | 185 ++ tests/test_fg_motion.py | 96 + tests/test_finalized_obsolete.py | 144 ++ tests/test_finish_installed.py | 380 ++++ tests/test_finish_output.py | 36 +- tests/test_first_start.py | 165 ++ tests/test_flush_nothing_held.py | 223 ++ tests/test_fma_soft.py | 220 ++ tests/test_fma_steps.py | 246 +++ tests/test_fps_cap_monitor.py | 4 +- tests/test_fps_overlay_order.py | 4 + tests/test_frame_rate_cap_step.py | 6 +- tests/test_frame_trace.cpp | 6 +- tests/test_framegen_idle.py | 147 ++ tests/test_framegen_refresh.py | 119 ++ tests/test_framegen_set_bounds.py | 116 ++ tests/test_gpl_library_cache.py | 2 +- tests/test_gpu_choice.cpp | 20 + tests/test_gpu_choice.py | 12 + tests/test_gpu_clock.py | 146 ++ tests/test_gpu_waits.py | 113 ++ tests/test_guest_census.py | 44 +- tests/test_guest_holes.py | 227 +++ tests/test_heap_commit_lazy.py | 156 ++ tests/test_helper_spin.py | 2 +- tests/test_host_free_marks.py | 157 ++ tests/test_host_zstd.py | 101 +- tests/test_import_budget.py | 148 ++ tests/test_import_chunk_default.py | 143 ++ tests/test_import_leaving.py | 168 ++ tests/test_import_windows.py | 3 +- tests/test_index_rebase.py | 129 ++ tests/test_input_background.py | 3 + tests/test_invalidate_quick.py | 374 ++++ tests/test_invalidate_vcore.py | 4 +- tests/test_jobs_wait_for.py | 2 +- tests/test_jobs_wait_runs_queued.py | 2 +- tests/test_lab_ab_each.py | 186 ++ tests/test_late_inline.py | 130 ++ tests/test_lazy_wb_host.py | 5 + tests/test_lift_key_env.py | 254 +++ tests/test_lift_key_files.py | 4 +- tests/test_lifted_c_scan.py | 218 ++ tests/test_linux_pressure.cpp | 140 ++ tests/test_linux_pressure.py | 242 +++ tests/test_lock_census.py | 381 ++++ tests/test_log_batch.py | 7 +- tests/test_log_off_lock.py | 206 ++ tests/test_make_game_cache.py | 99 + tests/test_mem_parts.py | 40 + tests/test_mem_trim.py | 6 +- tests/test_memory_priority.py | 205 ++ tests/test_mesa_spilling_rate.py | 121 ++ tests/test_mesh_fallback.py | 2 +- tests/test_mirror_bookkeeping.py | 123 ++ tests/test_mirror_budget.py | 93 + tests/test_mirror_piece_idle.py | 200 ++ tests/test_mirror_spare.py | 106 + tests/test_missing_nup.py | 2 + tests/test_mods_entry_input.py | 3 +- tests/test_mods_entry_sync.py | 191 ++ tests/test_mods_music.py | 234 +++ tests/test_mods_page_anim.py | 133 ++ tests/test_movie_pace.py | 303 +++ tests/test_msync_pass.py | 275 +++ tests/test_nvn_host_tables.py | 5 + tests/test_obj_cache_size.py | 226 +++ tests/test_occ_answer.py | 29 +- tests/test_occ_depth_only.py | 203 ++ tests/test_occ_gate.py | 240 +++ tests/test_occ_lean.py | 180 ++ tests/test_oot_menu_linux_half.py | 148 +- tests/test_oot_menu_mouse.py | 3 + tests/test_oot_shadow_step.py | 137 ++ tests/test_oot_tex_compact.py | 84 + tests/test_open_file_vcore.py | 71 + tests/test_order_wake.py | 256 +++ tests/test_os_yield.py | 307 +++ tests/test_pace_steady.py | 3 +- tests/test_pace_wait.py | 5 +- tests/test_page_table_stream.py | 101 + tests/test_parallel_fixes.py | 2 +- tests/test_pass_hazard.py | 8 +- tests/test_pcache_bound.py | 2 +- tests/test_pcache_carry.py | 9 +- tests/test_pcache_gate.py | 8 +- tests/test_pcache_head_read.py | 173 ++ tests/test_pcache_libs.py | 170 ++ tests/test_pcache_own_passes.py | 198 ++ tests/test_pcache_play.py | 9 +- tests/test_perf_overlay.py | 3 +- tests/test_pipe_ahead.py | 106 + tests/test_pipe_depth_budget.py | 260 +++ tests/test_pipe_order.cpp | 44 +- tests/test_pipe_order.py | 4 +- tests/test_pipe_warm.py | 4 +- tests/test_pipeline_contrib.py | 148 ++ tests/test_pipeline_states.py | 6 +- tests/test_precompile_save.py | 279 +++ tests/test_predict_check.py | 286 +++ tests/test_predict_stream.py | 147 ++ tests/test_predict_types.py | 294 +++ tests/test_prepare_again.py | 140 ++ tests/test_prepare_threads.cpp | 6 + tests/test_prepare_threads.py | 7 +- tests/test_prepare_timing.py | 4 +- tests/test_prepared_stamps.py | 12 +- tests/test_present_device_loss.py | 1 + tests/test_present_image_free.py | 307 +++ tests/test_present_period_line.py | 8 +- tests/test_present_rate.py | 6 +- tests/test_present_semaphores.py | 9 +- tests/test_present_surface_lost.py | 632 ++++++ tests/test_present_trace.py | 4 + tests/test_prompt_guides.py | 76 + tests/test_prompt_widen_cache.py | 123 ++ tests/test_prompts_background.py | 87 +- tests/test_push_state.py | 2 +- tests/test_radv_gcn.py | 120 ++ tests/test_radv_mesh_optimizer.py | 150 ++ tests/test_radv_vrs_drop.py | 140 ++ tests/test_range_walk.py | 202 ++ tests/test_readback_late.py | 179 ++ tests/test_recorder_auto.py | 120 ++ tests/test_recorder_sync.py | 19 + tests/test_recording_admission.py | 15 +- tests/test_redraw_state.py | 149 ++ tests/test_reflex_pace.py | 1 + tests/test_reflex_swapchain_lifetime.py | 1 + tests/test_region_free_vcore.py | 349 ++++ tests/test_release_queue.py | 236 +++ tests/test_replay_hot_trims.py | 319 +++ tests/test_replay_pin_soft.py | 454 +++++ tests/test_replay_prio.py | 21 + tests/test_replay_threads_row.py | 2 +- tests/test_replay_threads_tuner.py | 16 +- tests/test_report_words.py | 1 + tests/test_retired_give_back.py | 218 ++ tests/test_rt_pool_minimum.py | 2 + tests/test_same_dyn.py | 36 +- tests/test_segment_heap.py | 154 ++ tests/test_settle_gate.py | 298 +++ tests/test_shader_stamp_cache.py | 38 +- tests/test_shadow_prev.py | 5 +- tests/test_shadow_slot.py | 112 ++ tests/test_short_sleep_timer.py | 436 ++++ tests/test_sim_cores.py | 310 +++ tests/test_slot_reset_ahead.py | 137 ++ tests/test_slot_retire.py | 108 + tests/test_split_before_draw.py | 137 ++ tests/test_split_ub_flush.py | 107 + tests/test_staging_burst.py | 275 +++ tests/test_stall_host.py | 387 ++++ tests/test_steady_period.py | 6 +- tests/test_step_steady.py | 2 +- tests/test_sticky_hot_pieces.py | 391 ++++ tests/test_sweep_walk.py | 2 +- tests/test_taau_env_lookup.py | 75 +- tests/test_target_image_pool.py | 13 +- tests/test_target_overlap.cpp | 30 + tests/test_target_overlap.py | 10 +- tests/test_target_pool_rules.py | 193 ++ tests/test_tex_bc1.cpp | 56 +- tests/test_tex_bc1.py | 6 +- tests/test_tex_count_blocks.py | 88 + tests/test_tex_pressure.py | 2 +- tests/test_tex_same_staged.py | 193 ++ tests/test_tex_warm_list.py | 257 +++ tests/test_texels_vcore.py | 129 ++ tests/test_texture_pipelines_warm.py | 192 ++ tests/test_texture_pool_ids.py | 2 +- tests/test_tls_once.py | 90 + tests/test_translate_wait.py | 229 +++ tests/test_ub_update_batch.py | 154 ++ tests/test_udmabuf_limit_unreadable.py | 140 ++ tests/test_upscale_fallback.py | 102 +- tests/test_upscale_off.py | 115 ++ tests/test_upscaler_rows.cpp | 25 + tests/test_upscaler_rows.py | 25 + tests/test_vbytes.py | 196 ++ tests/test_vcore_wide.py | 339 ++++ tests/test_vecfp.py | 291 +++ tests/test_video_acquire_wait.py | 144 ++ tests/test_video_av1_order.py | 228 +++ tests/test_video_catchup.py | 208 ++ tests/test_video_input_limit.py | 251 +++ tests/test_video_lazy_rgba.py | 213 ++ tests/test_vk_pgo.py | 102 + tests/test_vram_compact.cpp | 16 + tests/test_vram_compact.py | 7 +- tests/test_wait_hold_pause.py | 434 ++++ tests/test_warm_computes.cpp | 32 + tests/test_warm_computes.py | 65 + tests/test_wb_diff_stale_shadow.py | 1 + tests/test_wb_excl_clip.cpp | 111 + tests/test_wb_excl_clip.py | 46 + tests/test_wb_forget_order.py | 152 ++ tests/test_wb_lazy_state.cpp | 49 +- tests/test_wb_rec_wait.cpp | 110 + tests/test_wb_rec_wait.py | 66 +- tests/test_work_lock.py | 232 +++ tests/test_wrote_regions.py | 122 ++ tests/test_x3d_placement.py | 101 +- tests/tex_count_test.cpp | 82 + tests/texture_staging_publication_test.cpp | 1 + tests/ub_update_batch_test.cpp | 241 +++ tests/upload_same_test.cpp | 1 + tests/vbytes_test.c | 129 ++ tests/vcore_spin_test.c | 4 + tests/vecfp_test.c | 170 ++ tests/video_acquire_test.c | 141 ++ tests/video_fixture.py | 66 + tests/video_play.h | 103 + tests/video_standins.h | 265 +++ 529 files changed, 55958 insertions(+), 1898 deletions(-) create mode 100644 gpu/vk/achievement_rows.h create mode 100644 gpu/vk/astc_reuse.h create mode 100644 gpu/vk/bc_scratch.h create mode 100644 gpu/vk/built_check.h create mode 100644 gpu/vk/cap_spare.h create mode 100644 gpu/vk/capture_tables.h create mode 100644 gpu/vk/commit_room.h create mode 100644 gpu/vk/compute_pin.h create mode 100644 gpu/vk/cpu_count.h create mode 100644 gpu/vk/dead_count.h create mode 100644 gpu/vk/desc_counts.h create mode 100644 gpu/vk/desc_pool_grow.h create mode 100644 gpu/vk/dlss_feature.h create mode 100644 gpu/vk/dmabuf_gate.h create mode 100644 gpu/vk/drawn_page_map.h create mode 100644 gpu/vk/driver_faults.h create mode 100644 gpu/vk/evict_reason.h create mode 100644 gpu/vk/fallback_size_policy.h create mode 100644 gpu/vk/fg_motion.h create mode 100644 gpu/vk/finalized_obsolete.h create mode 100644 gpu/vk/first_start.h create mode 100644 gpu/vk/framegen_idle.h create mode 100644 gpu/vk/framegen_refresh.h create mode 100644 gpu/vk/gpu_clock.h create mode 100644 gpu/vk/gpu_wait_kinds.h create mode 100644 gpu/vk/guest_holes.h create mode 100644 gpu/vk/import_budget.h create mode 100644 gpu/vk/index_rebase.h create mode 100644 gpu/vk/late_inline.h create mode 100644 gpu/vk/linux_pressure.h create mode 100644 gpu/vk/lock_census.h create mode 100644 gpu/vk/memory_priority.h create mode 100644 gpu/vk/mesa_spilling.h create mode 100644 gpu/vk/mirror_budget.h create mode 100644 gpu/vk/mirror_spare.h create mode 100644 gpu/vk/mods_page_anim.h create mode 100644 gpu/vk/msync_pass.h create mode 100644 gpu/vk/new_wait.h create mode 100644 gpu/vk/occ_lean.h create mode 100644 gpu/vk/page_marks.h create mode 100644 gpu/vk/pcache_libs.h create mode 100644 gpu/vk/piece_idle.h create mode 100644 gpu/vk/pipe_ahead.h create mode 100644 gpu/vk/precompile_save.h create mode 100644 gpu/vk/predict_check.h create mode 100644 gpu/vk/predict_stream.h create mode 100644 gpu/vk/predict_types.h create mode 100644 gpu/vk/prepare_again.h create mode 100644 gpu/vk/range_walk.h create mode 100644 gpu/vk/readback_late.h create mode 100644 gpu/vk/recorder_auto.h create mode 100644 gpu/vk/redraw_state.h create mode 100644 gpu/vk/release_queue.h create mode 100644 gpu/vk/replay_pin.h create mode 100644 gpu/vk/retired_give_back.h create mode 100644 gpu/vk/settle_gate.h create mode 100644 gpu/vk/shaders/sim_gpu_burn.comp create mode 100644 gpu/vk/shaders/sim_gpu_burn_comp_spv.h create mode 100644 gpu/vk/sim_gpu.h create mode 100644 gpu/vk/slot_retire.h create mode 100644 gpu/vk/staging_keep.h create mode 100644 gpu/vk/stub_same_size.h create mode 100644 gpu/vk/surface_lost.h create mode 100644 gpu/vk/table_stream.h create mode 100644 gpu/vk/target_pool.h create mode 100644 gpu/vk/tex_count.h create mode 100644 gpu/vk/translate_wait.h create mode 100644 gpu/vk/ub_batch.h create mode 100644 gpu/vk/upscale_off.h create mode 100644 gpu/vk/vk_gpu_clock.cpp create mode 100644 gpu/vk/vk_gpu_clock.h create mode 100644 gpu/vk/wait_hold.h create mode 100644 gpu/vk/warm_computes.h create mode 100644 gpu/vk/wb_excl_clip.h create mode 100644 gpu/vk/wrote_regions.h create mode 100644 host/achievements.c create mode 100644 host/achievements.h create mode 100644 host/heap_commit.h create mode 100644 host/hos_sim_cpu.h create mode 100644 host/host_alloc.h create mode 100644 host/mods_music.c create mode 100644 host/mods_music.h create mode 100644 host/movie_pace.h create mode 100644 host/oot_shadow_step.c create mode 100644 host/oot_shadow_step.h create mode 100644 host/oot_tex_compact.h create mode 100644 host/shadow_slot.h create mode 100644 host/short_sleep.h create mode 100644 host/stall_host.h create mode 100644 host/video_av1.h create mode 100644 host/video_catchup.h create mode 100644 host/video_input.h create mode 100644 host/video_movie.h create mode 100644 host/video_nv12.h create mode 100644 nx2/cold_parts.py create mode 100644 nx2/fastvec.py create mode 100644 nx2/fma_soft.py create mode 100644 nx2/fma_steps.py create mode 100644 nx2/test_nup_in_place.py create mode 100644 nx2/vecfp.py mode change 100644 => 100755 scripts/linux/appimage.sh mode change 100644 => 100755 scripts/linux/check_player_build.sh create mode 100644 tests/achievement_rows_test.cpp create mode 100644 tests/achievements_host_test.c create mode 100644 tests/achievements_test.c create mode 100644 tests/audio_mix_effects_test.c create mode 100644 tests/dmb_light_test.c create mode 100644 tests/fma_soft_test.c create mode 100644 tests/fma_steps_side.c create mode 100644 tests/fma_steps_test.c create mode 100644 tests/gpu_waits_test.cpp create mode 100644 tests/mirror_bookkeeping_test.cpp create mode 100644 tests/page_table_stream_test.cpp create mode 100644 tests/test_achievements.py create mode 100644 tests/test_alt_menu.py create mode 100644 tests/test_astc_reuse.py create mode 100644 tests/test_audio_mix_effects.py create mode 100644 tests/test_bc_scratch_layers.py create mode 100644 tests/test_bc_small_layers.py create mode 100644 tests/test_build_priority.py create mode 100644 tests/test_built_check.py create mode 100644 tests/test_cap_free_async.py create mode 100644 tests/test_capture_spare_reuse.py create mode 100644 tests/test_capture_tables.py create mode 100644 tests/test_chunk_default.py create mode 100644 tests/test_chunk_parts.py create mode 100644 tests/test_commit_limit.py create mode 100644 tests/test_commit_room.py create mode 100644 tests/test_compact_start.py create mode 100644 tests/test_compile_stops_on_exception.py create mode 100644 tests/test_compute_pin.py create mode 100644 tests/test_dead_count.py create mode 100644 tests/test_dead_pool_now.py create mode 100644 tests/test_desc_counts.py create mode 100644 tests/test_desc_pool_grow.py create mode 100644 tests/test_dispatch_quiet.py create mode 100644 tests/test_dlss_feature.py create mode 100644 tests/test_dlss_prewarm.py create mode 100644 tests/test_dlssg_depth_clip.py create mode 100644 tests/test_dmabuf_driver_gate.py create mode 100644 tests/test_dmb_light.py create mode 100644 tests/test_drawn_page_map.py create mode 100644 tests/test_driver_cache_folder.py create mode 100644 tests/test_driver_cache_room.py create mode 100644 tests/test_driver_faults.py create mode 100644 tests/test_evict_excess.py create mode 100644 tests/test_evict_reason.cpp create mode 100644 tests/test_evict_reason.py create mode 100644 tests/test_fallback_loaded.py create mode 100644 tests/test_fallback_size_policy.py create mode 100644 tests/test_fg_motion.py create mode 100644 tests/test_finalized_obsolete.py create mode 100644 tests/test_finish_installed.py create mode 100644 tests/test_first_start.py create mode 100644 tests/test_flush_nothing_held.py create mode 100644 tests/test_fma_soft.py create mode 100644 tests/test_fma_steps.py create mode 100644 tests/test_framegen_idle.py create mode 100644 tests/test_framegen_refresh.py create mode 100644 tests/test_framegen_set_bounds.py create mode 100644 tests/test_gpu_clock.py create mode 100644 tests/test_gpu_waits.py create mode 100644 tests/test_guest_holes.py create mode 100644 tests/test_heap_commit_lazy.py create mode 100644 tests/test_host_free_marks.py create mode 100644 tests/test_import_budget.py create mode 100644 tests/test_import_chunk_default.py create mode 100644 tests/test_import_leaving.py create mode 100644 tests/test_index_rebase.py create mode 100644 tests/test_invalidate_quick.py create mode 100644 tests/test_lab_ab_each.py create mode 100644 tests/test_late_inline.py create mode 100644 tests/test_lift_key_env.py create mode 100644 tests/test_lifted_c_scan.py create mode 100644 tests/test_linux_pressure.cpp create mode 100644 tests/test_linux_pressure.py create mode 100644 tests/test_lock_census.py create mode 100644 tests/test_log_off_lock.py create mode 100644 tests/test_memory_priority.py create mode 100644 tests/test_mesa_spilling_rate.py create mode 100644 tests/test_mirror_bookkeeping.py create mode 100644 tests/test_mirror_budget.py create mode 100644 tests/test_mirror_piece_idle.py create mode 100644 tests/test_mirror_spare.py create mode 100644 tests/test_mods_entry_sync.py create mode 100644 tests/test_mods_music.py create mode 100644 tests/test_mods_page_anim.py create mode 100644 tests/test_movie_pace.py create mode 100644 tests/test_msync_pass.py create mode 100644 tests/test_obj_cache_size.py create mode 100644 tests/test_occ_depth_only.py create mode 100644 tests/test_occ_gate.py create mode 100644 tests/test_occ_lean.py create mode 100644 tests/test_oot_shadow_step.py create mode 100644 tests/test_open_file_vcore.py create mode 100644 tests/test_order_wake.py create mode 100644 tests/test_os_yield.py create mode 100644 tests/test_page_table_stream.py create mode 100644 tests/test_pcache_head_read.py create mode 100644 tests/test_pcache_libs.py create mode 100644 tests/test_pcache_own_passes.py create mode 100644 tests/test_pipe_ahead.py create mode 100644 tests/test_pipe_depth_budget.py create mode 100644 tests/test_pipeline_contrib.py create mode 100644 tests/test_precompile_save.py create mode 100644 tests/test_predict_check.py create mode 100644 tests/test_predict_stream.py create mode 100644 tests/test_predict_types.py create mode 100644 tests/test_prepare_again.py create mode 100644 tests/test_present_image_free.py create mode 100644 tests/test_present_surface_lost.py create mode 100644 tests/test_prompt_guides.py create mode 100644 tests/test_prompt_widen_cache.py create mode 100644 tests/test_radv_gcn.py create mode 100644 tests/test_radv_mesh_optimizer.py create mode 100644 tests/test_radv_vrs_drop.py create mode 100644 tests/test_range_walk.py create mode 100644 tests/test_readback_late.py create mode 100644 tests/test_recorder_auto.py create mode 100644 tests/test_redraw_state.py create mode 100644 tests/test_region_free_vcore.py create mode 100644 tests/test_release_queue.py create mode 100644 tests/test_replay_hot_trims.py create mode 100644 tests/test_replay_pin_soft.py create mode 100644 tests/test_retired_give_back.py create mode 100644 tests/test_segment_heap.py create mode 100644 tests/test_settle_gate.py create mode 100644 tests/test_shadow_slot.py create mode 100644 tests/test_short_sleep_timer.py create mode 100644 tests/test_sim_cores.py create mode 100644 tests/test_slot_reset_ahead.py create mode 100644 tests/test_slot_retire.py create mode 100644 tests/test_split_before_draw.py create mode 100644 tests/test_split_ub_flush.py create mode 100644 tests/test_staging_burst.py create mode 100644 tests/test_stall_host.py create mode 100644 tests/test_sticky_hot_pieces.py create mode 100644 tests/test_target_pool_rules.py create mode 100644 tests/test_tex_count_blocks.py create mode 100644 tests/test_tex_same_staged.py create mode 100644 tests/test_tex_warm_list.py create mode 100644 tests/test_texels_vcore.py create mode 100644 tests/test_texture_pipelines_warm.py create mode 100644 tests/test_tls_once.py create mode 100644 tests/test_translate_wait.py create mode 100644 tests/test_ub_update_batch.py create mode 100644 tests/test_udmabuf_limit_unreadable.py create mode 100644 tests/test_upscale_off.py create mode 100644 tests/test_vbytes.py create mode 100644 tests/test_vcore_wide.py create mode 100644 tests/test_vecfp.py create mode 100644 tests/test_video_acquire_wait.py create mode 100644 tests/test_video_av1_order.py create mode 100644 tests/test_video_catchup.py create mode 100644 tests/test_video_input_limit.py create mode 100644 tests/test_video_lazy_rgba.py create mode 100644 tests/test_vk_pgo.py create mode 100644 tests/test_wait_hold_pause.py create mode 100644 tests/test_warm_computes.cpp create mode 100644 tests/test_warm_computes.py create mode 100644 tests/test_wb_excl_clip.cpp create mode 100644 tests/test_wb_excl_clip.py create mode 100644 tests/test_wb_forget_order.py create mode 100644 tests/test_work_lock.py create mode 100644 tests/test_wrote_regions.py create mode 100644 tests/tex_count_test.cpp create mode 100644 tests/ub_update_batch_test.cpp create mode 100644 tests/vbytes_test.c create mode 100644 tests/vecfp_test.c create mode 100644 tests/video_acquire_test.c create mode 100644 tests/video_fixture.py create mode 100644 tests/video_play.h create mode 100644 tests/video_standins.h diff --git a/builder/HookshotBuilder/Runner.cs b/builder/HookshotBuilder/Runner.cs index 06fe9d6..174a7ef 100644 --- a/builder/HookshotBuilder/Runner.cs +++ b/builder/HookshotBuilder/Runner.cs @@ -130,7 +130,12 @@ sealed class Runner process = p; p.ErrorDataReceived += (_, e) => { if (e.Data != null) Line?.Invoke(e.Data); }; p.BeginErrorReadLine(); - try { p.PriorityClass = ProcessPriorityClass.BelowNormal; } catch (Exception) { } + // (Triforce, 11-22, the performance audit's OS-57, after Source A's) The build tools run at normal priority; + // below normal, as every build was before, only with HOOKSHOT_LOW_PRIORITY=1. Source A's reason, a report + // it did not measure: below normal, Windows kept the steps that are one process (the read, the lift) on a + // hybrid Intel CPU's efficiency cores while the performance cores idled. + if (Environment.GetEnvironmentVariable("HOOKSHOT_LOW_PRIORITY") == "1") + try { p.PriorityClass = ProcessPriorityClass.BelowNormal; } catch (Exception) { } while (p.StandardOutput.ReadLine() is { } line) { if (line.StartsWith("@@error ")) error = line[8..]; diff --git a/builder/py/hookshot_builder/core.py b/builder/py/hookshot_builder/core.py index c2b6157..aa6307a 100644 --- a/builder/py/hookshot_builder/core.py +++ b/builder/py/hookshot_builder/core.py @@ -17,6 +17,15 @@ CREATE_NO_WINDOW = 0x08000000 BELOW_NORMAL_PRIORITY_CLASS = 0x00004000 +def build_priority(env=None): + """(Triforce, 11-22, the performance audit's OS-57, after Source A's) The priority class the build tools are started + with, as a creation flag: none (normal priority), and below normal, as every build was before, only with + HOOKSHOT_LOW_PRIORITY=1. Source A's reason, a report it did not measure: below normal, Windows kept the steps that are + one process (the read, the lift) on a hybrid Intel CPU's efficiency cores while the performance cores idled.""" + env = os.environ if env is None else env + return BELOW_NORMAL_PRIORITY_CLASS if env.get("HOOKSHOT_LOW_PRIORITY") == "1" else 0 + + # ---------------- the build tools (Runner.cs: Tools) ---------------- class Tools: @@ -96,7 +105,7 @@ class Runner: p = subprocess.Popen(build_command(self.tools, self.args), cwd=self.tools.root, env=build_env(self.tools, self.workers, self.ngx_sdk), stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.DEVNULL, - creationflags=CREATE_NO_WINDOW | BELOW_NORMAL_PRIORITY_CLASS) + creationflags=CREATE_NO_WINDOW | build_priority()) except OSError as e: self.on_finished("The build tools could not start: %s" % e) return diff --git a/docs/design.md b/docs/design.md index 2f6292c..1bad690 100644 --- a/docs/design.md +++ b/docs/design.md @@ -263,6 +263,6 @@ dump, the runtime made game-neutral, the stage-1 tools checked on MKW's dump as repeat with a delay that keeps ~45 fps. The MKW session's mirror page skip fired 0 times here and was removed. Not an instrument's doing: with nothing attached, photographed from outside the window (37 s, ~2200 photographs): 16 with `RX_VK_FRAMEGEN=0`, none at the user's 4x. With `RX_VK_SUBMIT_DELAY_US`: 7 at 300, 0 at 1000; a repeat at 1000 with shots still had 7 events against 50-60. So it is a race with the game's threads: less overlap, fewer. The delay is a test, not a fix. -- The builder's first full build: 43 minutes at below-normal priority, 0 failures; the folder starts from a double click and reaches the title at 60 fps. +- The builder's first full build: 43 minutes at below-normal priority (the builders' start then; since the performance audit's OS-57 normal, below normal with `HOOKSHOT_LOW_PRIORITY=1`), 0 failures; the folder starts from a double click and reaches the title at 60 fps. - Render conditions here gate single lights and read reports earlier in the same stream, so they cannot be read at record time. diff --git a/docs/development.md b/docs/development.md index 46ab189..7d0fb42 100644 --- a/docs/development.md +++ b/docs/development.md @@ -1,7 +1,7 @@ # Hookshot (oot-recomp) **Hookshot** is the port's name (as Skybound is TOTKPort's and BlueShell is mkw-recomp's); the maintainer's repository -folder stays `oot-recomp`. The game builds as `hookshot.exe` + `hookshot_vk.dll` (`hookshot` + `hookshot_vk.so` on Linux). +folder stays `oot-recomp`. The game builds as `Triforce.exe` + `triforce_vk.dll` (`triforce` + `triforce_vk.so` on Linux). A static recompilation of the Switch 2 *The Legend of Zelda: Ocarina of Time* remake to native x86-64 Windows and Linux: the game's code lifted to C (NotRexed's a64-kit, Rexcomp's AArch64 kit), the Horizon OS and NVN2 implemented @@ -79,21 +79,41 @@ screen. It costs some GPU time on cards that cull quickly, so it is off unless s | `HOS_WINDOW_SIZE=x` | The window's size in pixels (1280x720 by default, wider for an ultrawide picture; 64 to 16384 a side) | | `HOS_WINDOW_NOACTIVATE` (set) | The window opens behind the others and does not take the focus (Windows) | | `HOS_FULLSCREEN=1` | Starts in fullscreen; the Insert menu's Fullscreen row is then left alone | +| `HOS_ALT_MENU=1` | (Triforce, 11-22, after Casual Gamer's #dev-fixes post) A lone Alt is left to Windows, as before: it enters the menu loop on the window's thread, which during the start-up screen is the one waiting for the shaders, and the screen froze until a key was pressed. By default the window, which has no menu bar, answers that command itself (`host/display.c` `keymenu_dropped`, `tests/test_alt_menu.py`); Alt+Space's system menu, Alt+Enter, F10 and Alt+F4 are as they were | | `HOS_AUDIO=0` | No sound device is opened (the mixer still runs); `scripts/run-vk.ps1` sets it unless `-Sound` | +| `HOS_AUDIO_EFFECTS=0` | The game's plain reverb, delay and biquad filter effects on its mixes and its splitter destinations' biquad filters are accepted and left out, as before they were emulated (the sound unfiltered and at full level: the "Volcano eruption audio bugged" ticket); the I3DL2 reverb still runs. By default they run on the mix buffers in the order the game added them (`host/nn_audio.c` "mix effects"; `tests/test_audio_mix_effects.py`) | | `HOS_LANGUAGE=` | The game's language for this run, a code from `language.txt` | | `HOS_OUTPUT_RES=<0..3>`, `HOS_HANDHELD=1`, `HOS_ASPECT=` | Over the options file: the output resolution (0 2048x1152, 1 2560x1440, 2 3840x2160, 3 1920x1080), the handheld profile, a picture wider than 16:9 (2.3889 for 3440x1440; the Ultrawide row's 21:9 is 2.3704, 64:27, and its 32:9 3.5556) | | `HOS_DYNRES=0`, `HOS_REALTIME_STEP=0`, `HOS_TEXT_SPEED=<0..3>` | Over the options file: Resolution 100% (non-zero: Auto), Game speed Per frame (1: Real time), Text speed | +| `RX_VK_FIRST_START_AUTO=1`, `RX_VK_FIRST_START_AUTO=` | (Triforce, 11-22, the performance audit's PRESENT-46, after Source A's; off by default) A first start sized to the card: with no options file, `RX_VK_LOW_SPEC` not asked and `HOS_DYNRES` not set, a weak card starts with the Render resolution row at 50% in place of Auto, or at 33% for an integrated GPU beside 4 processor threads or fewer (`gpu/vk/first_start.h`, `gpu/vk/vk_menu.cpp` `menu_first_start`, called from `mkwvk_init` once the device is made). Weak is Source A's tier 1: an integrated GPU (a handheld's is one; the `SteamDeck` variable is not read, it took desktops for handhelds), a card without mesh shaders, or 4.5 GB of video memory or less. Only that row is set: the upscaler, the textures and the output stay, and nothing is saved, so each start chooses again until the player saves options. A number over 1 plays a card of that many MB (the lab: `=4096` on a large card). Not set or `=0`: as before. Off until the owner decides what a first start on a weak card gets; unmeasured (Source A states no gain and tried it on one RTX 4070 Ti). One `menu: first start:` line says the choice | | `HOS_STEP_STEADY=0` | (Triforce, on by default since the lab's A/B; `=0` the mean of the last five as before) The game's real-time step kept steady, after NX2navi's steady step (`host/step_steady.h`, `host/nn_vi.c` `realtime_step`): the mean of the last 12 present intervals without their longest and shortest in place of the mean of the last five (`HOS_REALTIME_WINDOW`), the paced period itself where that mean is within 3% of it, and what the display showed and the game was not given repaid at a 64th a frame, 10% of the step at most (a quarter second owed at most). A cap the game holds steps it by the period exactly, and a hitch is paid back over most of a second instead of in the next five frames (a 50 ms frame at 240: 200 frames of at most 10% more, where the mean of five gave five steps of 13 ms). Game time still follows the clock | +| `RX_VK_FG_KEEP=1` | (Triforce, 11-21, the performance audit's PRESENT-18) Frame generation's images are kept once made, as before. The default lets them go after 120 presents in a row with frame generation off (about two seconds at 60, so a row stepped through keeps them): the built-in interpolation's frames and made-frame sets, which stayed in VRAM until a resize after it was turned off or left for NVIDIA's own, and AMD FSR 4 frame generation's four images of the picture without the HUD once that row is left. The display thread is drained and the GPU idle first; they are made again at the next frame that needs them. The log says how many MB went (`gpu/vk/framegen_idle.h`, `vk_framegen.cpp` `framegen_idle`) | +| `RX_VK_CAPTURE_TABLES=0` | (Triforce, 11-21, the upscaler audit's UPS-02) The stored draws' two tables as `std::unordered_map`, emptied and filled every frame, as before. While NVIDIA DLSS, AMD FSR 4 or AMD FSR 3.1 upscales, every plain draw into the scene stores its positions and is drawn again into the motion vectors; the count of a model's draws and the frame before's draws by their key took a node from the heap for every stored draw every frame (about 1,400 of each in the owner's log, whose status line said 0.65 ms of CPU a frame for the stored draws in all). The default keeps their memory from frame to frame, so a frame the size of the one before takes nothing from the heap, and gives the next frame's list of draws its room at once. The status line's `ms CPU a frame` is the measure (`gpu/vk/capture_tables.h`, `vk_temporal.cpp` `capture_motion`). Not built or run in the game when it was written | +| `RX_VK_REDRAW_STATE=0` | (Triforce, 11-21, the upscaler audit's UPS-03) Every stored draw drawn again into the motion vectors sets the primitive topology, the primitive restart and its index buffer before it draws, as before. The default sets each only where the draw's differs from what the pass has set so far (its first draw sets all three): about 900 draws a frame are drawn again in the owner's log, nearly all of them triangle lists without restart, so two commands a draw are left out, and the index buffer's where two draws in a row bind the same one at the same offset (`gpu/vk/redraw_state.h`, `vk_temporal.cpp` `capture_motion`). Not built or run in the game when it was written | +| `RX_VK_DLSS_MODE_SLACK=0` | (Triforce, 11-22, the performance audit's PRESENT-38) DLSS's quality mode found strictly, as before: a render size is a mode's only inside the range NGX answers. Not set: a size outside a range by a rounding (a 64th of the range's largest size, 2 pixels at least: `gpu/vk/dlss_checks.h` `mode_for`) is that mode's, so the Render resolution row's 33% at 2560x1440 (854x480, NGX's is 853x480) draws with DLSS and not the game's own upscale; and the feature there is is let go only once a mode took the new size. The lab's `dlssstrict` phases switch it within a run | +| `RX_VK_DLSS_MODEL_SETTLE=` | (Triforce, 11-21, the performance audit's PRESENT-37) The DLSS model row steps K, M, L, NVIDIA's default and applies live: going from K to L made a feature for M on the way, tens of ms each, and NGX keeps a released feature's video memory for some seconds. A new preset is taken once the row has held it for `` frames, 30 when not set; the feature already made draws on meanwhile, and with no feature the row's preset is taken at once. `0`: at once, as before (`gpu/vk/dlss_checks.h` `PresetSettle`, `vk_dlss.cpp` `prepare`) | +| `RX_VK_DLSS_CNN_VRAM=1` | (Triforce, 11-22, the performance audit's PRESENT-36, after Source A's and Source B's; off by default) On a card with less than 7 GB of video memory the DLSS model row's K (the default) asks NGX for preset E, NVIDIA's last CNN model, and the row reads "CNN (E)" (`gpu/vk/upscaler_rows.h` `cnn_for_vram`, `gpu/vk/vk_dlss.cpp` `wanted_preset`, `dlss_cnn_vram`). M, L and NVIDIA's default stay as chosen, `RX_VK_DLSS_PRESET` goes over it, and cards of 7 GB and more are not touched. Source A measured it on a 6 GB laptop RTX 3050 (1080p, DLSS Performance, one run a preset): 15.75 to 14.20 ms of GPU a frame, NGX's own memory 85 to 58 MB. Off here, as in Source B: preset E ghosts more on grass and thin objects at the smallest render sizes and NVIDIA marks it deprecated, so the picture is the owner's to look at first. A DLSS runtime without E falls back to NVIDIA's default; `RX_VK_DLSS_LOG=1` shows the preset NGX took. One `vk: NVIDIA DLSS: ... MB of video memory: the K row draws with preset E` line says it is in use; `RX_VK_TEST_CARD_MB=6144` plays a 6 GB card | +| `RX_VK_DLSS_FEATURE_MAX=1` | (Triforce, 11-21, the upscaler audit's UPS-01) NVIDIA DLSS's feature made for the largest render size its quality mode allows, whatever the Render resolution row says, as before. NGX answers that largest size as the output's own, so at a fixed share the feature was made for 2.25 times the pixels it was ever given at Quality (4 at Performance, 9 at Ultra Performance), and the chain's colour, depth and motion images with it; the owner's log has DLSS's own video memory at 1069 MB for such a feature with preset M, where NVIDIA's table has 456 MB at 4K, and NVIDIA's guide says the cost of presets L and M follows the size the feature is initialized for. The default makes a fixed share's feature for its own render size; the row's Auto (the game's dynamic resolution), and any run in which the render size is seen changing under one output, make it for the mode's whole range as before (one feature made again when that is first seen). DLAA's is the output's either way. The `feature N made` line says which, and the status line's `its own ... MB` and `its images ... MB` are the measure (`gpu/vk/dlss_feature.h`, `vk_dlss.cpp` `prepare`). Not built or run in the game when it was written | +| `RX_VK_DLSS_PREWARM=1` | (Triforce, 11-22, the performance audit's PRESENT-32, after Source A's: its first step; off by default) NVIDIA DLSS at the output's size takes the scaled quality mode whose range holds that size, and only NVIDIA DLAA the one-size mode (`gpu/vk/dlss_checks.h` `dlaa_feature`, `gpu/vk/vk_dlss.cpp` `prepare`). With the Render resolution row at Auto the game starts at the output's size and lowers it some frames later: before, a one-size feature and then a second one made in play (Source A's save load-in: 119 ms, then 29.5 ms 17 frames on); with the switch one feature, for its mode's whole range. Off by default because DLSS at full size then draws with the scaled mode's network, not DLAA's: a picture to compare first. The rest of the entry is not built: making the feature, the chain's images and a first evaluation ahead in a light frame (a title or loading screen) needs the first gameplay upscale's sizes and colour kind before it has happened, which only a run with DLSS can settle. `tests/test_dlss_prewarm.py` runs the decisions over a load-in | +| `RX_VK_FRAMEGEN_REFRESH=1` | (Triforce, 11-21, the upscaler audit's UPS-05; off unless set) The Frame generation row's 2x to 8x make no more frames for each of the game's than the display's refreshes that fall in one frame of the game: the game's recent frame time over the display's period, rounded up, a tenth of a refresh forgiven, never more than the row's number and never fewer than the game's own frame. Without it the row's number is made whatever the display shows: the owner's log has 5x on a 165 Hz display with the game near 66 frames a second, 331 frames a second presented, each present call 2.2 to 3.0 ms (0.03 ms without frame generation) and the display thread's line `0 frames shown on time, 1655 late`. Auto and AMD FSR's row are not touched. A `vk: frame interpolation: N frame(s) for each of the game's` line says each change (`gpu/vk/framegen_refresh.h`, `vk_framegen.cpp` `framegen_record`). Not built or run in the game when it was written | | `RX_VK_FRAMEGEN=0`, `RX_VK_FRAMEGEN_MULT=<0..9>`, `RX_VK_PACE=0`, `RX_VK_UPSCALE=<0..2>` | No frame generation whatever the menu says; over the options file: the Frame generation row's value (0 Off, 1 Auto, 2 to 8 the multiplier, 9 AMD FSR frame generation: a lab run's pin for each generator; `gpu/vk/framegen_rows.h` `env_row`), frame pacing, the upscaling filter | | `RX_VK_FG_OWN_MOTION=0` | (Triforce, FG-4) Frame generation's motion the camera's alone, as before, for comparing. Without it, while NVIDIA DLSS or AMD FSR 4 upscale, the depth and motion frame generation takes (AMD's, NVIDIA DLSS frame generation, FSR 3.1's interpolation) are made from the upscaler chain's frame, with what moves by itself drawn into the motion (`gpu/vk/vk_temporal.cpp` `fg_inputs_from_chain`, `gpu/vk/vk_fsr3_inputs.cpp` `chain_motion`): also where the gameplay network does not translate on the card and the upscaler draws it in its place (RDNA 3), where they were never made before and AMD's frame generation made no frame | +| `RX_VK_DLSSG_OWN_MOTION=1` | (Triforce, 11-22) NVIDIA DLSS frame generation is given the upscaler chain's motion, as before (`RX_VK_FG_OWN_MOTION` above). By default it is given the camera's alone, from FSR's inputs: with the chain's the owner's flicker of the whole picture while the camera moved came back on 11-22, with frame generation at 2x and DLSS at 50%, and it stayed with the upscaler audit's commits set back; with `RX_VK_FG_OWN_MOTION=0` it was gone. DLSS frame generation finds what moves by itself with its own optical flow. AMD's frame generation and FSR 3.1's interpolation keep the chain's (`gpu/vk/fg_motion.h`, `vk_fsr3_inputs.cpp` `chain_motion`). Not built when it was written: the same as the run with `RX_VK_FG_OWN_MOTION=0`, for DLSS frame generation alone | +| `RX_VK_DLSSG_CLIP=0` | (Triforce, 11-20) NVIDIA DLSS frame generation is given the game's own camera matrices, as before, for comparing. Without it they are in the clip space of the depth it is given (FSR's copy, 1 at the near plane): the game's clip z runs from -1 at the near plane to 1 at the far one, and with its matrices that depth read as a clip z put what is far next to the camera, which showed as a flicker of the whole picture while the camera moved (`gpu/vk/dlss_math.h` `depth_clip_projection`, `gpu/vk/vk_dlssg.cpp` `frame`) | | `RX_VK_FG_HUDLESS=0` | (Triforce, FG-2) No picture without the HUD is taken for AMD's frame generation, so its library makes no frame (the game's frame alone): a test. Without it the gameplay upscale's output is taken before the HUD is drawn into it, sRGB-encoded as the game's last pass encodes it, and given as `HUDLessColor` (`gpu/vk/vk_framegen.cpp` `framegen_hudless_note`, `hudless_for`); a present without it shows the game's frame alone; with HDR Native or RenoDX's RGBA16F scene AMD's library is not asked and the renderer's own makes the frame, as at 2x | | `RX_VK_FRAMEGEN_FINE=0` | Frame generation's own made frames (Auto, 2x to 8x) the old way: a vector per 16x16 pixels in steps of 4 pixels, a blend of four of them per pixel, a pixel alike in both frames taken as still. Without it, a vector per 8x8 pixels to a fraction of a pixel, settled among its neighbours, and one motion per pixel (`gpu/vk/vk_framegen.cpp` `fine_vectors`, `shaders/fg_fine_*.comp`), which keeps moving straight lines straight and thin lines drawn (`tests/test_framegen_lines.py` measures both) | | `RX_VK_PACE_SMOOTH=0` | (Triforce, on by default since the lab's A/B; `=0` off) The Frame pacing row on the plain present too, without frame generation: with the row on, a picture of the game's that comes sooner than seven eighths of the recent interval after the last one waits for that, its blit on the queue and the queue's lock let go; a late one goes at once, and none waits more than 50 ms (`gpu/vk/present_smooth.h`, after NX2navi's `pace_smooth`). Without it the row spaces only frame generation's frames, as before. With `RX_VK_PRESENT_THREAD=0` the present is on the game's presenting thread, so a wait also holds that thread's next frame: what that does to the rate under the cap is for a measurement before it is on. A `vk: frame pacing` line every 1200 pictures says how many waited and how long | | `RX_VK_PRESENT_THREAD=1` | (off by default since the lab showed no gain) (Triforce, on by default; `=0` as before) The game's presents go out from a present thread of their own (`gpu/vk/present_thread.h`, after NX2navi's present thread): the presenting thread records and submits the blit as before (the frame fence, the swapchain's image, the overlays, the bug capture's copy and the frame log's: the same commands in the same order on the queue), then hands the pacing's wait (`RX_VK_PACE_SMOOTH`) and the present call to that thread and goes on to its next frame; the next present waits first until the one handed over is out, so one is in hand at most. Not with frame generation, NVIDIA DLSS frame generation or AMD Anti-Lag 2 (presents and stages of their own); NVIDIA Reflex's present markers carry their frame's id. The `vk: a frame's present` line every 120 presents says how many went out from it, the presenting thread's time in the present and how much of it was the wait for the present before, and the pacing's wait and the present call (on whichever thread); the stall line's present is the presenting thread's, with that wait named | +| `RX_VK_ACQUIRE_FENCE=0` | (Triforce, on by default; `=0` as before) The swapchain's image waited for on the CPU, not on the replay's queue (`gpu/vk/vk_replay.cpp` `present_image`, `image_free`): the acquire signals a fence, the present waits for it, at most 250 ms, before it records anything for the image, and the blit's submit waits for nothing. AMD's Windows driver returns from the acquire at once, the image not yet free: with the window covered (Alt+Tab, another window or OBS in front) the blit's wait for the acquire held the queue, every later submit queued behind it, and the game froze for as long as the cover. An image not free in time is held (`vk: swapchain image ... held`): no frame is shown, the game goes on, and the next presents look at it again without waiting until it is back. The wait counts as the pacing's in the present line. With DLSS frame generation the acquire's semaphore as before | +| `RX_VK_SURFACE_RECOVER=0` | (Triforce, 11-22, the performance audit's PRESENT-45, after Source A's; on by default) A lost window surface is not handled, as before: after `VK_ERROR_SURFACE_LOST_KHR` (a display driver reset, a monitor change, a remote session) every acquire fails without a word and the picture stays frozen until a restart. Not set, an acquire, a present or a swapchain's creation that answers it lets go of what hangs on the surface, in order: the present thread's present out, the device idle, the overlays' views, the images' semaphores, the swapchain, then the surface (`gpu/vk/vk_main.cpp` `invalidate_lost_surface`); and the next present's `create_swapchain` first makes a surface for the window again with the startup's calls (`open_window_surface`), only while the window has a size (a minimized window waits), one try each 250 ms at most, with the presents' own queue chosen again for it. A loss seen on the present thread is only noted there and handled by the next present. Nothing changes while no surface is lost (every new statement is behind that result or a missing surface, and the startup's own code is as it was), nor while NVIDIA DLSS frame generation runs (the swapchain is Streamline's). `vk: the window's surface is lost` and `... is made again` lines say it. Never seen on a real driver here, and the window's path is not in the scripted (headless) runs: `tests/test_present_surface_lost.py` runs the code against a model of a driver that loses its surface at each call | | `RX_VK_REFLEX_LIMIT=1` | (Triforce, off by default) With the NVIDIA Reflex row on and no frame generation, Reflex's sleep holds the Game frame rate cap (the sleep mode's minimum interval) where each frame starts, in place of the game's own wait; and with DLSS frame generation, Streamline's Reflex holds it the same way (`gpu/vk/vk_dlssg.cpp` `pace`; options_version 7: the Reflex frame limit row is gone, and this is its only switch until that path has been played on an RTX 40/50 card; `vk: dlssg: frame limit:` lines say who holds it): one wait a frame, timed by the driver (`gpu/vk/vk_reflex.cpp` `reflex_pace`, the rule `gpu/vk/dlssg_pace.h`'s; after NX2navi's `low_latency`). The game's own wait holds a movie, a frame RivaTuner holds, a window that showed nothing for 200 ms, and the rest of the run once over half of 60 frames came sooner than three quarters of the period, or when the driver refuses the interval. `vk: NVIDIA Reflex: frame limit:` lines say who holds it; the Full overlay's Frame limit line says NVIDIA Reflex after the rate. | | `HOS_PACE_HANDOFF=0` | The game's own wait paces every frame, with DLSS frame generation's Reflex too and whether RivaTuner Statistics Server runs: the renderer is never asked whether another limiter holds a frame, and RivaTuner gets no limit from the game (`host/nn_vi.c` `pace_handed`, `gpu/vk/vk_pace.cpp`) | | `HOS_MEM_QUERY_USED=0` | (Triforce, 11-19) `nn::os::QueryMemoryInfo` answers the memory in use as the program and the system's part only, as before. By default it is the program, the heap and every page the game mapped beyond it, at most the total, as Horizon answers (`host/hos_mem.c` `query_used`): the game's resource cache sizes what it may still take from it, and without it kept every area it ever loaded (12.5 GB of its 9809 MB after four areas). Needs the census on (`HOS_MEM_CENSUS`). `HOS_MEM_QUERY_SLACK_MB=` (1024 by default): that much less of the game's pages answered as in use, room for its cache beyond a Switch's (our game needs ~3.2 GB of regions at a save load where a Switch leaves ~2.8; with 0 it evicted in every scene) | +| `HOS_OBJ_CACHE=` | (Triforce) The per-thread cache that finds a guest nn::os object (mutex, event, condition variable ...) by its address (`host/nn_os_sync.c` `obj_lookup_cached`, `tests/test_obj_cache_size.py`): by default 16,384 entries, four ways to a 64-byte set picked by murmur3's finalizer, the oldest way replaced; 256 KB a thread, allocated at its first lookup. The direct-mapped cache missed at nearly every step of the game's walk over some 4,000 mutexes 0x80 apart. n: 256 to 65,536 entries (down to a power of two); `0`: the direct-mapped 4,096 entries, as before. Every 10 s the log's `sync: object cache` line counts the lookups and misses | +| `HOS_NO_OBJ_CACHE=1` | No per-thread object cache at all (either kind): every nn::os call looks its object up in the shared table | +| `HOS_GUEST_MEMFD=0`, `=1` | (Linux) The guest window as a memfd's, its pages handed to the GPU as dma-bufs (udmabuf), where `/dev/udmabuf` can be opened: amdgpu checks every page of a host-pointer import again at each submission, 15-45 ms a submission in Kokiri. (Triforce, 11-21) Also where udmabuf's `size_limit_mb` cannot be read (Flatpak and other sandboxes hide `/sys/module`): its default, 64 MB, is assumed and the log says so; only a limit that reads 0 keeps the window anonymous. The log's `mem: the guest window is a memfd's` line. `=0`: anonymous memory, host-pointer imports; `=1`: the memfd all the same | | `HOS_MEM_TRACE=1` | (Triforce, lab) Logs every address region and page call of the game with its size and caller (`memtrace:` lines; a region's with its call stack) and every `QueryMemoryInfo` answer | +| `RX_VK_WB_EXCL_CLIP=0` | (Triforce, 11-22; on by default with `RX_VK_CAP_ONLY_GPU=1`) A written binding is copied back to guest memory whole at the submission's end, as in 0.6.0. By default the parts of it in memory the capture leaves out (heap memory between the game's pools inside an imported chunk) are not: the collection does not stamp the CPU's writes there, so the mirror's bytes there are stale, and the write-back diff copied them over the game's newer ones. A pointer table's bare entry (its 64-byte head marked written, `gpu/vk/vk_draw.cpp`) did so over a heap block 0x18 past its base: the 0.6.0 crashes at area changes in guest 0x803b44e0 (a block header of 0x9ed00000001 or 0x1aa88000013fe), 0x80a30a00 and 0x80a31030 (`gpu/vk/wb_excl_clip.h`, `gpu/vk/vk_memory.cpp` `gpu_buffer_bind`; `tests/test_wb_excl_clip.py`). The first 20 clipped write-backs are logged (`vk: write-back of ... not copied back`) | | `HOS_FPS_STEADY=0`, `=1`, `=2` | (Triforce, off by default: the Frame pacing row's two Steady values, over them) 1 any rate, 2 steps of the display's refresh: the game's frames paced to a period this PC holds in the scene, never shorter than the Game frame rate cap's (`host/pace_steady.h`, `host/nn_vi.c` `steady_period`). The rate is chosen by probes (`pace_steady_probed`): 32 frames at the cap, unpaced, the last 24 of them measured, at the start, 2 s after a load, at least every 12 s, and when a quarter of the last 64 frames ran late (2 s after the last probe at the soonest, twice that each time a probe changes nothing, up to 12 s); the rate becomes the first the last three probes' 90th percentile fits in with 2% to spare where that is slower, or the first it fits in with 5% to spare where that is faster. A frame paced slower does more work (the lab's Market: 11.6 ms unpaced, 15.5 ms paced at 55 fps), so the frames' own measure as they run chased the rate down; frames late now only bring a probe on, and the rate is never under what a probe shows the PC holds. `HOS_FPS_STEADY_PROBE=0`: no probes, the rule below as before. Without probes: what a frame took is its present interval less the longest of the pacing's waits in it (the acquire's, on the presenting thread or another, and the present's hold), and what it needed the interval less all of them (OoT's main thread and Presentation Thread take turns; where the waits add up to the whole interval, the longest: `pace_steady_measure`); a quarter of the last 64 frames that took more than the period lowers it at once to the first rate their 90th percentile fits in with 2% to spare, and once it has held 3 s since a raise (6 s since a lowering, doubled for each lowering within 30 s of a raise, up to 120 s, and back to 6 s at a load) it rises to the fastest rate the 90th percentile of what the last 512 frames needed fits in with 5% to spare (one rate at a time where the game acquires on another thread). A frame that took over 2.5 times the median of the last 64 (hitches among them), a present interval over 250 ms, and the frames of the 2 s after such a load (the scene is not its measure yet) are left out; the first lowering after a load waits for 128 frames of the new scene and takes their 90th percentile. The acquire's wait starts the frames a period apart (the steady step then gives the game the period as its time step) and the present holds each picture of the game's to the period after the last, less a 64th of it (`gpu/vk/present_smooth.h` `present_smooth_paced_at`, with or without the Frame pacing row; not with frame generation). Another limiter holding the frames (RivaTuner, Reflex) puts the cap back while it does. `vi: steady frame rate:` lines at each change say what the frames needed, and every 5 s what the rule was given (what the frames took and needed at p50 and p90, the present interval's p50, the waits a frame by kind, the frames kept and left out by reason, how many of the last 64 kept were late, and the probes and what the last three took unpaced); the `vk: frame pacing` line counts the present's holds | | `HOS_FPS_STEADY_PROBE=0` | (Triforce, with `HOS_FPS_STEADY` or the Steady frame rate row) No unpaced probes: the rate by the frames' own measure as they run, as before test31 (`host/pace_steady.h` `pace_steady_probed`, `host/nn_vi.c` `steady_period`); the frames' work grows when they are paced slower, and that rule slides down | | `HOS_PACE_SPIN=0`, `HOS_PACE_SPIN=` | (Triforce) The game's own frame wait on the high-resolution timer for the whole wait, as before, which wakes up to about 0.5 ms after the frame's time. By default the timer takes the wait up to 0.6 ms before the frame's time and the rest is yielded away on the clock, so the frame starts on its time (`host/pace_wait.h`, `host/nn_vi.c` `pace_sleep`); a number is another margin. Every 5 s in which frames waited a `vi: frame waits` line says how long after their time the waits ended, on average and at most | @@ -146,11 +166,16 @@ case-sensitive, and an unknown one is ignored. | `HOS_THREAD_TIMES=` | Every interval, the threads that used over 5% of a core | | `HOS_FPS_LOG` (set), `HOS_PRESENT_TRACE=` | The game's frame limiter, VRR flags and nominal rate every 300 presents; a CSV line per game present | | `RX_VK_FRAME_LOG=` | A CSV line per present: frame, seconds, frame time in ms | -| `RX_VK_FRAME_TRACE=1` | A line a frame in `triforce_frames.csv` beside the game (the lab's scenarios keep it as `frames.csv`): present to present, the waits for shaders and pipelines, the uploads, and the replay's parts (`memsync_ms` ... `tex_lookup_ms`: the draws' parts are sampled and summed over every thread that records). (Triforce, wp20, `gpu/vk/frame_trace.h`) Then each frame split as the stall line splits a long one: `replay_ms` (the submits and the present phase), `present_ms`, `dispatch_ms`, `qsubmit_ms`, `par_wait_ms` (the replay thread's waits for its helpers' segments), `outside_ms` (the frame less the replay and the present: the game's own update), `game_wait_ms` (the game's fence waits), `captures_ms` (the captures at its submits), `wb_wait_ms` and `wb_held_ms` (the waits for write-backs, and their part after the writer's recording was done: `RX_VK_WB_WAIT_REC`), `qs_bg_ms` (the queue submits beside a background import). `python -I scripts/frame_rank.py ` ranks what made the frames over the 99th percentile and over 1.5x the median by these, per window (the Market, the tour's stays, the loads) | -| `RX_VK_WB_WAIT_REC=1` | (Triforce, off by default; the handoff's §12.9, `gpu/vk/wb_rec_wait.h`) A wait for a write-back still on the GPU (a capture's late pages on the game's submitting thread, a read or binding of a page put off) ends once the recording its submission was finished in is done on the GPU, not when the submissions' timeline reaches it: that timeline is held below any submission stopped at an event wait, and the lab's Market had 20-46 ms waits of the game's thread with the GPU 4 submissions behind. Whatever the switch, the wait for the recording comes first and the rest is counted: the stall line's `N of them X ms after their writer's recording was done`, the `late:` line's `ms of the wait after the recording that finished it was done`, the frame trace's `wb_held_ms`. Off: the captures' late-page wait orders something not known yet (`RX_VK_LATE_NEW`), so it is tried with the lab's flicker count first | +| `RX_VK_FRAME_TRACE=1` | A line a frame in `triforce_frames.csv` beside the game (the lab's scenarios keep it as `frames.csv`): present to present, the waits for shaders and pipelines, the uploads, and the replay's parts (`memsync_ms` ... `tex_lookup_ms`: the draws' parts are sampled and summed over every thread that records). (Triforce, wp20, `gpu/vk/frame_trace.h`) Then each frame split as the stall line splits a long one: `replay_ms` (the submits and the present phase), `present_ms`, `dispatch_ms`, `qsubmit_ms`, `par_wait_ms` (the replay thread's waits for its helpers' segments), `outside_ms` (the frame less the replay and the present: the game's own update), `game_wait_ms` (the game's fence waits), `captures_ms` (the captures at its submits), `wb_wait_ms` and `wb_held_ms` (the waits for write-backs, and their part after the writer's recording was done: `RX_VK_WB_WAIT_REC`), `qs_bg_ms` (the queue submits beside a background import), `gpu_waits_ms` ((Triforce, 11-22, the performance audit's SYNC-27) every thread's time in calls that block on the GPU, `RX_VK_GPU_WAITS` below). `python -I scripts/frame_rank.py ` ranks what made the frames over the 99th percentile and over 1.5x the median by these, per window (the Market, the tour's stays, the loads) | +| `RX_VK_SETTLE_GATE=1`, `=2` | (Triforce, 11-22, off by default; the performance audit's SYNC-29, after Source A's; `gpu/vk/settle_gate.h`) A CPU read that lets the stopped queues whose waits are met go on (the readback's look, which `RX_VK_RB_SETTLE=1` turns on, and the two looks made for a range a texture may be in) takes the order's lock whenever a queue is stopped, and that lock is held through whole replays and presents. With the gate a site looks only while a queue is stopped **and** the CPU has signalled an event since the last finished look, or a stopped queue waits on an event whose value is read from guest memory; any other met wait was met by a signal in the stream, and the thread that replayed it looked before it let the lock go. `1`: the readback's look; `2`: the texture sites' too (new ground: those looks order the exposure meter's readback, so `2` needs the lab's flicker count). Unset or `0`: every look as before. The `RX_VK_LAB_AB` knobs `settlegate` and `texgate` are `1` and `2` in their phases (`rbsettle+settlegate` is the pair the audit asks for). The 3000-readback line and the texture catch-ups' 300-call line count the looks made and skipped | +| `RX_VK_READBACK_LATE=[,]`, `RX_VK_LOG_READBACK_RANGES=1` | (Triforce, 11-22, the performance audit's SYNC-19, after Source A's; `gpu/vk/readback_late.h`) Ranges the CPU reads back **without** the wait for their write-back, from what guest memory holds (the last write-back made): a rule is a size in hex (`3980`: every range of 0x3980 bytes, the culling camera block Source A reads late by default) or `newest` (a range whose writer is the newest submission handed out: the game cannot have waited for it). **Unset: no rule**, every readback waits as before, until the lab has run it where the waits are long (a handheld, an AMD iGPU, a weak discrete card); `=0`: none either. The `RX_VK_LAB_AB` knob `rblate` is `3980` in its phases. A late read hands the game the frame before's values, and when the CPU resets a range read late the pending write-back lands over its reset (the exposure meter's flashes): `newest` can catch that meter, so it is never a default. `RX_VK_LOG_READBACK_RANGES=1`: with every 3000-readback line, the 12 ranges that waited longest (readbacks, waits, ms, how often their writer was the newest submission, how often read late), to find such ranges on other hardware; nothing is noted without it | +| `RX_VK_WB_WAIT_REC=1`, `=2` | (Triforce, off by default; the handoff's §12.9, `gpu/vk/wb_rec_wait.h`) A wait for a write-back still on the GPU (a capture's late pages on the game's submitting thread, a read or binding of a page put off) ends once the recording its submission was finished in is done on the GPU, not when the submissions' timeline reaches it: that timeline is held below any submission stopped at an event wait, and the lab's Market had 20-46 ms waits of the game's thread with the GPU 4 submissions behind. Whatever the switch, the wait for the recording comes first and the rest is counted: the stall line's `N of them X ms after their writer's recording was done`, the `late:` line's `ms of the wait after the recording that finished it was done`, the frame trace's `wb_held_ms`. Off: the captures' late-page wait orders something not known yet (`RX_VK_LATE_NEW`), so it is tried with the lab's flicker count first. (Triforce, 11-22, the performance audit's SYNC-122, after Source B's) The reads' side: a readback whose write-back is not recorded yet waited on the submissions' timeline with no bound and no second look; it is counted now (the 3000-readback line's second line), and the stall line names the held part by caller (takes, reads, binds, readbacks). The variable is a mode: `1` as above, and such a readback waits in 1 ms slices, looks its writer's recording up between them and ends with it; `2` is the check, where no wait ends earlier and the readback's slices count the part after the recording was done (what `1` would not have waited); unset, `0` or any other text: off, one wait as before. The check of the pages a later write-back changes (the audit's SYNC-101) is not built. `RX_VK_TL_REC=1` and the lab knob `tlrec` ended a capture's wait at a submission's first part, before its burst's write-backs were recorded: they now end it at the recording the submission was finished in, as `=1` here does; `RX_VK_TL_REC=2` keeps the old lookup | +| `RX_VK_LATE_INLINE=1` | (Triforce, 11-22, the performance audit's MEM-80, after Source B's; `gpu/vk/late_inline.h`; **off unless set**, not yet run in the game) A late page (one the CPU wrote while a submission still on the GPU writes it back) is waited for and taken again by default (`RX_VK_LATE_QUEUE`: at the queue's next submit). `RX_VK_LATE_NEW=1` (knob `latenew`) turns four no-wait rules on, each of which brought brightness steps back at the Castle Town save. This turns on the partial rule alone (knob `lateinline`): a page only part of which a submission on the GPU writes back has its other parts synced from the capture, no wait; pages of pools with no mirror, texture storage, pages written back whole and pages whose write-backs' ranges are not known are waited for as by default. The partial rule, here and under `RX_VK_LATE_NEW`, now asks which ranges are still on the GPU against the progress the capture read before its copies, not a second read at the take: a write-back done between the two counts as in flight, and its range is not synced from the older copy. `RX_VK_LOG_LATE` shows the pages waited for | | `RX_VK_PASS_TIMING=1` or `2` | GPU time per pass (2: per program too), logged every 600 presents | | `RX_VK_SLOW_LOG=` | The replay's times for each frame that took longer than this | | `RX_VK_STALL_LOG=` or `0` | On in every run (`gpu/vk/stall_watch.h`): every 600 of the game's frames a `vk: frame times` line (the longest frame and how many were over 33, 50, 100 and 250 ms, present to present and in the replay's own time), and for each frame of 50 ms or more a `vk: stall` line of where its time went: the replay's phases, memory sync's parts, the waits for write-backs still on the GPU (how far ahead, and whether behind a present's blit), the waits for shaders or pipelines, the uploads, the pools imported where they were bound, the GPU behind, the present (its image, fence, call and Reflex's sleep), the game's fence waits, the rest, and a pool's trial in guest memory if one is under way (`RX_VK_DEMOTE_QUICK`); and the new textures left for the next frame and whether the frame's picture was held (`RX_VK_HOLD_MS`), whose holds the 600-frame line counts too. At most 3 stall lines a second and 1500 a session. ``: the stall line's threshold; `0`: neither line | +| `RX_VK_LOCK_CENSUS=1` | (Triforce, 11-22, the performance audit's THREAD-68, after Source B's; off unless set) The lock census: who holds the lock a thread waits for. A lock that is busy when a thread comes for it notes the waiter, the thread that held it and how long the wait was, and every 10 s a thread of the census's own logs the 12 pairs that waited longest, as `vk: lock census: <- () x ms/s (n/s, y us each)`. Counted: `replay_mu`, `sync_order_mu`, `held_mu`, and in `vk_memory.cpp` `pools_mu`, `sync_mu` and `cap_mu`. Threads are named by their descriptions (the game's threads and the renderer's own have them). Off, each lock is taken exactly as before; on, a free lock is one try and a store, and only a busy one reads the clock. The holder is read without the lock, so one that lets go at that moment can be misnamed. With `RX_VK_LAB_AB` naming a knob `lockcensus` it counts only in that knob's phases. Only the counting of Source B's is here, not its sampling of the holders' stacks (`gpu/vk/lock_census.h`) | +| `HOS_STALL_HOST=0` | (Triforce, 11-22, the performance audit's OS-51, after Source B's; on whenever the stall line is) The stall line names a long frame's time on the replay's side and calls the rest outside the replay and the present. A present that comes `RX_VK_STALL_LOG`'s threshold (50 ms) or more after the last one also logs the host's part of that frame, a `vi: stall, the host's part` line: the game's file reads, writes, flushes and save commits (how many, MB, their time and the longest one), the discards, zero-fills and decommits of memory it let go and the give-back's passes, the waits for a virtual core summed over the game's threads, the process's page faults (soft and hard together) and its CPU time in the kernel and in its own code. Causes that did not happen are left out; `none of the host's causes` rules them out. At most 3 lines a second and 1500 in a session, as the stall line. It costs a clock read before and after each counted call and two system calls a present. `0`: none of it; so with `RX_VK_STALL_LOG=0` (`host/stall_host.h`, `host/mkw_glue.c` `hos_stall_host_presented`) | | `RX_VK_SLOW_TAKE_MS=` or `0` | (Triforce, 20 by default) A submit-time capture take that long or longer (`gpu/vk/vk_memory.cpp` `take_capture`): a `vk: slow capture take` line with where its time went (the marks and late pages, the pools lock, the retires, the stamps, the texture shadow, the buffer kept or given back), the first 200 of them. A Linux RTX 4090 near the Market stuttered with `memory sync [capture taken 30 to 110 ms]` while the 600-take average said 1.3 ms. `0`: none | | `HOS_PGO=` | In a run of an `RX_PGO=gen` build: `.reset` clears the counters, `.dump` writes them to ``. It is read before `hookshot_env.txt`, so set it in the environment | @@ -162,62 +187,106 @@ case-sensitive, and an unknown one is ignored. | `RX_VK_CHECKPOINTS=0` | No NVIDIA checkpoints (on by default; after a lost device they name the draw it stopped at). (Triforce) The boundaries between a frame's work have markers too: the write-back diff pass, each recording's start, uploads and end, the present's blit and its menu (`gpu/vk/checkpoint_kinds.h`); the report reads the two stages' last markers together, says what followed the last one reached, and logs the switches, the queue's timeline and the mirrors' piece binds the submits wait for. (P4) Each marking thread numbers its own markers in a ring of its own (15 of 4096; threads after the 15th share a 16th), no atomic shared by the recording threads; the report names a marker's ring and reads the markers around it in that ring | | `RX_VK_HANG_DUMP=` or `0` | The hang watch (Windows) is on in every run: once no frame was shown for `RX_VK_HANG_SEC` seconds (20) with the game's window in front, the top of every thread's stack goes into the log and the whole stacks into `triforce_hang.txt` in the game folder, once a stall and at most three times a run. ``: that file instead, and every second counts whatever window is in front. `0`: no watch (Triforce) The same report can be asked for at any time: put a file named `triforce_hang.txt.now` in the game folder (the report's name with `.now`); within a second the watch takes it away and writes every thread's stack, for a game that goes on showing frames and gets nowhere (a loading screen that never ends). | | `RX_VK_NOOPT` (set) | Pipelines built without the driver's optimisations | +| `RX_VK_MESH_OPT=1` or `=0` | (Triforce) Task and mesh pipelines on AMD. Unset: built without the driver's optimizer on AMD's own drivers (it hung the GPU minutes into Kokiri on an RX 6600) and with it on RADV (without it, the foliage culling's task programs timed out the GPU's ring at the save load on a Steam Deck); the driver's part is made once more on RADV with mesh shaders. `=1`: with the optimizer everywhere. `=0`: without it on every AMD driver, RADV too, as before | +| `RX_VK_GPU_CLOCK=0`, `=peak`, `=standard` | (Triforce, Linux) The Steam Deck's GPU held at amdgpu's peak stable power state while the game runs (`gpu/vk/vk_gpu_clock.cpp`): the Deck's automatic clock dropped while the frame waited on the CPU or a readback, and the GPU work after it ran slow (Kokiri handheld on a Deck OLED: 11.1 to 16.2 fps at the same 15 W; up to 4 W more where the APU was under its cap, so shorter battery life). Unset: only on the Deck's APU, and only while `power_dpm_force_performance_level` is `auto` (a level set by hand is left alone). The render node VK_EXT_physical_device_drm names is opened and the state set on an amdgpu context of the renderer's own (no root); the kernel gives the level back at the process's end, a crash too. The log's `vk: GPU clock:` line says what was done (AMD only). `=0`: nothing opened; `=peak` (or `=1`), `=standard`: that state on any AMD GPU, whatever the level | +| `HOS_HALF_RATE_SHADING=1` | (Triforce, Linux) Keeps `RADV_FORCE_VRS_CONFIG_FILE`, which Steam's Game Mode gives every game (its Half Rate Shading). By default it is dropped before Vulkan loads when it names a file that exists: while it does, RADV on RDNA 2 and newer puts a forced shading rate into every pipeline's key, whatever the file holds, so the pipelines prepared in Desktop Mode or by the builder all compiled again (over 25 minutes to the title on a Deck). The slider then does nothing for this game, little loss: RADV keeps fragment programs that read their position at full rate, and nearly all of this game's do. `RADV_FORCE_VRS` is left alone. The log's boot: line says what was done. Can be set in `triforce_env.txt` | +| `RX_VK_COMPUTE_PIN=all`, `lanes` or `shared` | (Triforce, the performance audit's PIPE-52, after Source A's and Source B's) Which compute programs' pipelines ask for subgroups of 32 lanes (`gpu/vk/compute_pin.h`, `gpu/vk/vk_draw.cpp` `pin_compute`). `all`, the default on every driver but Mesa's Intel one: every compute program, as before. `lanes`, the default on Mesa's Intel driver: only the programs that can tell how many lanes run beside them (a vote, a lane mask, the lane's index, a shuffle, FSWZADD, a matrix multiply, a barrier of Subgroup scope in the module, or shared memory in a workgroup of 32 invocations or fewer); the rest get the driver's choice, which Mesa's Intel compiler builds in a fraction of the time and memory (Source A, an Iris Xe: 0.8-1 GB over 23-25 s pinned, at most 300 MB unpinned). `shared`: as `lanes`, and any shared memory pins, whatever the workgroup. The log has `vk: compute programs held to 32 lanes: ...` where it is not `all` by default, and the pinned and free counts after the precompile. An unpinned program that did need 32 lanes computes wrongly: `=all` is the old pipelines. Not run on an Intel GPU here | +| `RX_VK_FALLBACK_PHI_LIMIT=` | (Triforce, the performance audit's PIPE-56, after Source A's) On Mesa's RADV for GCN 1 to 4 GPUs (HD 7000 to RX 500: RADV without 16-bit storage input and output) and on Mesa's Intel driver only: a mesh-fallback program with a compute stage of more than `` `OpPhi` instructions is not handed to the driver; it is left out as a program the driver refused is, with a log line (8 at most), and its draws are not drawn (`gpu/vk/fallback_size_policy.h`, `gpu/vk/vk_draw.cpp` `fallback_compute_pipelines`). Unset: 50,000 on GCN (Source A: RADV for GFX6-8 had not built a 79,947-phi stage after 45-90 s and was killed at 25 GB), 20,000 on Mesa's Intel driver. `0`: no limit. No other driver has a limit, whatever is set. Not run on either driver here | +| `RX_VK_FALLBACK_CALL_LIMIT=` | (Triforce, PIPE-56) The same for a stage's `OpFunctionCall` instructions. Unset: 200 on Mesa's Intel driver (Source A, an Iris Xe: a 251-call stage unfinished after 27 minutes at 14 GB), none on RADV for GCN. `0`: no limit | +| `RX_VK_NO_SPILLING_RATE=1` | (Triforce, the performance audit's PIPE-55, after Source A's; Linux build only) The Linux build sets `shader_spilling_rate=2` in its environment before the Vulkan instance is made (`gpu/vk/mesa_spilling.h`, `gpu/vk/vk_main.cpp` `create_instance`), unless the player has set that variable. Only Mesa's Intel shader compiler reads it: a program whose registers do not fit is built in fewer, larger steps (Source A, an Iris Xe: one compute program not built after 20 minutes and 13 GB, 32 s and 1.7 GB at rate 2). `=1` sets nothing, so the driver keeps its own rate. On Mesa's Intel driver the log has `vk: Mesa's Intel compiler spills at rate ...`. The Windows build sets nothing: under Proton, add `shader_spilling_rate=2` to the launch options. Not run on an Intel GPU here | +| `RX_VK_DRIVER_FAULTS=0` or `2` | (Triforce, 11-22, the performance audit's PIPE-57, after Source A's; `gpu/vk/driver_faults.h`) On AMD's own Windows driver only (`VK_DRIVER_ID_AMD_PROPRIETARY`), unless `0`: a program whose pipeline build faulted inside the driver is not asked of the driver again this run, and its key is appended to `triforce_driver_faults.txt` beside the executable (one line: vendor, driver version, program) by the fault's handler, before a fault during the start's shader work restarts the game, so the next start with that driver skips it instead of faulting again (`vk: driver faults: N programs ... kept out of it`; delete the file to try them again, another driver version ignores its lines). And one build of a program is inside the driver at a time: a second waits outside it, a probe of the caches is told a compile is required. `2`: on every driver (the lab's); `0`: nowhere. On other drivers nothing of it runs (`gpu/vk/vk_draw.cpp` `fault_quarantine`, `build_graphics`, `guarded_graphics`) | | `RX_VK_PASS_BARRIER=game` or `full` | The barrier a draw pass starts with (`gpu/vk/pass_barrier.h`). `game`: a pass right after another, with nothing recorded between (no barrier of the game's, no copy, clear, uniform update or dispatch), waits for the earlier work in its fragment stages only, so its vertex, task and mesh work may overlap the earlier pass's fragments; a pass the replay itself split off for a hazard (`a pass of its own` in the log) and `RX_VK_BARRIER_EVERY_DRAW`'s keep a full one. (Triforce, P4) "Nothing recorded between" is the command buffer's own count (`RX_VK_BARRIER_CONTEXT`), so beside the Replay helper threads too. `full`: a full barrier at every pass start, as before. Unset: `game` with NVIDIA's driver, `full` on every other card. The log's `vk: pass barriers:` line says which, and the frame statistics' `pass starts ordered in their fragment stages only` how many a frame | -| `RX_VK_SPLIT_BARRIER=pass` or `full` | (Triforce) A long submission goes to the GPU in parts of 120 draws and dispatches (`RX_VK_CHUNK`); `gpu/vk/barrier_rules.h`. `pass`: a pass open at a part's split goes on in the next part, begun again with the same targets behind a barrier of its attachments' outputs alone and with what its draws sampled and wrote, so the GPU does not drain between two draws of one of the game's passes; where the part's end records anything (its uniform updates, its write-backs) or an event's write is held, the full barriers as before. A parallel replay's segments are not split into parts: each begins and ends with a full barrier, so nothing goes on across a segment's edge. `full`: a full barrier at each part's end and the next one's start, as before. Unset: `pass` with NVIDIA's driver, `full` on every other card. The log's `vk: a submission's parts:` lines say which and, every 3,600 splits, how many went on in their pass and how many of those parts had uploads (whose own full barriers order them after the part before all the same) | +| `RX_VK_CHUNK=` | (Triforce, 11-22, the performance audit's SYNC-66, after Source B's; `gpu/vk/barrier_rules.h` `part_draws`) How many draws and dispatches a part of a long submission has in a serial replay: the GPU starts on the first part while the rest is recorded, and each split costs a queue submit, two full barriers and the part's bookkeeping (about 27 a frame at 120). Unset: the driver's default, **120 on every driver** until the lab has measured one (Source B cuts at 128 where the GPU idles while the CPU records and at 4096 on desktop drivers); `0`: whole submissions, never split; `120`: today's size whatever a later default says. The size was read once into a static: now the `RX_VK_LAB_AB` knob `chunk:` changes it in its phases while the variable is unset (`RX_VK_LAB_AB=chunk:480,chunk:240` with `RX_VK_PARALLEL=0`). The log's first `vk: a submission's parts:` line says the size in force and the driver's default | +| `RX_VK_SPLIT_BARRIER=pass` or `full` | (Triforce) A long submission goes to the GPU in parts of 120 draws and dispatches (`RX_VK_CHUNK`); `gpu/vk/barrier_rules.h`. `pass`: a pass open at a part's split goes on in the next part, begun again with the same targets behind a barrier of its attachments' outputs alone and with what its draws sampled and wrote, so the GPU does not drain between two draws of one of the game's passes; where the part's end records anything (its uniform updates, its write-backs) or an event's write is held, the full barriers as before. A parallel replay's segments are not split into parts: each begins and ends with a full barrier, so nothing goes on across a segment's edge. `full`: a full barrier at each part's end and the next one's start, as before. Unset: `full` on every card ((Triforce, 11-22, SYNC-66) this row said `pass` with NVIDIA's driver; `pass` has been off unless set since 11-16, when the lab's Market showed no gain from it: `barrier_rules.h` `split_in_pass`). The log's `vk: a submission's parts:` lines say which and, every 3,600 splits, how many went on in their pass and how many of those parts had uploads (whose own full barriers order them after the part before all the same) | +| `RX_VK_GPU_WAITS=0` | (Triforce, 11-22, the performance audit's SYNC-27; on by default; not yet run in the game) The host's calls that block on the GPU are made without being timed, as before. By default each is timed where it is wrapped (`gpu/vk/vk_work.h`, `gpu/vk/gpu_wait_kinds.h`) and summed by kind: `vkWaitForFences` and `vkWaitSemaphores` with a timeout above 0 (fences, timelines), `vkQueueWaitIdle` and `vkDeviceWaitIdle` (idle), `vkGetQueryPoolResults` with the wait bit (queries). Polls count nothing, a wait inside a wait counts once, and the low-latency sleep's wait is left out as pacing. After each 120-present line the log has `vk: host waits for the GPU: ... ms a frame (fences, timelines, idle, queries; all threads)`, which counts what that line's "GPU-bound wait" (the recording slots' waits alone) does not, and the frame trace has `gpu_waits_ms`. The cost is two clock reads a blocking wait. The `RX_VK_LAB_AB` knob `nogpuwaits` turns the count off in its phases | +| `RX_VK_ORDER_WAKE=1` | (Triforce, 11-22, the performance audit's SYNC-59; off unless set; not yet run in the game) A queue stopped at a wait for an event goes on when a submit finds the wait met. A signal of the CPU's (`nvnEventSignal`) wakes no one: the stop waits for the order thread's next `Sleep(1)`, or a game thread's wait yields up to 3 ms and then forces it. With the switch (`gpu/vk/vk_replay.cpp` `sync_watch`, `event_signal_queued`) the order thread waits on an auto-reset event, 1 ms at most, which the signal sets while a queue is stopped: one `SetEvent`, no lock and no replay on the signalling thread. And a game thread's wait for stopped work that sees a signal pending lets the queues whose waits are met go on first, as `sync_settle` does; nothing is forced. The order line counts the wakes and those that let work on: under about one a second, leave it off. The `RX_VK_LAB_AB` knob `orderwake` turns it on in its phases | +| `RX_VK_RELEASE_QUEUE=1` | (Triforce, 11-22, the performance audit's MEM-85, after Source B's idea; `gpu/vk/release_queue.h`; **off unless set**, not yet run in the game) Without the replay thread (`RX_VK_ASYNC` unset), a texture or program the game finalizes is let go by its own thread: held for the replaying thread only while a queue is stopped at a wait (PR 430), otherwise after `sync_drain` and the replay's lock, which a replay holds through whole submissions. With this on, every such release goes to a list the replaying thread applies at the start of the next submission's replay or present when no queue is stopped (after the stopped work otherwise, as PR 430's); a texture's release counts the game's writes from the write generation of its call, so writes after the call are still uploaded. The `RX_VK_ASYNC` job path is not changed. The `RX_VK_LAB_AB` knob `relq` turns it on in its phases. On in every run, a count: every 600 releases a `vk: releases` line when one of them waited 0.5 ms or more for the replay or its order (how many, the time in all, the longest) or went to the list. Not built: the entry's `rel_wait_ms` column in frames.csv | +| `RX_VK_UB_BATCH=1` or `2` | (Triforce, 11-22, the performance audit's SYNC-21; off unless set; not yet run in the game) A recording's uniform updates are written into guest memory at its end between two full barriers, one `vkCmdUpdateBuffer` an update (about 80 a frame) and nothing between two updates of the same bytes. `=1` (`gpu/vk/ub_batch.h`, `gpu/vk/vk_replay.cpp` `flush_ub_batched`): each update's bytes go into the recording's stream (256 bytes at least) and each run of updates into one buffer is one `vkCmdCopyBuffer` of many regions; a run ends where an update would write bytes the run already wrote, or where the destination buffer changes, and the next starts behind a full barrier, so overlapping updates reach memory in the stream's order. A piece that is not 4-byte aligned becomes a region too (it was a CPU write at record time); with the stream full a piece is written as before. `=2`: written as before, the overlaps counted (by guest address). Both log a line every 600 flushes: updates, runs ended by an overlap and by another buffer, copies and regions. Look at the counts first: a barrier at every run end costs GPU time if they are common. The `RX_VK_LAB_AB` knob `ubbatch` is `=1` in its phases | +| `RX_VK_SPLIT_BEFORE_DRAW=0` or `1` | (Triforce, the performance audit's REPLAY-14, after Source A's) A part of a long submission (and a helper thread's segment) ends only right before a plain draw that will draw (`NOP_DRAW`, `_IDX`, `_INDIRECT`; rendering on, no rasterizer discard), not after any draw or dispatch (`gpu/vk/barrier_rules.h` `split_before_draw`, `cut_ok`; `gpu/vk/vk_replay.cpp`). On by default on Intel's GPUs only: on an Arc B580 a command buffer that begins with mesh draws drew its later plain draws as exploded triangles (Source A: 40 shard frames of 1,121, then 0 of 1,101). `=0` off, `=1` on with any GPU. A run of mesh draws or dispatches is then not cut, so a part can be longer than `RX_VK_CHUNK`. Not run on an Intel GPU here | +| `RX_VK_SPLIT_UBFLUSH=0` or `1` | (Triforce, REPLAY-14) Whether a part's end writes the submission's uniform updates so far into the game's memory (`split_ub_flush`). On by default, as before, except on Intel's GPUs without mirrors of the game's pools (integrated ones), where only the submission's last part writes them, as an unsplit submission does: written at each part's end they reach the memory later parts' other reads and the game's CPU see (Source A: white blocks in 1 run of 3 on a Meteor Lake iGPU, then 0 of 3). `=0` off, `=1` on with any GPU. The log's `vk: a submission's parts: ended only right before a plain draw ...` line says both | | `RX_VK_EVENT_BARRIER=late` or `full` | (Triforce) An event signal in the game's command stream (`gpu/vk/barrier_rules.h`). `late`: nothing is recorded at the signal and its pass goes on (the word's buffer is bound there, as before); the event's word is written right after the next full barrier on its command buffer (the game's next barrier, a dispatch's or pass start's), so after all the work before the signal, and a recording's end writes what is still held behind a full barrier before its write-backs copy the words to the game's memory; a wait the replay finds met for an event whose write is held gets a full barrier first. The game sees the word at that barrier rather than at the signal. `full`: the pass ended, a full barrier and the write at each signal, as before. Unset: with the pass barriers' game mode (`RX_VK_PASS_BARRIER`: NVIDIA's driver). Not with the parallel replay. The `vk: event signals:` line says which | | `RX_VK_TAIL_EARLY=1` or `=0` | (Triforce, on by default since options_version 6; over the Debug tab's Start next frame early row, `tail_early=0` to turn it off; read at the start; after NX2navi's `tail_early`) The early tail (`gpu/vk/tail_early.h`, `gpu/vk/vk_replay.cpp`): the replay cuts a frame's recording at the gameplay upscale's dispatch (the game's network, or DLSS, DLAA, FSR in its place) and records the rest, the upscaler and the interface, as one command buffer; its event signals are held. At the recording's end a tail that wrote no binding (its inline uniform updates aside), bound no guest memory itself, ran no program of the game's that reaches guest memory through the page table or a texel buffer, and wrote no texture back has its event words written in its upload command buffer, after the scene's write-backs and ahead of its own commands: OoT's present poll then sees its frame's event while the GPU still runs the tail. Any other tail has the words written at its end, as before. Also with the parallel replay (its segments end at that dispatch) and with the lazy write-back. Not with `RX_VK_BENCH` or a frame dump. A `vk: the early tail` line at the start says on, and every 1200 tails how many went early and why the others did not. Not yet run in the game | | `RX_VK_DISPATCH_BARRIER=game` | (Triforce) Off unless set, not yet run in the game: a dispatch right after another, nothing recorded between, records no full barrier of its own (not an indirect one, nor a program a stand-in may take over), as NX2navi's `barrier_game` does; the game's own barriers order its dispatches. Only the game shows whether that is always so (a one-frame band or flicker where it is not): a switch for the lab. A `vk: RX_VK_DISPATCH_BARRIER=game:` line every 20,000 dispatches it left without one. (wp14, P4) It no longer stands aside while the Replay helper threads record (most frames at the Market, so the lab's earlier run of it measured little): "nothing recorded between" is the command buffer's own count of its commands (`gpu/vk/cmd_work.h`, `RX_VK_BARRIER_CONTEXT`), a segment being a command buffer of its own | | `RX_VK_BARRIER_CONTEXT=0` | (Triforce, P4; after NX2navi's per-context barrier state) What the barrier rules ask, "was anything recorded into this command buffer since its last full barrier, its last pass's end, its last dispatch", is answered by each command buffer's own count of its commands (`gpu/vk/cmd_work.h`), where one count of every thread's commands served before (an atomic add on every command by every recording thread). So the rules hold while the Replay helper threads record: the left-out repeats of a full barrier, `RX_VK_PASS_BARRIER`'s pass starts and `RX_VK_DISPATCH_BARRIER`. A command buffer's begin starts its state afresh; each segment begins and ends with a full barrier, so no rule relaxes across a segment's edge. `RX_VK_SPLIT_BARRIER` asks also whether anything went to the queue between two parts (every submit is counted). The rules' defaults are as they were. `0`: the one count, and the rules standing aside while the helper threads record, as before. The `vk: barrier state:` line at the start says which; the frame statistics' `full barriers recorded, ... left out` and `pass starts ordered in their fragment stages only` show the difference | +| `RX_VK_BATCH_IDLE=1` | (Triforce, 11-22, off by default; the performance audit's SYNC-09, after Source A's; `gpu/vk/barrier_rules.h` `batch_idle`) With held submits (`RX_VK_BATCH`, the row below: on by default with Mesa's RADV only), a submit that joins the held batch sends it at once when the GPU has finished every batch sent; while the GPU is busy, submits still group. A held batch otherwise waits for a wait, a present, the 2 ms poll or the cap of 32 with the GPU idle beside it. Source A has this on wherever batching is, and off where guest memory stayed host-pointer imports (each submit there pays amdgpu's page check); here it is **off unless set on every driver** until a Linux RADV machine has run it, and no guard for host-pointer imports is built. Nothing changes without held submits (NVIDIA, AMD's and Intel's Windows drivers). The 120-present `vk: batched submits:` line counts its flushes as `idle` | | `RX_VK_BATCH=parts`, `1` or `0` | Held submits (`gpu/vk/vk_replay.cpp` `batch_way`). `1` (the default with Mesa's RADV): every submit held until something needs the GPU's results or the queue's order. (Triforce) `parts`, off unless set: only a whole submission's recordings are held, and only while the game's submit is being replayed; a part goes at once with those held before it, as do 4 held (`RX_VK_BATCH_MAX`), and the held go when the replay of the game's submit returns. Neither changes the command buffers or their order on the queue. `0`: each submit at once (the default elsewhere). The `vk: batched submits` lines say which and how many command buffers went in each queue call | +| `RX_VK_PEND_HELD=0` | (Triforce, on by default) A flush for a fence wait or a progress poll looks at a flag before the queue's lock (`dev.present_mu`) and returns when nothing is held (always, with batching off): it took the lock to find nothing, behind a present or a late page's fix waiting on the GPU. A held submit's recording is made known to the waiters only once it is held with the flag raised, so a waiter that finds it flushes it (before, it could flush nothing and wait for the held signal until the next poll, present or cap); a recordings' wait whose recording the recordings' timeline has passed returns without a flush. The batched submits' line counts the looks that found nothing held. `=0`: the lock every time and the registration first, as before; `RX_VK_LAB_AB=nopendheld` the same in its phases (`tests/test_flush_nothing_held.py`) | +| `RX_VK_IMPORT_HIGH=` or `0`, `RX_VK_IMPORT_LOW=` | The idle imports' valve (`gpu/vk/vk_memory.cpp` `imports_give_back`): past `HIGH` percent (85) of the system heap's budget, the imported pools bound least lately, idle for 600 submissions and without a mirror, are given back down to `LOW` percent (75; 10 to 95). On by default where that budget is under 12 GB (on Windows only with the memory handling on); `RX_VK_IMPORT_HIGH=85` turns it on at any budget, `=0` off altogether. Setting either keeps the whole budget as the measure (no cap, below) | +| `RX_VK_IMPORT_CAP_MB=` or `0` | (Triforce, the performance audit's MEM-22, after Source A's) What the valve above measures the heap against, where that is less than the budget (`gpu/vk/import_budget.h`). Unset: 4,500 MB with Mesa's RADV at a budget of 12 GB or more, where the valve was off and live imports piled up for the session (Source A, an RX 9070 XT with 32 GB: 221 live imports and 7.4 GB after two hours against 50-64 and 2.3-2.7 GB capped); no cap on any other driver or at a smaller budget. ``: that cap on any driver and budget, which turns the valve on. `0`: no cap, RADV's behaviour before. A pass against the cap gives back 1 GB at most. Not run on RADV here | | `RX_VK_FOLIAGE_SHADOW_CULL=[f]` | (Triforce, after NX2navi's 754ded9) On from layer 1 unless set (`off`: as the game draws them): the draws of mesh programs into the sun's shadow cascades (depth only, square, wider than 768) from that layer of the cascades on (0 to 2; NX2navi's is 1) that cull neither face are drawn with their back faces culled (the front ones with `f`, as `1f`): the foliage's double-sided cards draw half their fragments there (`gpu/vk/foliage_shadow.h`). Not quite the same picture: a lone card facing away from the sun casts no shadow in the middle and far cascades (where cards stand in pairs or clumps the others cover it; the near cascade is drawn as the game asks). The log says `vk: RX_VK_FOLIAGE_SHADOW_CULL` at the start and names the first draws it culled (`vk: foliage shadow cull:`) | | `RX_VK_FOLIAGE_SHADOW_VRS=` | (Triforce, after NX2navi's 754ded9) On from layer 2 unless set (`off`: the extension not asked for, no pipeline changed): the same draws of native mesh programs, from that layer of the cascades on, shade one fragment per 2 x 2 pixels (`VK_KHR_fragment_shading_rate`, where the device has `pipelineFragmentShadingRate` and native mesh shaders; elsewhere nothing changes): the leaves' alpha test runs a quarter as often, depth still per texel. The rate is dynamic state only in the pipelines of native mesh programs with no colour target and a depth one (and in their two shader libraries, keyed apart), which the driver builds again once: on such a device the driver's part of the preparation makes those libraries too (and the `RX_VK_LIB_CHECK` sample asks for them), once more at the first start after the update, before the game (the other libraries come back from the cache), and the light precompile builds each depth-only pipeline of a native mesh program once more; their keys, the program cache and every other pipeline stay as they were. The log says `vk: the far shadow cascades' foliage at 2 x 2:` at the start and names the first draws (`vk: foliage shadow 2 x 2:`) | | `RX_VK_FOG_CULL=1` | (Triforce, after NX2navi's 0379b00) Off unless set: the volumetric fog's first compute pass skips, as the game does, the froxels behind a coarse depth table of the frame before and writes them as empty, which the fog's history smears into dark trails beside near objects (worst at the screen edges, in foggy places such as the Lost Woods and Kokiri Forest). By default those froxels are computed too: the comparison is made never true in that one module, found by its shape in the SPIR-V (`gpu/vk/spv_fog.h`: a block of five image writes, four of (0,0,0,1) and last 1/255, and the one branch to it an ordered greater-than decides). It is applied where the module is made (`make_module`), outside the translation stamp: the program cache's records stay as they were translated, so nothing is translated again, and only that one compute pipeline is new to the driver's caches. With the precompile keeping none (`RX_VK_PRECOMPILE_KEEP`, less RAM), its record of the compute programs built has the patch and the matcher's version in its key (`gpu/vk/vk_draw.cpp` kFogPatch times `spv_fog::kMatcher`, raised whenever the matcher finds modules it did not: a record written by the matcher before 11-17's, which patched none of Triforce's fog modules, is built again), so the first start with the patch, or with a new matcher, builds the listed compute programs once more before the game (every one but the fog's a cache hit, the fog's compiled into the play cache, `mkw_vk_pipelines_play.bin`, or the main one where there is none, and saved before the record says built) rather than compiling the fog's at its first dispatch; with `RX_VK_FOG_CULL=1` that key is the one it was. A little more GPU time in the fog's pass. The log says `vk: fog: module , the volumetric fog's first pass:` and what was done (also for a near miss: the exit found but not its comparison) | | `RX_VK_FIFO` (set), `RX_VK_IMMEDIATE` (set) | The swapchain's present mode for one run, over the Insert menu's VSync row: FIFO, as with vsync forced by a driver; immediate, where the surface lists it. Unset, the row decides (`gpu/vk/present_mode.h`): On is mailbox where the driver has it (immediate on Windows where it has none, FIFO elsewhere), Off immediate wherever the surface lists it | | `RX_VK_SKIP_PROG=,...` | Leaves out the draws of these programs (default `53509044401d78d2`; `-` leaves out none) | | `RX_VK_PIPE_SKIP=0`, `1` or `2` | Default/`1`: wait for shader translation; a pending pipeline can be skipped only for attachment-only draws whose targets hold an earlier frame. `0`: wait for every pipeline too. `2`: restore the old skip for untranslated draws with persistent targets; this can omit writes before the shader is known. Translated memory-writing draws still wait | +| `RX_VK_TRANSLATE_WAIT_MS=` | (Triforce, 11-22, the performance audit's PIPE-30, after Source A's; `gpu/vk/translate_wait.h`) Off by default: a draw whose program is still translating waits for it, as before. Set (1 to 100, digits only; 8 to try): a draw into targets that hold an earlier frame, of a program whose key's translated variants write no memory (noted at each new pipeline and at the precompile's loads), waits at most that many ms a frame in all, the translation hurried but not run on the draw's thread, then is left out; a spent budget leaves the next one out at once. A key never translated, one with a variant that writes memory, and a draw into new targets wait as before; `RX_VK_PIPE_SKIP=0` and `=2` keep their meaning. The log says how many waited and were left out (`vk: programs still translating, so far: ...`). More draws left out, mostly in load frames: for a lab to weigh (`gpu/vk/vk_draw.cpp` `draw`, `translate_note`) | +| `RX_VK_PIPE_DEPTH_BUDGET=1` | (Triforce, 11-22, the performance audit's PIPE-29, after Source B's; `gpu/vk/pipe_wait.h` `depth_budget`, `choose`) Off by default on every driver, Intel's too (the entry would have it on for Intel's Windows driver: that waits for the owner's word). Set: a depth-tested draw into a pass drawn once whose shaders write no memory, outside a samples-passed count and not into targets kept from an earlier frame (the item icon capture's), waits for its pipeline from the frame's wait pool (`RX_VK_PIPE_FRAME_WAIT_MS`, 50 ms) and is then left out; that pipeline is then left out at once until it is built (the list emptied every 600 presents). The first 16 left out are named, and a line counts them (`vk: depth-tested draws into passes drawn once, so far: ...`). A render into a new depth target left out can stay missing until it is drawn again: the trade Source B made for an Arc B580's 11-13 s freezes (`gpu/vk/vk_draw.cpp` `draw`, `pipe_depth_budget`) | | `RX_VK_PIPE_FRAME_WAIT_MS=`, `RX_VK_PIPE_ONESHOT_BUDGET=1`, `RX_VK_PIPE_SKIP_NEW=0` | A draw whose pipeline is not ready (`gpu/vk/pipe_wait.h`, NX2navi's rules). A full-screen draw (at most 6 vertices over half the pass or more: a lighting, fog or tone mapping pass) into targets that hold an earlier frame is left out only where the hold of the last picture takes its frame (`RX_VK_HOLD_PIPELINES`); where it cannot (at its bound, resting after one, or off) the draw waits for its pipeline up to this many ms a frame in all (50, at most 500), then 40 ms more, and is left out after both; its job goes to the front of the workers' queue meanwhile. `0`: left out, as before. `RX_VK_PIPE_ONESHOT_BUDGET=1` (off; NX2navi has it on): draws into targets drawn once and draws whose shaders write memory wait from the same budget and are left out once it is spent, instead of waiting until their pipeline is made. `RX_VK_PIPE_SKIP_NEW=0`: a pass whose colour target is drawn for the first time is kept by its depth target alone, as before (by default its draws wait, as for any target drawn once: the game's item icon capture). The log's `vk: pipelines not ready` line counts them | | `RX_VK_PIPE_FAST_WORKERS=0`, `RX_VK_PIPE_LINK_MS=` | A pipeline whose stage libraries were cold has those libraries built by a worker first and is linked from them, ready with that link, and its whole optimised build follows as a job of its own (`gpu/vk/pipe_wait.h` `libs_first`, NX2navi's `pipe_fast_workers`); a draw that waits for it runs the libraries and the link, not the whole build. `0`: the whole build first and the cold libraries after it, as before. `RX_VK_PIPE_LINK_MS=` (0 by default, at most 100; NX2navi's 6): the links made on the draws' threads in a frame take at most this long; a new pipeline after that is linked by a worker, its draws left out or waited for meanwhile (needs the fast pipeline link and the first switch on) | | `RX_VK_PRECOMPILE=0` or `1` | The shader preparation before the title off or on (on in builds made with `RX_RELEASE=1`, as the builders' are) | | `RX_VK_PREPARE_THREADS=` | The threads the shaders are translated and their driver's part made on before the title, 1 to 64. Unset: every processor, a thread past half of them for each GB of RAM free at the start, and half in a process started above normal priority (the log's "the shaders' preparation runs on" line says how many) | +| `RX_VK_WARM_COMPUTES=listed` or `all` | (Triforce, 11-22; `gpu/vk/warm_computes.h`, `tests/test_warm_computes.py`) The compute pipelines the shader preparation's translation pass has the driver compile. `all`: every compute program's texture variants (about 9,700), as before. `listed`: only the programs play dispatched (`triforce_computes.bin`, about a tenth of them; the precompile builds the same ones), the others translated as before and compiled at their first dispatch; with no list yet (nothing played) every one. Unset: `listed` on Intel's GPUs (an Arc A750 spent 5175 s of a 5858 s pass on them, an RTX 4070 SUPER 170 s on the whole pass), `all` on the others. No stamp holds it | +| `RX_VK_FALLBACK_SHARED_RECORDS=0` | (Triforce, 11-21, the performance audit's MEM-154) With the compute fallback for task and mesh shaders, each recording thread keeps buffer pairs: 64 MB of records, the same size for every program, and a payload sized by the task program. A payload that fits no pair now makes a pair that takes the newest pair's records and only a new payload; before, 64 MB more of records each time, kept for the session. The log's `vk: mesh fallback: a buffer pair` lines say made or shared. `0`: its own records each time, as before (`gpu/vk/vk_draw.cpp` `fallback_buffers_of`) | | `RX_VK_TASK_MESH_FALLBACK=0` or `1` | Task + mesh draws through the compute fallback (1) or the driver's mesh shaders (0). By default the fallback is on for AMD's Windows driver for RDNA 2 (every RX 6000 card and that generation's integrated GPUs: the driver whose `vk: vendor fixes` line names the "AMD proprietary shader compiler"), where the native path timed out the driver on the RX 6700 XT and RX 6800 XT (issue 996) | +| `RX_VK_FALLBACK_LOADED=0` | (Triforce, 11-22, the performance audit's PIPE-62, after Source A's) On the task and mesh fallback, the precompile is handed only the fallback programs whose records are still on the disk, as before. The default also hands it every fallback program already in memory that is ready and not failed (`gpu/vk/vk_shader.cpp` `program_cache_load_some`, in the part outside the translation stamp), so that their compute pipelines are built on the precompile's workers before the title: in a start that prepares, `state_list_prepare` has read the state lists' programs in before the precompile asks, and those got no pipelines until their first draw, which waited up to 50 ms on the recording thread and was left out until they were built. The log says how many: `vk: N mesh fallback programs read in earlier in this start are made ready with the rest`. Only GPUs on the fallback (no mesh shaders, RX 6000 and RDNA2 integrated GPUs on Windows, the menu's Compute row, `RX_VK_TASK_MESH_FALLBACK=1`); the preparing start's load grows by the compiles moved out of play | | `RX_VK_MESH_AHEAD=0` | (Triforce, on by default, after NX2navi's `mesh_ahead`; `=0` as before) An indirect mesh draw's records are copied into a ring as `{count, 1, 1}` before it, outside the pass, so each one ended its pass and the next draw began it again (OoT's foliage, grass and leaves: about 16 a frame at the Market). Now the first indirect mesh draw of a run copies the records of the indirect mesh draws after it as well, in the same three barriers, and those stay in the pass: a run is the indirect mesh draws after it with only state, draws, counter reports and render conditions between (`gpu/vk/vk_replay.cpp` `mesh_run_kind`); a dispatch, a barrier, a copy, a clear, a uniform update, new targets or an event ends it, so no work of the game's between can write the records after they were copied. A draw takes only the entry copied for it (its op, records, count and stride); at most 32 draws and 1 MB of the ring a run (`gpu/vk/mesh_ahead.h`). A `vk: indirect mesh draws` line every 20,000 conversions says how many took their records from an earlier draw | | `RX_VK_NO_PROGRAM_CACHE` (set) | Neither reads nor writes `hookshot_programs.bin` and the pipeline notes | | `RX_VK_PROGRAM_CARRY=0` | The program cache (`triforce_programs.bin`) is good for one build of the translator only, as before: any change to its files, a log line or a host option's number in `gpu/vk/vk_api.h` too, translates every program again, and the pipeline cache, the pipeline notes and the prepare's record go with it. By default the cache is carried over an update that changed nothing of what translates (`gpu/vk/program_carry.h` has the rule, `tests/test_program_carry.py` holds it): the translation stamp (`nx2/build.py` `translation_stamp`) is the translator's files without `vk_api.h`, without the parts of `vk_shader.cpp` and `vk_bank.cpp` marked as outside it, and without comments, blank lines and spaces at lines' ends. The build log's `build: translation stamp` line says whether it is the last release's (`nx2/released_translation_stamp.json`, which players' caches are for, and which `tests/test_program_carry.py` holds this source to) or which stamped files moved it against that release; a second line, in brackets, compares it with the previous build in the same folder (its record: `translation_stamp.json` in the build folder, written once the objects are compiled). Both are about the build's part of the stamp: the runtime adds its own settings to it (`disk_stamp`). The log's `vk: program cache` line at the start says whether the programs were kept, carried over an update of the translator (and from which build), or translated again (and how many the cache had); while they are translated again a line now and then counts them. Switching it either way translates every program once more | +| `RX_VK_PREPARE_RECHECK=0` | (Triforce, 11-22, after Casual Gamer's #dev-fixes post) Both as before: a start whose record (`triforce_prepared.bin`) says the shaders are prepared takes its word, and the empty mark of a start that died in the driver's part without stage libraries (`triforce_prepared.bin.warming`) leaves that part out on any GPU and driver. By default (`gpu/vk/prepare_again.h`, `vk_draw.cpp` `prepare_run`): where the record says nothing to do, the program cache's head and size are read, and one made again since for another GPU or other settings (another stamp, or its head alone) holds none of the shaders: `preparing again` (logged), the stale record removed first; a cache carried over an update keeps the stamp and passes. And the mark holds what the part is made for (the GPU, the driver, the programs), as the libraries' journal does: another stamp's mark is another driver's, and the part goes on (`tests/test_prepare_again.py`) | | `RX_VK_CACHE_SET=0` | (Triforce) No earlier set of the shader caches is kept or put back (`gpu/vk/cache_set.h` has the rule, `gpu/vk/vk_cache_set.cpp` the files, `tests/test_cache_set.py`). By default, when the caches' stamp changes and the build of the translator did not (a setting of the translation: the menu's Robust buffers row, the mesh shaders row, the GPU, an `RX_VK_` switch of the translation), the set there (the program cache and its index, the pipeline cache and the play file with their `.for`, the prepare's record with its `.check`, `.warming` and `.warming.skip`, the pipeline notes and what is kept beside them by the stamp) is renamed to `..` when the disk has twice the set and 10 GB free, and `triforce_cache_set.kept.txt` lists it. One set is kept. A start whose stamp is the kept set's puts every piece of it back (a set with a piece missing is deleted), and the set it found is kept in its place; a start of another build deletes it. `triforce_cache_set.txt` names what the set there was made for, and the log's `vk: shader caches:` lines say which part of the stamp changed. A set put back while the player's `triforce_pipeline_states.bin` is not the one it was kept with leaves `triforce_cache_set.states.txt`, and the prepare before the title then goes through the state lists, so that what was met under the other set is built before the game. Files of a kept set's names that no record names (a start that ended in a swap) are deleted at the next start. `RX_VK_CACHE_SET=0` also deletes a kept set; `RX_VK_PCACHE_KEEP_OLD` and `RX_VK_NO_PROGRAM_CACHE` leave it off too (a kept set of this build stays, another build's is deleted) | | `RX_VK_GPL=1` or `0`, `RX_VK_GPL_KEEP_LINKS` (set) | The fast pipeline link (`VK_EXT_graphics_pipeline_library`; `gpu/vk/gpl_rules.h`), on by default since options_version 5: a new pipeline is linked from its program's two shader libraries where the driver has them compiled, and the optimised build replaces it. The Debug tab's Fast pipeline link row is the same without an environment to set (`fast_pipeline_link=1`); the switch goes over the row. With it the preparation has its driver's part ("Compiling shaders"), which now goes on after a start that died in it, leaving out the programs that start had in the driver's hands (`triforce_prepared.bin.warming` and `.warming.skip`), up to three starts in a row. `RX_VK_GPL_KEEP_LINKS`: the linked pipelines the optimised ones replaced are kept for the run, as before, instead of destroyed 600 presents and 10 s later. `RX_VK_NO_WARM` (set): no driver's part; `RX_VK_GPL_SYNC` (set): a cold library is compiled in the draw; `RX_VK_GPL_NO_CACHE` (set): the libraries are not kept in the pipeline cache | | `RX_VK_LIB_CHECK=0` | (Triforce, after NX2navi's e10ef60) On unless `0`. At a start whose record says the driver's part is made for these shaders and this driver, a sample of 12 kept program variants has its two stage libraries made as that part makes them, asking the caches only where the device can (the answer a draw's link gets; without the flag, `RX_VK_GPL_SYNC`, their time against a second making, which comes from the cache), twice, before the game (`gpu/vk/lib_check.h`). When more than half are cold (`mkw_vk_pipelines.bin` lost or refused, or the libraries built otherwise now without `kWarmRecipe` counted up) the driver's part is made again at once, as after a driver update, with its journal and skip list; warm, nothing more happens (some 50 library lookups). `triforce_prepared.bin.check` records that it did: a start that finds them cold again right after leaves them to the draws, as before (each try holds a start for minutes; remove the file to try again). A making again whose cache could not be saved is recorded the same way. A cache file over its bound is not made again at all. The log's `vk: stage libraries:` line says what was found | +| `RX_VK_BUILT_CHECK=0`, `=2` | (Triforce, 11-22, the performance audit's PIPE-21, after Source A's and Source B's; `gpu/vk/built_check.h`) On unless `0`. A light precompile (`RX_VK_PRECOMPILE_KEEP`: none kept) records what it built for the caches in `.built`, and later starts left all of it to its first draw or dispatch on the record's stamp alone. Now, before the record is trusted, up to 24 of the recorded notes are asked of Triforce's own pipeline caches as a draw asks them (without compiling, where the device has `pipelineCreationCacheControl`; without it each is built twice and timed: a first build over 4 times the second and 3 ms more was compiled), and with the record's compute key up to 6 of the listed compute programs as a dispatch asks them. Under 16 answers nothing is said; half of them or more not there: the caches lost them, and everything noted and listed is built again before the title and recorded anew (`vk: precompile's record: ...`). Lost again right after that (`.built.check`): left to the draws, as before, with a hint about the driver's shader cache size. Also counted: play builds of recorded pipelines that took 40 ms or more and recorded compute programs a dispatch did not find (`vk: precompile's record: in play so far, ...`, at most every 10 s while it grows). `2`: built again at every start (the lab's). `0`: neither the check nor the counter, as before (`gpu/vk/vk_draw.cpp` `built_check_cold`, `precompile_run`, `built_play_build`) | | `RX_VK_SPV_PACK=0` | Every shader module kept for the whole run, as before. By default the SPIR-V of graphics and mesh programs is kept packed in memory (`gpu/vk/spv_store.h`, to about 0.37 of its size) and their shader modules, which the driver otherwise holds for the whole run, are made only for the time a pipeline is created with them, on the thread that builds it. Compute programs, the programs the preparation lets go and the programs a draw reads from the cache itself keep their modules. The log's `vk: shader code packed` lines say what it comes to and what it costs | | `RX_VK_PCACHE_RAM=1` | Off by default (in its one session on, v0.4.0-test9, it gave back nothing the working set showed). With it, once the PC has had under 1.5 GB of RAM available at two looks a second apart, the pipeline cache (when it is 256 MB or more) is destroyed once no pipeline creation is using it (`gpu/vk/pcache_gate.h`): the pipelines built after that go without it and it is not saved again that session, so `mkw_vk_pipelines.bin` keeps its last save. The log's `vk: RAM short ...: the pipeline cache ... is let go` line says when and what the working set did. Whatever this switch says, the cache's save in play waits while a copy of it would not fit in the RAM available with 2 GB to spare (the save at the game's close does not), and the builder of the predicted pipelines behind the game rests while under 3 GB (or a tenth of a PC of more than 30 GB) is available (`gpu/vk/mem_parts.h`) | | `RX_VK_WRITERS_WAIT_MS=` | (Triforce) How long the game's close waits for the files the renderer is writing on threads of their own (`gpu/vk/file_writers.h`; 2000 by default, `0`: no wait and nothing refused, as before). The close ends the process through ExitProcess, which stops every other thread where it is: the pipeline cache's save in play, the order file, a bug capture's or a debug picture, the hang report, the predicted pipelines' list, the frame log. At the close (`HOS_OPT_EXITING`) no such write begins any more and the ones under way are waited for; one still going at the bound is cut, and is one of the files written whole beside the old (`.part` / `.tmp`, then put in place: the prepared record `triforce_prepared.bin` and the `.for` files are now too), or a list whose reader takes whole records only. The log's `vk: exit: the files being written ... were waited for` line says how long, and what was cut | | `RX_VK_OPTIONS_MERGE=0` | (Triforce) The menu writes its own values over `triforce_options.ini` in place at every change, as before. By default (`gpu/vk/ini_merge.h`, `vk_menu.cpp` `save`) each line it writes is merged by its key with the file as it is then: a line changed outside the game while it runs (a text editor, a tool) and not changed in the game since the game read or wrote the file is written as the file has it, so a change made outside and one made in the menu within the same second both stay (a row changed on both sides is the game's); the game takes such a line at its next start. The file is written whole beside the old one and put in its place (in place only where another program holds it open so that it cannot be replaced). The log's `menu: options: ... changed in triforce_options.ini outside the game` line names such lines | | `RX_VK_MEM_PARTS=0` | No `vk: memory, the parts` line. By default every 600 presents it gives the working set, the game's own memory committed and in RAM (a sample of its pages), the rest of the working set in RAM by what it is (a sample too: the program's code, the driver's code, other code, mapped files and views, write-combined or uncached memory, the heaps and other private memory), the shader modules alive and the packed stages, the pipeline cache, the pipelines and stage libraries kept (how many a draw has had, how many were given back) and the allocator's blocks in system memory (`gpu/vk/mem_parts.h`), and what its walks and samples took. It is written on a thread of its own that nothing waits for; the guest window's walk is made again every eighth line | +| `HOS_MEM_ZERO_MARK=1` | (Triforce, 11-22, the performance audit's MEM-13; off unless set; not yet run in the game) Freed guest pages that a Vulkan import holds are zeroed in place, not discarded (`host/hos_mem.c` `discard_pages` and `FreeMemoryPages`, `host/hos_svc.c` UnmapPhysicalMemory). The renderer does not see a bare memset, so its mirror, its captures and the write-back's shadow keep the freed bytes, and a later write-back could put them back over the zeros. With the switch the fill (`host/hos.h` `hos_guest_fill`) marks the pages written, after the memset. Off by default until it is measured: the marked zeros (up to 5.2 GB over a tour) are pages the captures and the mirror syncs then copy, which could hitch an arrival. To measure: two runs a side, the log's "capture ... copied (MB)" and "mirror syncs ... dirty pages", and the frames over 50 ms in each arrival's first 5 seconds | +| `RX_VK_SWAP_RESERVE_MB=`, `RX_VK_TEST_SWAP_MB=` | (Linux; Triforce, 11-22, the performance audit's MEM-141, after Source A's; off unless set, not yet run on Linux) A memory pressure hold that counts the swap. The RAM rules look at `MemAvailable` alone, which a PC with swap keeps up by pushing the game's pages out (a 16 GB Steam Deck's swap filled while it said little). With the reserve set (Source A's is 2560), every 30 presents the PC is short when `MemAvailable` is under 1536 MB or `MemAvailable` and `SwapFree` together are under the reserve, and until both are 256 MB clear of their lines a hold is on: a capture buffer is kept as a spare only up to 32 MB of spares, the staging arenas the GPU is done with are freed as it begins and then after 5 s unused, `malloc_trim` is asked every 600 presents from 64 MB free, idle imports are given back (in all of one hold at most the bytes the PC was short at its worst, still only after 3,600 idle submissions), and an integrated GPU's texture budget is less by what the PC is short, never under 512 MB, coming back 256 MB a look. A PC without swap, or whose swap cannot be read, is never short by the sum. The log's `vk: swap hold: on` and `off` lines say when and what each gave. `0` or not set: no hold, as before; never on Windows. `RX_VK_TEST_SWAP_MB=` plays that much swap free, as `RX_VK_TEST_AVAIL_MB` plays `MemAvailable` (`gpu/vk/linux_pressure.h`, `gpu/vk/vk_memory.cpp` `swap_hold_step`) | | `HOS_MEM_CENSUS=` or `0` | (Triforce) The guest memory census (`host/guest_census.h`), on by default every 5 seconds, on a thread of its own: a `mem: census` line with what the runtime handed out of the guest window by kind (the program and the system region, the heap, the address regions and the game's own pages in them, what lies outside them: the physical maps and first touches), the game's own count against the 9809 MB it is told it has, the pages the game let go that stay committed (zeroed under an import, or discarded with a freed region) and what the give-back gave back, the process's commit and working set, and the PC's commit and RAM; and a `mem: census, the renderer` line with the Vulkan back end's big holders (its imports, mirrors, VRAM in use, the allocator's blocks in system memory, the pipeline cache, the pipelines and stage libraries kept, the shader modules). A count of bits a 4 KB page, kept as each place commits, maps or lets go of guest pages. `0`: no line (and, with the give-back off, nothing kept) | | `HOS_MEM_GIVE_BACK=1` or `2` | (Triforce) Off by default, not yet run in the game: the pages the game let go that stay committed because an import held them (`FreeMemoryPages`, `UnmapPhysicalMemory`) or because they were discarded with a freed address region are decommitted once nothing of the renderer imports them, from 4 GB up (below it pages are handed out without a commit), up to 256 MB a census pass, in runs of 16 MB at most; the renderer answers and runs the decommit with its import locks held (`gpu/vk/vk_memory.cpp` `guest_release_unimported`), so no import can lock them between. `1`: every pass; `2`: only while the PC is short (its commit past 85% of its limit, or under 1.5 GB of RAM available). The census line's `given back` says how much | | `HOS_MEM_MAP_RESULT=0` | (Triforce) `MapPhysicalMemory` answers success even where Windows refused the commit, as before. By default it answers Horizon's LimitReached (`0x10801`), which the SDK turns into its out-of-memory result, and the log says `mem: MapPhysicalMemory ...: the commit failed` | -| `RX_VK_PRECOMPILE_KEEP=1` or `0` | The precompile keeps every pipeline it builds for the session (1), or keeps none of them: they are built for the caches and let go, and a draw that first needs one builds it from there (0). Unset or `auto`: kept where the RAM available at the precompile and what the game holds already (less what reading the pipeline cache added, except on AMD's driver: `own_for_keep`) come to 22 GB or more, on a PC of 20 GB or more (`gpu/vk/mem_parts.h` keep_all_pipelines); the log's `vk: precompile:` line says which and why | +| `RX_VK_PRECOMPILE_KEEP=1` or `0` | The precompile keeps every pipeline it builds for the session (1), or keeps none of them: they are built for the caches and let go, and a draw that first needs one builds it from there (0). Unset or `auto`: kept where the RAM available at the precompile and what the game holds already (less what reading the pipeline cache added, except on AMD's driver: `own_for_keep`) come to 22 GB or more, on a PC of 20 GB or more (`gpu/vk/mem_parts.h` keep_all_pipelines); the log's `vk: precompile:` line says which and why. (Triforce, 11-22, after Casual Gamer's #dev-fixes post) Unset (not `auto`): on a discrete GPU in a PC of under 20 GB every pipeline is kept while the game is told its mapped pages are used (`HOS_MEM_QUERY_USED`, `keep_all_small_pc`): a 16 GB laptop, ~0.6 GB more private memory, frames of 1 s or more 15 and 11 in two tours -> 3 and 6. A card the driver allows under 6.5 GB still leaves them to their first draw first; integrated GPUs keep the RAM rule; `auto`: the RAM rule, as before | +| `RX_VK_TEX_PIPES_AHEAD=0` | (Triforce, on by default; `=0` as before) The texture converters' compute pipelines (the GPU ASTC decoder, the BC encoder, the unswizzle and the swizzle back: `gpu/vk/vk_texture.cpp` `texture_compute_pipeline`, `texture_pipelines_warm`) made through the pipeline cache kept across runs, and all four before the game: in the precompile, right after the cache is read, or at the start when the preparation was skipped at the build. Each was made without a cache inside the first upload that needed it, which with an empty driver cache (a first start, a new driver) waited for the compile, a `vk: slow upload` of 100-320 ms. The ASTC decoder and the BC encoder only where the GPU does not sample ASTC. The `vk: texture converters ready in` line says what it took | | `RX_VK_PIPE_RELEASE=0`, `RX_VK_PIPE_RELEASE_EVERY=` | Every kept pipeline stays for the session, as before. By default, while the PC's RAM is short (under 1.5 GB available at two looks in a row), every 300 presents up to 512 of the kept pipelines no draw has had yet are given back to the driver (`gpu/vk/pipe_release.h`); a draw that needs one later has it built again by a worker. The log's `vk: RAM short: ... kept pipelines no draw has had given back` lines say how many and what the working set did | | `RX_VK_PCACHE_CARRY=0` | The pipeline cache's file carried over a stamp change (`gpu/vk/pcache_carry.h`): at an update's preparation (Setup's "Preparing shaders", `HOOKSHOT_PREPARE_ONLY`) a `mkw_vk_pipelines.bin` made for other shaders is read all the same when it is this device's and driver's, within its bound, and at most 1.25 times its base (the size of the last set built fresh, kept beside it in `mkw_vk_pipelines.bin.base` with the carries since): the pipelines of shaders the update did not change are hits instead of hours of compiling. The cap is over every carry since the base was built, not a quarter more each time, and a carried cache may never pass the file's bound either. The preparation looks at a carried cache's size every 10 s while it builds and at each save: past the cap (or the bound), nothing more is built into it, it is not written, `mkw_vk_pipelines.bin.nocarry` names the new stamp (a preparation begun again after a Cancel starts fresh), and the preparation builds the driver's part again into a fresh cache: the compute pipelines the carried run built, from the program cache, then the stage libraries, nothing translated again (from the driver's own cache for what it had just compiled), which gets a new base. A carried preparation that is stopped (a Cancel, a closed window) saves its cache under the new stamp with its history marked open in `.base`: the preparation begun again reads it as carried still, capped the same (or, when `.nocarry` names the stamp, not at all), until a carried preparation goes through to its end. The log says `pipeline cache carried over a stamp change (N MB, base M MB)`. `0`: a file for other shaders is never read, as before (the rule of 11-14). The play file keeps its own rule | +| `HOS_DRIVER_CACHE_ROOM=0` | (Linux; Triforce, 11-22, the performance audit's PIPE-97, after Source A's and Source B's; not yet run on Linux) The game builds the pipelines its notes predict (about 41,400) into the driver's own shader cache on disk, and NVIDIA's and Mesa's caches hold 1 GiB unless told otherwise and trim themselves to it: part of a full preparation is dropped and compiled again in play. By default the game sets `__GL_SHADER_DISK_CACHE_SIZE` (NVIDIA) and `MESA_SHADER_CACHE_MAX_SIZE` (Mesa: RADV, the Steam Deck) to 10 GiB before Vulkan loads, for its own run and the builder's `--prepare-shaders` run; a size already in the environment stays, and the cache only grows by what is built. The log's `boot: room in the drivers' shader caches:` line gives both values. `HOS_DRIVER_CACHE_ROOM=0`, in the environment or `triforce_env.txt`: neither is set, as before. Nothing is done on Windows, where NVIDIA's driver takes its size from its control panel (`host/hos_main.c` `driver_cache_room`) | +| `HOS_DRIVER_CACHE_FOLDER=0` | (Linux; Triforce, 11-22, the performance audit's PIPE-98, after Source A's; not yet run on Linux) NVIDIA's Linux driver keeps no shader cache at all when `__GL_SHADER_DISK_CACHE_PATH` names a folder that is not there: it neither makes it nor falls back to its own folder, and every prepared pipeline is compiled again at each start. By default the game makes the folder the variable names, with its parents, for the user alone, before Vulkan loads (the builder's `--prepare-shaders` run too); the value is never changed, nothing is made when the variable is not set, and a `boot: __GL_SHADER_DISK_CACHE_PATH= is not a folder and cannot be made one` line says when it could not. `HOS_DRIVER_CACHE_FOLDER=0`, in the environment or `triforce_env.txt`: nothing is made, as before. The driver files its cache by the program's file name: the Linux binary stays `triforce` (`tests/test_driver_cache_folder.py` pins `nx2/build.py`'s name), and `__GL_SHADER_DISK_CACHE_APP_NAME` is never set (`host/hos_main.c` `driver_cache_folder`) | | `RX_VK_PCACHE_CARRY_PROJECT=0` | The early drop of a carried pipeline cache (`gpu/vk/pcache_carry.h`). When the stage libraries are built into a cache carried over a stamp change, every 32nd program of the warm's order is built first, the others waiting for it, and the cache's growth over that sample, per program, times all the programs, is where it ends at this rate. With at least 1500 libraries in the sample, a projected growth of at least twice the room it had when they began (its cap, or the bound, less its size then) drops the carry at once, exactly as past its limit (not written, `.nocarry`, the driver's part built again into a fresh cache), a few minutes in rather than when it gets there: an update whose file was built by an older translator misses nearly every library and filled the cache for 34 minutes before it was let go. The sample is spread over the whole order, not its head: the notes' programs go first and the rest in the program cache's file order, and hits and misses come in stretches of it, so a head that misses while the rest hits would have been dropped for nothing. The notes' programs wait for the sample (a few minutes cold). The log says `the pipeline cache carried over a stamp change would grow to about N MB at this rate`. The compute pipelines' pass and the precompile keep the look past the limit alone. `0`: dropped only once past its limit, and the warm in its own order, as before | | `RX_VK_PCACHE_MAX_MB=` | The bound on the pipeline cache's file, `mkw_vk_pipelines.bin` (`gpu/vk/pcache_bound.h`): a file over it is not read (the cache starts empty and the next save writes a new one), and data over it is not written (the file keeps its last save). Unset: an eighth of the PC's RAM, 2048 MB at least and 8192 MB at most. `0`: no bound, as before | +| `RX_VK_PCACHE_LIBS_ONLY=1` or `2` | (Triforce, 11-22, the performance audit's PIPE-78, after Source A's and Source B's; `gpu/vk/pcache_libs.h`) Off unless set. `1`: when a fresh pipeline cache file begins (none read for these shaders), on NVIDIA's own driver with 20 GB of RAM or less, 512 MB or more in the driver's own cache folder (GLCache) and the stage libraries kept in our cache, the file keeps the stage libraries and the compute pipelines alone: whole graphics pipelines are left to the driver's cache and there is no play file, so a file of ~650 MB instead of 1.1-1.6 GB is read and held. Recorded in `.lean` for that file's life (a full file stays full until the next fresh one); not decided in a start that only prepares. `vk: pipeline cache: libraries only (...)`. `2`: lean at every fresh file whatever the PC (the lab's). Whole pipelines then load 3-4 times slower from the driver's shared, capped cache (`gpu/vk/vk_draw.cpp` `pcache_libs_decide`, `build_graphics`, `pcache_play_open`) | | `RX_VK_PCACHE_CARRY_MB=` | (Triforce, 11-18) The carry of the pipeline cache's file over an update that changed the translation (`gpu/vk/pcache_bound.h` `carried`): a file for other shaders is read all the same when it is no larger than this (1536 MB unset), leaves room under the bound for as much again and the PC has 31 GB of RAM or more (Windows), so the driver finds what it compiled for the shaders that did not change (AMD's driver keeps no cache of its own for these). `0`: never carried, the rule of 11-14 alone | | `RX_VK_PCACHE_PLAY_FILE=0`, `RX_VK_PCACHE_FOLD_MB=` | The pipeline cache's play file (`gpu/vk/pcache_play.h`, NX2navi's play cache): once the preparation is done, what a run compiles goes into a second cache with a file of its own, `mkw_vk_pipelines_play.bin` beside the first (a creation asks the first cache without compiling, then the play cache), and a save writes that small file; `mkw_vk_pipelines.bin` is written only when the preparation built into it. Before, a save after a few new pipelines wrote the whole file again (2.6 GB at the owner's start after its precompile). At a start whose play file has reached `RX_VK_PCACHE_FOLD_MB` (64) it is folded into the first cache, that session builds into it as before, and its first save writes it and removes the play file. Needs `pipelineCreationCacheControl`. `0`: no play file, as before | +| `RX_VK_PCACHE_OWN_PASSES=0` | (Triforce, 11-22, the performance audit's PIPE-95, after Source A's) The back end's own compute passes (FSR 3.1's dozen and its HDR output pass, the anti-aliasing's, the texture converters') are made with a bare creation into the main pipeline cache, as before. The default makes them the way a dispatch's pipeline is made (`gpu/vk/vk_draw.cpp` `shared_cache_compute`, `cached_compute`; `gpu/vk/pcache_play.h`): the main cache asked without compiling, a miss compiled into the play cache, a driver fault caught (the pass is then not made). Bare, they grew the main cache by a little, whose file is written in play only after 64 MB of growth, so they waited for the exit's save of the whole main file (650 MB fresh, 2 to 3 GB with the stage libraries), and an exit that cut that write short had them compiled again at the next start, inside the first upscaled frame. Without a play cache (no `pipelineCreationCacheControl`, or `RX_VK_PCACHE_PLAY_FILE=0`) only the guard is new | | `RX_VK_WARM_SAVE_MIN=` | (Triforce) Checkpoint saves of the pipeline cache during the shader preparation's long builds (`gpu/vk/warm_checkpoint.h`, `tests/test_warm_checkpoint.py`): while the driver's part (the stage libraries, about two hours cold on an RX 7900 XT) and the precompile build, `mkw_vk_pipelines.bin` is saved every 10 minutes once at least 1000 were built since the last save, so that a restart, a crash, a power cut or a kill goes on from the last checkpoint (what it saved comes back as hits) instead of from nothing. Each goes through the preparation's own save: the file's bound, a carried session's cap with its history kept open as a stopped one's (`RX_VK_PCACHE_CARRY`), the file written whole with its `.for` and `.base`; it is left out while a copy of the cache would not fit in the RAM available with 2 GB to spare or the disk lacks its size and 1 GB (asked again a minute later), fetched with room for what is built meanwhile, written to the disk before it takes the old file's place, given up when the window is closed or Setup's Cancel is pressed (the stop's own save writes the cache), due only while the part moves (a stuck one goes silent), and it does not count the part as made (the next start still runs it). Beside the file, `.own` says it holds this part's pipelines alone, so that a part over the file's bound after a crash ends as a new file over the bound, not as an older build's file made again from nothing. The log says `vk: the driver's part so far saved for a restart: N MB in X s`. Not on Intel unless `RX_VK_PCACHE_PLAY_SAVE=1` (its driver crashed fetching the cache while pipelines were built with it); `RX_VK_PCACHE_PLAY_SAVE=0`: on no card. `0`: saved at the end alone, as before | +| `RX_VK_PRECOMPILE_SAVE_S=` | (Triforce, 11-22, the performance audit's PIPE-23, after Source A's; `gpu/vk/precompile_save.h`) A precompile that keeps its pipelines and built for this many seconds or more (30 by default, at most a day) saves the pipeline cache before the title, and says what that wrote: `vk: precompile: N s of building: the pipeline cache saved before the title (... KB in ... ms)`, or why nothing was. It was first saved in play (present 300, after 32 more builds; on Intel only at a clean close), so a crash, a kill or a power cut early in play cost the whole build again at the next start. The save is the play file's alone while there is a play cache (`gpu/vk/pcache_play.h`): the main file, 2 to 3 GB with the stage libraries, is not written before the title. Not when the game went on while workers still build, after a stop, or in a start that only prepares (those save by themselves); the precompile that keeps nothing saves before its record, as before. Also: a preparation without stage libraries counts the compute pipelines it builds into the main cache for that file's save, and saves the cache at its end before its record says the part is made (it ended unsaved). `0`: none of this, as before (`gpu/vk/vk_draw.cpp` `precompile_end_save`, `prepare_compute_pipeline`, `prepare_run`, `pipeline_cache_save`) | | `RX_VK_FETCH_BOUNDS=0` or `1`, `RX_VK_NAN_BITS=0` or `1`, `RX_VK_GATHER_OFFSET=0` or `1` | The translation's vendor and driver workarounds (`gpu/vk/vk_vendor.h`) off or on for one run, whatever the GPU; the log's `vk: vendor fixes` line says which are on. By default: out-of-bounds texel fetches answered with 0 on every GPU but NVIDIA's; NaN tests on a value's bits on AMD's older Windows shader compiler (RX 5000 and RX 6000); gathers nudged by 1/512 of a texel on AMD's and Intel's drivers. They change the shaders' translation (one preparation of the shaders) | | `RX_VK_NAN_SELECTS=0` | (Triforce, after NX2navi's e38f42a, issue #156) Where the NaN tests are made on the bits (`RX_VK_NAN_BITS`: by default AMD's older Windows shader compiler, RX 5000 and RX 6000), a float select of a constant whose bits are a NaN (an integer mask the translation types as a float) is also made a select of the bits, so that compiler cannot fold it into stray black or white blocks (`gpu/vk/spv_nan_select.h`: 32-bit selects, and 16-bit ones in a module that has a 16-bit integer type). It is applied where a module is made (`make_module`), outside the translation stamp: no program is translated again, nothing changes on any other card, and on those cards only the modules that have such a select are new to the driver's caches. So that those are compiled before the game rather than at their first draw or dispatch, the driver's part of the preparation, the light precompile's record and the predicted pipelines' record have a stamp of their own there: the first start on those cards goes through them once more, everything but those modules coming back from the caches (the variants on the driver's part's skip list are carried over). `RX_VK_NAN_SELECTS=0` leaves the selects as they are (the NaN tests on bits stay). The log says `vk: NaN selects: module , float select(s) of a NaN constant made select(s) of the bits` for the first four modules and every 64th | -| `RX_VK_MIRROR=0`, `RX_VK_MIRROR_MB=` | No VRAM mirrors of the game's memory pools (on for discrete cards); their budget | +| `RX_VK_VB_RANGE=0`, `RX_VK_MESH_VB_REBIND=1` | A draw's vertex streams (`vk_draw.cpp` `draw`). `RX_VK_VB_RANGE=0`: all 16 bound again whenever one changed, as before v0.6.0 (by default only the ones that changed, `vertex_bind_range`). `RX_VK_MESH_VB_REBIND=1` (off by default: about 3% at the Market in the lab, and its tickets not reproduced): after a mesh pipeline's bind the next vertex draw binds all 16 streams again with their strides (by default they stay as they were bound, as before: a mesh pipeline has the strides static, and its bind leaves them undefined for the next pipeline that has them dynamic, as it does the topology; `tests/test_draw_binds.py`) | +| `RX_VK_MIRROR=0`, `RX_VK_MIRROR_MB=` | No VRAM mirrors of the game's memory pools (on for discrete cards); their budget (unset: by the card, `gpu/vk/mirror_budget.h`: a quarter of the video memory on a discrete card over 4.5 GB, at most 3 GB; an eighth on cards of 4.5 GB or less and on integrated GPUs up to 6.5 GB; the log's `vk: VRAM mirrors' budget` line. `=768` is what a 6 GB card had before) | +| `RX_VK_MIRROR=0`, `RX_VK_MIRROR_MB=` | No VRAM mirrors of the game's memory pools (on for discrete cards); their budget. Unset, the budget goes by the card (`gpu/vk/mirror_budget.h`): a quarter, at most 3072 MB; an eighth on cards of 4.5 GB or less and on integrated GPUs up to 6.5 GB. Discrete cards between 4.5 and 6.5 GB had the eighth before 11-21: `RX_VK_MIRROR_MB=768` is a 6 GB card's old size. (Triforce, 11-22, the performance audit's MEM-51) Without mirrors (`RX_VK_MIRROR=0`, Intel's integrated GPUs) the replay no longer notes what only a mirror's write-back reads (`gpu/vk/vk_memory.cpp` `wb_notes_off`: the report words of `memory_cpu_owned`, the written pages of `shadow_written_to` and `guest_cpu_wrote`): those lists grew to 65,536 words and 8 million pages, were dropped and grew again. No switch of its own: with mirrors on everything is noted as before | | `RX_VK_DEMOTE_QUICK=0` | A pool the CPU and the GPU both write is tried in guest memory (`vk_memory.cpp` `demote_trials`); by default a trial whose GPU time a frame is over 3 times its baseline and 5 ms more, after 3 frames or more, ends then, settling or measured, and is judged as a slower one is (mirrored again, measured again: `gpu/vk/demote_watch.h`). `0`: every trial runs its 10 settling and its measured frames, as before (in the owner's logs, some twenty frames of 100 ms for a pool at 106 ms against 7.4) | +| `RX_VK_IMPORT_CHUNK_MB=` | The game's memory is imported in whole chunks of its address space, 16 MB each (`0`: by the game's own pools). (Triforce, 11-22, the performance audit's MEM-16, after Source A's) Unset, the chunk is 4 MB where the mirrors are on and whole on an NVIDIA card (NVIDIA before Turing by default, a card without sparse residency buffers, `RX_VK_MIRROR_PIECES=0`): a whole mirror copies all of its import into video memory, the padding too, and Source A measured about a quarter less mirror memory with 4 MB for about half again as many imports. With the mirrors in pieces, without mirrors, and on AMD's and Intel's cards it stays 16 MB (an import stalled the replay 20 to 75 ms on an AMD card). The log says `vk: imports in chunks of N MB` once. An imports-ahead file written at 16 MB is read as it is. `16`: as before on every card | | `RX_VK_IMPORT_AHEAD=1` or `0` | On by default with NVIDIA's driver (`=0` off). Off for test12 to test14: the GPU hung at the title in 6 of 12 lab runs; the cause, the game's freed pages decommitted under an import the background thread had not finished (`bg_held`), is closed and the lab's A/B with it was clean in 10 of 10 (`gpu/vk/import_ahead.h`). The ranges of the game's memory a run imported on a binding's own thread (8 to 17 ms each on an RTX 5090), whole as they were imported (their span over the game's pools included; the first version kept 16 MB chunks, which a binding across their ends imported again as one), are kept in `triforce_imports_ahead.txt` beside the game, at most 256, and the next runs import them on the background import thread as soon as the game's pools reach them, 4 at a look, at each change of the pool table and every 8 submissions (`gpu/vk/import_ahead.h`), each range once a session, and none while the PC's RAM is low (under 3 GB available, or a tenth of the RAM: the RAM's rule gives idle imports back then, and the two took turns on the same ranges in the owner's test9 session). A file of the first version's chunks is read as it is. Unset: off (it was on with NVIDIA's driver only, as `RX_VK_ASYNC_IMPORT` is); `1`: on any card; `0`: off. (Triforce) A race that fits the hang is closed since: the host decommitted the game's freed pages under an import the background thread had locked and not yet put in the pool table (`vk_memory.cpp` `bg_held`; `RX_VK_BG_IMPORT_HOLD=0`: not, as before, for the A/B); it stays off until a lab A/B with `=1` hangs no more. The log's `vk: imports ahead` lines count them and name the ranges sent, `vk: the game's pages ... are freed under a background import` the frees that window met; the stall line's "pools imported where bound" is what is left | | `RX_VK_IMPORT_TOUCH=0` | Off: the pages of a range of the game's memory are not read before the driver imports it. On by default, on Windows, for the imports made on the background thread (`gpu/vk/import_touch.h`): the driver locks a range's pages inside `vkAllocateMemory`, faulting in the untouched ones, and a queue submit from another thread waits for it; the ranges imported ahead are up to 1 GB. Read first, outside the driver, the allocate of the 288 to 752 MB ranges went from 16 to 93 ms to 4 to 13 ms on an RTX 5090, and the longest queue submit beside an import in play from 71 to 93 ms to 10 to 12 ms. The log's `vk: import` lines give a slow import's parts: the first 12 of a session, and the first 48 of 64 MB or more. | +| `RX_VK_IMPORT_LEAVING=0` | (Triforce, 11-22, after Casual Gamer's #dev-fixes post) The game's freed pages under an import already given back whose memory is not freed yet (an evicted pool, a retired pool's import given back) are discarded or decommitted, as before. By default they are zeroed in place, as under a live import (`gpu/vk/vk_memory.cpp` `memory_guest_imported`, `imports_leaving_overlap`): the driver keeps those pages locked until the GPU is past them, and a discard there lost the device (v0.4.0 on a 16 GB laptop: 2 tours of 5 without, 0 of 53 with). The log says `freed under an import still being given back` for the first 8 and every 64th (`tests/test_import_leaving.py`). | +| `RX_VK_KEEP_RETIRED=1` or `0` | A retired pool's import (a pool merged into a larger one) kept for the session (1) or given back once the GPU is past it (0; `gpu/vk/vk_memory.cpp` `retired_imports_give_back`). Unset: given back on Linux and with NVIDIA's driver, kept on Windows with AMD's and Intel's (device losses right after they went back there, issues 17, 309, 571). (Triforce, 11-22, after Casual Gamer's #dev-fixes post) Unset, those kept are given back all the same past the import rule's high mark of the system heap's budget (`RX_VK_IMPORT_HIGH`, 85%) or for a minute after an import was refused for want of room (`gpu/vk/mem_parts.h` `retired_pressed`; the log says `retired imports given back after all`): on a Radeon 780M the retired imports filled that heap, and 24.9% of the draws were left out for want of their pools (0 with it; refused imports 16 -> 1). `=1` keeps them always, as before. | +| `RX_VK_CAP_SPARE_MB=` or `0` | The submit-time capture's spare buffers, kept for the next captures (12 at most, `gpu/vk/vk_memory.cpp` `cap_spare_max`), held to that many MB; `0`: up to 768 MB, as before. Unset: 32 MB on a card the driver allows 6.5 GB or less, and (Triforce, 11-22, after Casual Gamer's #dev-fixes post) on a PC of under 20 GB of RAM (`gpu/vk/mem_parts.h` `spare_capped`; the log says `capture buffers: 32 MB of spares at most`): on a 16 GB laptop the spares held ~0.45 GB that Windows paged out and in again, capped 10-18% fewer page-ins. Not on AMD's cards, where a capture buffer made anew costs about 100 ms: they keep their spares | +| `RX_VK_IMPORT_RAM=1` | The imports' second trigger (`gpu/vk/vk_memory.cpp` `imports_give_back`): with the PC's RAM short (under 1.5 GB available) every import unbound for ~10 s of play is given back, up to 1 GB a pass. (Triforce, 11-22, after Casual Gamer's #dev-fixes post) Off unless set: a 16 GB PC has under 1.5 GB available nearly all the time, so it fired every ~20 s and the imports it gave back came straight back (690 to 1216 imports in a 13-minute tour). The system heap's trigger (`RX_VK_IMPORT_HIGH`) stays as it is. | +| `RX_VK_KEEP_RETIRED=wait`, `RX_VK_RETIRED_REFUSED=1` | (Triforce, 11-22, the performance audit's MEM-21, after Source A's; `gpu/vk/retired_give_back.h`; **off unless set**, not yet run in the game) A pool remade larger is retired, and once the GPU is past it its import is pinned RAM nothing binds. Unset, as before: given back at once on Linux and with NVIDIA's driver on Windows, kept for the session with AMD's and Intel's on Windows (given back at once there, an Arc B390, an Ally X and an RX 6600 lost the device right after); `RX_VK_KEEP_RETIRED=0` gives back at once and any other value keeps, as before. `wait`, on any driver: given back `RX_VK_RETIRE_SECS` (30) seconds after the pool retired, `RX_VK_RETIRE_MIN_SECS` (2) while the system heap is past 85% of its budget; with it the imports given back and not freed yet are left out of what the pressure give-back (`RX_VK_IMPORT_HIGH`) reads. `RX_VK_RETIRED_REFUSED=1`: also the retired pools whose mirror was refused (`RX_VK_NO_MIRROR_POOLS`, a pool over 1 GB, video memory refused), never given back before on any OS; on with `wait` unless `=0`. The `vk: imports:` line every 1200 mirror epochs says how many retired pools are still imported | | `HOS_CRASH_THREADS=0` | Off: a crash report has the faulting thread alone. On by default, on Windows (`host/crash_threads.h`): a crash or fatal error's report ends with one `CRASH: beside it, thread ...` line for every other thread, its host stack as module and offset, taken at the report's start on a helper thread; the threads running code come first, then the ones waiting in the system. For the crash while a save loads, which reads an object whose memory holds something else: the lines say what the other threads were doing. | | `HOS_CRASH_TEST_S=` | (the lab's) A host thread reads address 8 that long after the start: a crash report on purpose, to look at one from a run in the game (its other threads' lines above). Not for play. | | `EDEN_NX2_GM_LEAN=1` | Only the global-memory helpers a program's instructions call are put into its SPIR-V (`gpu/vk/eden/shader_recompiler/backend/spirv/gm_lean.h`). A stage that reads or writes through a pointer no storage buffer was tracked to carried every helper, six and for a storing stage eighteen, each with a search over every storage buffer in it. Off unless set: the SPIR-V is then what it was. An `EDEN_` name: each setting has its own program cache. The translator's files are in the translation stamp, so a build with this code translates every program again once, whatever the switch says. | | `RX_VK_UPLOAD_AGAIN_MS=` | A texture uploaded before whose memory the game wrote again, met by a draw after the frame's uploads took this long (16), is uploaded at the next frame instead; its last bytes are drawn meanwhile, at most 8 frames in a row (`gpu/vk/upload_owed.h` `later`). New textures are never left so (theirs is `RX_VK_HOLD_MS`'s rule). `0`: every one at once, as before. The log's "left for the next frame" lines and the stall line count them. `RX_VK_UPLOAD_AGAIN_FRAMES=`: a texture's upload is left so at most that many frames in a row (2: its last bytes drawn a frame or two at most; `8`: as before). With 16 ms nothing was ever left in the lab's runs (the frames with re-upload bursts took 9.8 ms of uploads); `3` spreads a burst over the frames after it, its textures a frame or two late meanwhile, which is why it is not the default | | `RX_VK_UPLOAD_TORN=0` | (Triforce, wp18-flash) A texture uploaded before, 1 MB or more, whose storage the CPU has rewritten since the copy its re-upload would read (a capture's copy of one of its whole pages, every 16th and the last, differs from guest memory at the draw's look) keeps its last bytes for that frame and is uploaded at the next frame with what that frame's capture took (`gpu/vk/upload_torn.h`): a copy taken while the game was still writing the texture was test29's white frames at the Market after a tour (BC6H bytes passing through, decoded to bright colours the bloom spread over the picture). At most one frame in a row, and not for a texture found so in the two presents before (one the CPU rewrites every frame is uploaded as before). Not a dispatch's upload. `0`: uploaded at once, as before. The log's `re-uploads of storage the CPU rewrote since its capture` lines count them, those found and those left (with `0` too: the measure of how often a copy may have been taken mid-write) | | `RX_VK_REUPLOAD_EARLY=0` | The textures the CPU keeps rewriting (uploaded again from its writes twice or more within 120 presents, 1 MB or more, sampled in the frame or the one before, not drawn into by the GPU) are looked at once on the replay thread before a parallel replay's segments begin, after the dry pass, their uploads in the recording before the segments: not made by a segment under the recording lock, where the other recording threads queued behind them (lab S2 after a tour: 4 to 12 such uploads of 28 to 84 MB in ~13% of the frames, the descriptor time 24 ms over the threads against 3). `0`: the segments' looks alone, as before. The log's `textures the CPU keeps rewriting, looked at before` line says how many and how long (`gpu/vk/reupload_early.h`) | +| `RX_VK_ZERO_HOLES=0` | (Triforce, Linux, on by default; `=0` as before) The all-zero check of a big texture's storage (32 MB or more: cleared on the GPU when every byte is 0; `RX_VK_ZERO_UPLOAD=0` turns the check off) reads only what the guest memfd holds data for (`lseek` SEEK_DATA and SEEK_HOLE, `gpu/vk/guest_holes.h`), not its holes: a page the game never wrote reads as zero, but a read through the shared mapping made it RAM (the 288 MB shadow map at each Hyrule Field arrival, 200-300 ms on a Steam Deck). Where the window is not a memfd, or a seek fails, the whole storage is read. The `is all zero` and `is not all zero` lines say how many MB were read. `=0`: the whole storage always | | `RX_VK_UPLOAD_SAME=0` | (Triforce) A texture uploaded before whose storage the CPU wrote again is checked against the bytes its image holds (`gpu/vk/upload_same.h`): the units of its storage the writes reached (each level's and layer's GOB-block rows, as its upload reads them) whose bytes hash as at their last upload are not uploaded, and when that is all of them nothing is. The upload of the rest reads the copy the check hashed, so that what is kept is what the image got; any other upload and any GPU write forget it. Textures of 16 MB or less uploaded on the CPU (block-compressed, or under 8 MB), not sparse; one whose checks find nothing the same 4 times in a row is uploaded unchecked for its next 64 re-uploads. Lab S2 after some tours: the game rewrote a few big BC textures the Market samples with the bytes they had, 40 to 60 MB every 4 to 5 frames (the Market at 62-78 fps against 83-96, ~26 GB of uploads in its stay). `0`: every re-upload made as before. The log's `re-uploads checked against the bytes their images hold` line says what it looked at, found the same and uploaded. `RX_VK_UPLOAD_SAME_KEEP=0`: the check hashes every byte it looks at; by default a page the submit-time capture keeps a copy of is hashed once and its hash kept with the copy until a capture copies other bytes into it (about half of those textures' pages passed through other bytes between two captures) | | `RX_VK_UPLOAD_SAME_ASTC=0` | (Triforce) ASTC textures decoded on the GPU (kept as RGBA8, BC3 or BC1; 2D only) are checked too, by `RX_VK_UPLOAD_SAME`'s check: one unit a layer, its levels from the first one the image keeps, which is what the GPU decoder stages of a layer a write reached; the staging reads the copy the check hashed. Lab S2's Market after a tour: the game rewrote such textures with the bytes they had (1536 x 1536 ASTC 5x5 of 11 levels, 768 x 768 kept as BC3, 512 x 512), staged and decoded again unchecked, about 1.4 GB in 10 s (90 fps down to 72, the 1% low 57 to 33). `0`: ASTC re-uploads made as before, every layer a write reached staged and decoded. The `re-uploads checked` line says what of it was ASTC; the `re-uploads not checked` line every 600 frames counts the re-uploads made without the check, by route and by why | | `RX_VK_TWIN_SAME=0` | (Triforce) The parallel replay's twin (a texture an earlier segment samples that a later segment of the same submission uploaded, uploaded again in the earlier segment's uploads so that its draws see the bytes as submitted) uploads only what that submission's uploads into the image asked for (`gpu/vk/upload_same.h` `note_upload`; the whole storage when one of them was whole or the image was made again), and where every unit it reads has a hash kept by `RX_VK_UPLOAD_SAME`'s check, reads a copy of them hashed as copied: when the copy holds the bytes the hashes are of, the hashes are kept. Before, the twin uploaded the whole storage and, unchecked, forgot the hashes the later segment's checked upload had just kept, so the next rewrite of the same bytes was uploaded again, the twin with it, every few frames: lab S2's Market after a tour, about 97% of what was still uploaded unchecked in those loops, up to 6.9 GB in one stay. `0`: the twin uploads the whole storage and forgets the hashes, as before. The log's `the parallel replay's twins` line every 600 frames counts them, the bytes uploaded against the whole, and how many kept the hashes | | `RX_VK_UPLOAD_EARLIEST=0` | (Triforce) A parallel replay's upload of a texture, made by the worker of a later segment of the submission while an earlier one is still being recorded, is recorded into the init command buffer of the earliest segment still being recorded (submitted before that segment's uploads and commands; `gpu/vk/upload_earliest.h`, `vk_texture.cpp` `UploadRedirect`). Every segment before it has ended its recording and no longer samples the texture, and every one from it on finds the upload at or before its place in the queue, so no earlier segment uploads the texture again (the twin, `RX_VK_TWIN_SAME`): it is uploaded once, the same bytes. Lab S2's heavy Market loops (round 48) uploaded 2.1 GB of twins in their last minutes, the big streamed or new textures each uploaded once by a later segment and again by an earlier one; those twins took the frames' upload budget (`RX_VK_UPLOAD_NEW_MB`, `RX_VK_UPLOAD_NEW_MS`), and the new textures after them were left for the next frame, the picture held. Only an upload staged from the CPU as it is recorded is moved: not one read on the GPU (the GPU unswizzle, from the pool) or decided from live memory (the all-zero clear), which in the earlier segment's init would run before a segment between writes that memory on the GPU. Not for a texture the GPU drew (its contents are the GPU's; the CPU's writes over it, `RX_VK_CPU_OVER_GPU`, keep their own path) or a sparse one: those keep the segment's own uploads, and the twin. The uploads' GPU time in an init command buffer is not in the segments' GPU time. `0`: every upload in its segment's own uploads, the twin as before. The log's `uploads recorded into an earlier segment's init command buffer` line every 600 frames (with `0` too) counts them and the bytes, and how often and how long a segment's end waited for its init lock, each segment's own, held while such an upload (or a new image's first layout change) is recorded into its init (its submit waits as long) | +| `RX_VK_SHADOW_STAGE=0`, `RX_VK_CAP_UNCACHED=1` | (on by default; the row is Triforce's, 11-22, with the performance audit's MEM-39) A capture buffer asks for cached host memory, but a driver can hand out write-combined memory, which the CPU reads at about 0.3 GB/s. Such a capture's texture pages are copied into a staging block in ordinary memory first, and to the buffer from there, and every CPU read of them uses the staging (`gpu/vk/vk_memory.cpp` `capture_make`, `cap_page_bytes`): the shadow's copies, and since MEM-39 the `RX_VK_TEX_SAME` compare of each copied texture page with its shadow copy, which still read the buffer. `RX_VK_SHADOW_STAGE=0`: no staging, every read from the capture buffer, as before. Where the buffers are cached (the 600-take `shadow staging` line counts them) nothing is staged and nothing changes; the first buffer that is not cached is logged with its memory type. Lab only: `RX_VK_CAP_UNCACHED=1` asks for the capture buffers as memory written in order, which is uncached where the driver has such a type, to make the slow case on any PC, and the `RX_VK_LAB_AB` knob `texsamemap` reads the buffer in the compare in its phases | | `RX_VK_SHADOW_KEEP=0` | (Triforce) A texture page the submit-time capture takes with the bytes its shadow copy holds (the copy the uploads read, `RX_VK_TEX_SAME`'s compare) keeps that copy as it is, not copied again, its kept hash with it (`gpu/vk/shadow_prev.h`). Lab S2's Market after some tours: about half of the ~400 texture pages a capture took, six captures a frame, held their copy's bytes. Only for a page the capture takes once; any other is copied as before, and the shadow holds what the capture holds either way. `0`: every page copied, as before. The log's `texture shadow (600 takes)` line counts the pages compared, left, given back and copied | | `RX_VK_SHADOW_PREV=0` | (Triforce) A texture page the capture takes with other bytes keeps the copy it had, and that copy's hash, beside the new one (`gpu/vk/shadow_prev.h` `Prev`), and a capture that brings those bytes back gives it back, hash and all: no copy, and the re-upload check after it finds the hashes kept (lab S2's Market after some tours: the other half of those pages went back and forth between two versions, and a check hashed each again after every change, 0.2 to 0.3 ms a frame). A kept copy that does not come back within 8 changes of its page, or within 64 captures, goes; at most 64 MB are kept. `0`: none kept, a changed page copied over as before. The `texture shadow` line says how many are kept and how many went | | `RX_VK_SHADOW_VERS` | (Triforce) The copies a texture page keeps beside its own in the shadow (`gpu/vk/shadow_prev.h` `Vers`): `RX_VK_SHADOW_PREV` keeps one, and a page whose captures bring more than two versions (A, B, C, A every few frames) is copied again for every version but the one before the last. `2` to `4` keep that many, newest first, and a capture that brings any kept version's bytes takes that one back, the copy it had becoming the newest, so each version a page cycles through while it is still among the kept copies is copied once: 4 KB a version, under the same 64 MB cap. Unset or `1`: one copy, as before. The `texture shadow` line counts the pages given a copy of their own, one before it, and two or more before | @@ -230,8 +299,15 @@ case-sensitive, and an unknown one is ignored. | `RX_VK_VIEW_KEEP=0` | Any texture's end (an eviction, a dead render target let go, a texture made BC3 from BC1, the compaction moving one) makes every texture look of every recording thread stale, and each is resolved anew when next met (the slot, the cache, the view, under the parallel replay's lock), as before. By default a thread that finds the destroy generation moved keeps the looks of the textures whose views did not go: those are noted by their creation number before the generation moves (`gpu/vk/view_keep.h`, `vk_draw.cpp` `texture_view_of`); never a look of no texture or no view, nor a flat texture's 3D copy. More than 1024 gone since a thread's last pass: all resolved anew. The log's `vk: texture looks kept` lines count them, the stall line the views resolved anew. After NX2navi's `tex_gen_undo` | | `RX_VK_DESC_KEEP=0` or `check` | The per-binding reuse of a draw's descriptors (`gpu/vk/desc_keep.h`, on by default): a draw that builds its descriptors takes, for each sampled texture, user uniform buffer, stage's constants and read-only storage buffer, what the program's last draw on that thread resolved it to, while everything it was resolved from is the same: the texture's handle, its slot's registration, the samplers', no texture destroyed or made again, no texture left for later and the same submission; the buffer's address and size, no inline uniform update, no pool moved or mirror made, the same recording. `0`: every descriptor resolved, as before (so also with `RX_VK_NO_DESC_REUSE`, `RX_VK_SAMPLER_CACHE=0` and the descriptor diagnostics). `check`: every descriptor resolved too, and one that comes out otherwise than what was kept for the same inputs is counted and logged (`vk: descriptor reuse check`): the way to try the rules. The frame statistics' `a frame's descriptors` line says how many were resolved and taken, how many pushes were whole, of the changed bindings or of nothing, and how many sets were written, through a template, or bound again | | `RX_VK_DESC_SIG=0` or `check` | The whole-draw skip by the signature of a draw's descriptor inputs (`gpu/vk/desc_sig.h`, on by default with the per-binding reuse): a draw whose bindings moved, of the program whose descriptors its last draw pushed in that command buffer, first reads what each of them is resolved from (a texture's handle and its slot's version, a user uniform buffer's address and size, a read-only storage buffer's address and size words, a driver bank's stage generation) and the stamps the per-binding reuse gives each kind; when that equals the signature the pushed descriptors were built from, and their build kept every one, the build and the push are left out (a build would take each of them again and push nothing). Push programs only, of the kinds the per-binding reuse keeps: not those with storage buffers they write, tables, texel buffers or storage images, not the mesh fallback or RenoDX's. `0`: built, as before. `check`: built too, and one that comes out otherwise than what is pushed is counted and logged (`vk: descriptor signature check`). The frame statistics' `a frame's descriptor signatures` line says how many draws read theirs and how many were kept whole | +| `RX_VK_DESC_COUNTS=0` | (Triforce, 11-22, the performance audit's DESC-22, after Source A's; `gpu/vk/desc_counts.h`) `build_descriptors` walks the program's stages at every build to sum its buffers, images and texel views, as before. The default keeps the three sums packed on the program (`Program::desc_n`, 10 bits each, in the padding after `ready` so that `sizeof(Program)`, which the shader caches' stamp holds, stays what it is): stored at the program's first build once it is translated, read at every later one. A program with more than 1023 of a kind walks at every build. A build that went by kept counts and had one of its three vectors move is made again, walked, and that program walks from then on (one log line). The lab's knob `descwalk` (`RX_VK_LAB_AB`) is this switch in its phases | +| `RX_VK_PROF_ALL=1` (the dispatch's pick clock) | (Triforce, 11-22, the performance audit's DESC-30, after Source A's and Source B's) Every profiling scope timed, and with it the clock pair around each compute dispatch's program pick, as before. Unset, the pick's time for the stall line is the change of the thread's exactly timed nanoseconds across the pick (its slow path is timed by `PROF_EXACT`), so a pick the memo answers reads no clock. DESC-30 has no switch of its own, the results being the same: besides the pick, `taau_key` answers from its list parsed once (it wrote the key out and walked the list as text several times a dispatch), `RX_VK_LOG_PUSH` formats its key only when set, the first-pipeline clock is read only when there is one to make, and `texture_frame` fills `RX_VK_LOG_BAKED`'s list only for that switch (read once) and finds its largest textures by a partial sort. The lab's knob `diagold` (`RX_VK_LAB_AB`) puts all of it back in its phases (`gpu/vk/vk_draw.cpp` `dispatch`, `taau_key`, `dispatch_diag_old`; `gpu/vk/vk_texture.cpp` `texture_frame`) | +| `RX_VK_LAB_AB=e_draw:`, `e_dispatch:`, `d_present_after:` | (Triforce, 11-22, the performance audit's REPLAY-51, after Source B's; lab knobs of the in-run A/B, `gpu/vk/lab_ab.h`, nothing without the variable) The `d_` knobs wait once a frame; `e_draw` and `e_dispatch` wait the knob's nanoseconds at every draw or dispatch, first in `draw()` and `dispatch()` so the profiles leave it out, on whichever thread records, and count each wait. The frame's change against the time spun says how much of that path is on the frame's critical path (200 ns at about 3,050 draws a frame is 0.6 ms of work). A phase's end logs `vk: lab A/B: phase e_draw: over:` with its waits and ms a second. `d_present_after` is `d_present`'s control: the same wait just after the present's `drain(0)`, where `d_present` is just before it. Pin `RX_VK_PARALLEL` with the `e_` knobs (the log warns otherwise): Automatic moves draws between threads. Off: one relaxed load a draw. `tests/test_lab_ab_each.py` | +| `HOS_OOT_SHADOW_STEP=` | (Triforce, 11-21, the performance audit's GAME-05) The game gives each nearby character or object a square depth shadow map sized by its size on screen, 384 to 1,920 a side, so while anything moves each frame asks for depth textures of sides not seen before. The one read of the side is rounded up to a multiple of `` (a power of two up to 256; 64 when not set): 25 sizes, which come round again, so the render targets' pool serves them. A map gets at most 63 more texels a side; its projection does not change. `0` or `1`: the game's own side, as before (`host/oot_shadow_step.h`, `host/oot_shadow_step.c`, `nx2/oot_patches.py` `patch_shadow_step`) | +| `HOS_SHADOW_BUCKET=` | (Triforce, 11-21, the performance audit's TEX-85) The game sizes each character's shadow map (square, D32, one level) from the camera's distance every frame and places a frame's render targets first fit in its target heap, so a map a few pixels larger moved every target after it, and each moved one is a new image to the renderer. Such a map wider than 384 is answered the storage size and alignment of the next multiple of the step up (384: sides 768, 1152, 1536, 1920); the texture's own size and layout do not change, only the bytes the game sets aside, by 1.8 to 5.3 MB a map. `0`: every map its own size, as before. Not set: 384, and 0 where the target heap is pinned (`HOS_RT_POOL_MB`, or a 5120-wide output's 900 MB). The log says the step and counts the stepped answers (`host/shadow_slot.h`, `host/nvn_native.c` `shadow_slot_layout`) | | `HOS_OOT_TEX_COMPACT_MS=` | (the lab's) The game's texture data compaction (its `aglTextureDataMgr::Compaction` thread moves a texture's data in the game's heap, a cycle of about 18 frames; idle in a fresh Market, at work at every ask after a tour, where each move is a texture at a new address for the renderer to capture and check): at most one cycle started every n ms. `-1` (a test): none started. Unset or 0: as shipped. The cycles started are logged every 10 s either way ("oot: texture compaction in"). `host/oot_tex_compact.c`, `tests/test_oot_tex_compact.py` | +| `HOS_OOT_TEX_ASK_MS=` | (Triforce) The game's texture data manager asks for a compaction round (above) at most once every n ms: its step, every frame, asks whenever no round is asked for and its flag says so, and each ask let through can start a round that moves a texture to a new address. Unset (or not a number): 1000. `0`: every ask, as shipped. `off`: none. A round started always finishes; an ask held back is made again at the next step. The asks let through and held are logged every 10 s ("oot: texture compaction asks in"). `host/oot_tex_compact.h`, `nx2/oot_patches.py` `patch_tex_ask`, `tests/test_oot_tex_compact.py` | | `RX_VK_TEX_WARM=1` | Textures made and uploaded ahead, while the frames are light (`gpu/vk/tex_warm.h`; off unless set). A texture is made and uploaded where a draw first samples it, and the frame's budget for that (12 ms, 64 MB, the picture held) applies only to targets drawn in the frame before, so an area's first frame, whose targets are new, takes every texture of the area at once: 350 to 800 uploads of 250 to 390 MB, 100 to 180 ms, in the lab's tours. With the switch every registered slot is noted, and at the start of a game's submit that follows a light frame (under 400 draws and dispatches: a loading screen, a menu) the noted textures, 8 frames old at least, are made and uploaded, 6 ms a frame. Bytes the game writes afterwards are uploaded again by the write marks, as before. "vk: textures made ahead in light frames" counts them. `tests/test_tex_warm.py` | +| `RX_VK_TEX_WARM_LIST=1` or `2` | (Triforce, 11-22, the performance audit's TEX-23, after Source A's; off unless set, and only with `RX_VK_TEX_WARM=1`) The textures made ahead by a list of what was drawn (`gpu/vk/tex_warm.h`). Every registered slot made ahead needs the 40% rule, and on an 8 GB card that rule stops the warming at once. With the switch, a BC or ASTC texture a draw uses within 90 s of the game making it is learnt: its shape and eight 64-byte samples of its bytes, at its storage's address. The list is kept beside the game as `triforce_textures_drawn.bin` (32,768 at most; saved every 300 presents when it changed; a file cut short or of another version is not read). At a light frame the ripe slots the list names are made while the textures and 256 MB more fit their budget and the render targets' pool is not being given back; `1`: the others after them, by the 40% rule, once no noted slot is left ripe; `2`: the listed alone. The look itself learns nothing. "vk: textures made ahead by the list" counts listed, unlisted, learnt and no-room. Unset or `0`: every registered slot as above. Not measured: lab proof first (the entry's own default is `1`). `tests/test_tex_warm_list.py` | | `HOS_MEM_LIMIT=1` or `` | (the lab's) The game's memory held to what it is told it has, as the console holds it. The game is told 9809 MB, but nothing was refused until the PC's own commit ran out, and it holds 11.7 GB by its own count in a fresh Market and 13.5 GB after a tour (the `mem: census` line), with its GPU heap within 15 MB of full and its texture compaction running every frame. `1`: `AllocateMemoryPages` and `MapPhysicalMemory` are refused past the 9809 MB, and `UsedMemorySize` is the game's count; a number above 1: refused past that many MB. Unset or 0: as before. What the game does when refused is what the lab is to find out. `host/hos_mem.c`, `tests/test_mem_limit.py` Refused, the game asks again thousands of times, each ask a region taken and freed; a region of the game's own freed with none of its pages in it is no longer discarded (`hos_region_discard_needed`), and the regions it frees with pages in them are discarded with the thread's virtual core given up. Those two were the stutter when walking with the limit on: in a three-area lab run the game's own part of the frame was over 8 ms in 24% of the frames before and in 0.3% after (as without the limit), 63.8 fps to 100.0. **`=1` does not fit this build:** the heap takes 6.9 GB of the 9.8 and an area's resources need 4.7 GB and more, so a save's load can starve and stay on its loading screen for good (the owner's, 2026-11-20, read with the hang watch's stacks: the game at 9,807 MB, its load threads idle). Use a size in MB: `12800` loaded and played four areas without reaching it. | | `RX_VK_DESC_SET_KEEP=0`, `RX_VK_DESC_TEMPLATE=0` | A program over the driver's push limit (32 descriptors on NVIDIA's, AMD's and Intel's drivers) has a set written for each draw that builds its descriptors. By default a set whose descriptors are all as those of the last one written for that layout in the recording is bound again instead, while no texture was destroyed or made again and no pool moved, and a set written is written through the program's descriptor update template (Vulkan 1.1). (Triforce, after NX2navi's `desc_template`) A program within the push limit has its descriptors pushed whole (in a new command buffer, or after another program at its bind point) through a push descriptor template of its own (`vkCmdPushDescriptorSetWithTemplateKHR`, one a program and bind point, from the same entries and packed array as a set's), not with the write structs the driver walks one by one; a push of the changed bindings only keeps its write structs. The frame statistics' `a frame's descriptors` line counts the pushes made through it. With `RX_VK_RECORDER` the packet copies the packed array (`gpu/vk/vk_recorder.h`). `RX_VK_DESC_SET_KEEP=0`: a new set each time; `RX_VK_DESC_TEMPLATE=0`: sets written and pushes made with the write structs; both as before | | `RX_VK_CREATE_NEW_MS=` | While the hold is on, a sampled texture seen for the first time once the frame's makes took this long (8), or its uploads their budget, is not made yet (`vk_texture.cpp` `create_later`): the draw samples the dummy, the frame is held, the next frame makes it. The frame's first make always goes; never a dispatch's. `0`: made at once, as before (a texture left for later was made all the same). The stall line counts the textures made, their time, those not made yet, and the images made | @@ -248,23 +324,50 @@ case-sensitive, and an unknown one is ignored. | `RX_VK_PIPELINE_WARM_FOLLOW=1` | Off by default (test19: the lab's tour showed no gain). `1`: the warm follows the draws: a program read within 64 places (in the notes' order) after one the draws consumed in the last 2 s has its libraries made and linked, one within 192 its libraries only, one farther (or once the draws stopped taking) nothing; and the libraries no draw has had are kept up to 2048, the oldest let go beyond (with the links made from it), so the bound never stops the warm (`gpu/vk/pipe_warm.h`, test18: 2266 links made, 276 taken, the libraries at their bound from the load on); it spends at most 250 ms of the workers' time a second (500 saved up), and never makes again a library it let go. Off: every program read is warmed whole and the libraries stop at 1024, as before | | `RX_VK_COMPUTE_WARM=1` | Off by default (test19: the lab's tour showed no gain). `1`: where the precompile keeps no pipelines, the read-ahead's worker that has just read a compute program play dispatched (`triforce_computes.bin`) makes its compute pipeline, asked of the caches only, as its first dispatch would; the dispatch that takes the program has it. One a program while it waits on the read-ahead's shelf, destroyed when the shelf lets the program go and given back while the RAM is short; once a program variant a session. Off: each compute pipeline is made at its first dispatch, as before | | `RX_VK_PIPELINE_ORDER=0` | Each session writes the order its draws first needed their pipelines in (a new one, or one the precompile kept that no draw had), the first 4096, into `triforce_pipeline_order.bin` beside the pipeline notes now and then, from the presents on a thread of its own (this session's first, then those of the sessions before it has not met). Where the pipeline warm runs (`RX_VK_PIPELINE_WARM`), the next session finds each one's note before play and the warm makes, for a program variant the read-ahead has read, the pipelines that order names for it and no other (32 at most a variant; the notes' other pipelines are not made then, and the warm does not follow the draws): their stage libraries asked of the caches and their links, held for the draws by their place in the order, 1024 at most (beyond, the farthest ahead goes, and none is made farther), and 2048 libraries no draw has had at most. Where the draws are: the middle place of the last 16 pipelines they needed first; a link held 512 places behind it goes with the libraries only it held, and none is made there (`gpu/vk/pipe_order.h`): the save's load-in frame made 86 to 92 ms of pipelines on the draws' threads, the warm's links from the notes let go before the load came (lab round 37: 1964 made, 54 taken). The first session after this writes the file; the next one has it. `0`: neither read nor written, the warm from the notes as before. The stall line counts the pipelines new to the draws, those linked ahead (of them in this order), and the links and stage libraries the draws made themselves; the log's `vk: pipelines made ahead in the order` lines the rest | +| `RX_VK_PIPE_AHEAD=` | (Triforce, 11-22, the performance audit's PIPE-81, after Source B's: its first step; `gpu/vk/pipe_ahead.h`) A count, nothing built: with the last sessions' pipeline order in use (light mode), each pipeline the draws first need is counted by its place against where the draws are: within this many places ahead (1024 unset, at most 4096), further ahead, behind, not in the order, and those within the window that the warm had linked. A line every 600 presents while it grows (`vk: pipelines first drawn, by their place in the order: ...`) says whether a builder of whole pipelines just ahead of the draws would meet a third of them or more, the entry's rule for building it at all. `0`: not counted (`gpu/vk/vk_draw.cpp` `order_met`) | | `RX_VK_VRAM_MB=` | The texture cache's budget as a fixed size, whatever the card. Unset: on a card the driver allows more than 8.5 GB, no size (`RX_VK_TEX_PRESSURE`); on a smaller one 45% of what the driver allows; with `RX_VK_TEX_PRESSURE=0` 70% of it above 8.5 GB, at most 6 GB (the rule before) | | `RX_VK_TEX_PRESSURE=0` or `=` | (Triforce, after NX2navi's `TEX_VRAM_PRESSURE`) On a card the driver allows more than 8.5 GB, with VK_EXT_memory_budget, not an integrated GPU, without `RX_VK_VRAM_MB` and without a quality preset whose budget applies, the texture budget follows the live use instead of the fixed 6 GB: textures go once the driver's count, less the free space in the blocks they can take back, is past 80% of what the driver allows now, the least recently used first, each unused 2 s or more, 64 a pass, down to 2% under that line (`gpu/vk/tex_pressure.h`); the 90% and 97% rules and the compaction's lines stay. While the system's commit is low the fixed size comes back (`RX_VK_TEX_COMMIT_FREE_MB`). `0`: the fixed size, as before; ``: that line (50 to 95). The log's `vk: texture budget` line at the start says which, and the `vk: VRAM:` line every 600 presents the budget (by live use: what the textures may hold before the line) and the line | | `RX_VK_TEX_KEEP_TAKEN=0` | (Triforce, after NX2navi's `TEX_KEEP_TAKEN`) While VRAM is short, a texture uploaded from memory that the game let go 30 presents ago goes first (`RX_VK_LET_GO_PCT`); one the game has taken back before waits 8 times the longest it stayed let go before a take-back, when that is longer (`gpu/vk/tex_pressure.h`): made again, the frame that wants it draws the dummy (NX2navi: Kakariko's haze gone every few seconds). The eviction by age still takes it. `0`: the 30 presents alone, as before | | `RX_VK_TEX_COME_BACK=0` | (Triforce, after NX2navi's pause for drops that come back) Where the budget follows the live use: when in 240 presents 64 or more textures the line let go went and half or more of them were made again within 600, the line stands aside for 600 presents, doubled at each such period up to 7200; a period with fewer back ends the doubling (`gpu/vk/tex_pressure.h` `ComeBack`). The 90% and 97% rules and the compaction's lines still hold meanwhile. The log says when a pause begins. `0`: no pause | | `RX_VK_TEX_STEP=` | The live textures one pass of `texture_sweep` over the budget lets go of at most (64 where the budget follows the live use; the render targets drawn over since do not count), the rest at the next look two presents on, not ten. Set, on any card, the fixed size's too; `0`: no step, one cut as before | | `RX_VK_TEX_COMMIT_FREE_MB=` | Where the budget follows the live use: while less of the system's commit than that is free (the default: the larger of 4 GB and a tenth of the limit, NX2navi's), the fixed size is the budget again (Windows charges a discrete card's memory to the commit, and an allocation that fails there ends the game). `0`: never. Not on Linux | +| `RX_VK_TEX_COMMIT_RESERVE_MB=` | (Triforce, 11-22, the performance audit's OS-05, after Source A's; off unless set, not measured here) On any card, while less of the system's commit than that is free (the page files' room to grow counted in, below), the textures are held to what they hold less the shortfall, never under 1 GB, as the 97% rule takes its excess: the cache's passes give blocks back before an allocation is refused at Windows' commit limit (`HOS_COMMIT_WAIT_MS` waits such a refusal out). Source A's is 2560, on by default there; one `vk: commit limit near` line when it begins and ends and every 600 presents while it holds. Not set or `0`: no cut, as before. Not on Linux (`gpu/vk/commit_room.h`, `gpu/vk/vk_texture.cpp` `texture_frame`) | +| `RX_VK_TEX_COMMIT_GROWTH=0` or `=1` | (Triforce, 11-22, OS-05) Whether the room the page files still have to grow (each one's largest size less its size now, a drive's page files capped together by its free space, asked of Windows at most once a second) counts as free commit, for the reserve above and for `RX_VK_TEX_COMMIT_FREE_MB`'s fall-back alike: Source A's first rule, without it, retired textures for 90 minutes while a page file Windows manages could still grow. Read only while less is free than the higher of the two lines. Not set: counted where `RX_VK_TEX_COMMIT_RESERVE_MB` is set, else not (as before); `0`: never; `1`: also without the reserve (`vk_texture.cpp` `commit_now`, `page_file_growth`) | | `RX_VK_SWEEP_ALL=1` | Every texture of the cache ordered by its last use for every pass of `vk_texture.cpp` `texture_sweep`, as before. By default a pass under the budget (every 10 presents, and the dead render targets' passes between them) orders only the textures it can take: those the GPU drew and those the game let go; the others it passes over all the same. Over the budget every texture is ordered. The log's "passes for the dead render targets between the looks" line says what a pass costs | | `RX_VK_TEX_RATE_HIGH=0` | The texture eviction's waits (`vk_texture.cpp` `texture_sweep`: 60 presents unused over the budget, 30 after the game let a texture go, 600 or 3600 for one the GPU drew, the compaction's 600 and 1800) count presents above 60 a second, as before. By default they count their seconds at any rate up to 480 a second (`gpu/vk/present_rate.h`, after NX2navi's M19): at the Market's 94 a second "60 presents" were 0.64 s. Below 60 they count seconds unless `RX_VK_TEX_RATE_LOW=0`. The log says the first time a run goes above 68 a second | | `RX_VK_TEX_DETAIL=full`, `reduced` or `low` | The texture detail for one run, over the Insert menu's Texture detail row: whole textures, or the large ones without their largest level, or without two (`gpu/vk/tex_detail.h`). Unset, the row decides, and its Automatic goes by what the driver allows: full from 10,240 MB, low under 6,144 MB, never full on an integrated GPU | +| `RX_VK_TEX_BC1_PARTS=0` | (Triforce, 11-21, the performance audit's TEX-50) An ASTC block of two to four partitions with a colour endpoint mode each is refused unread by the opaque-texture scan, as before, so a texture with one stays BC3. The default reads those modes as the decoder does (their bits below the weights) and takes the block as opaque when every partition's mode has no alpha and is not HDR; the blocks the decoder takes for an error are still refused. The `kept as BC1` log lines count the blocks read this way. And a first start, with no options file, now keeps opaque textures as BC1 as the row's default says: it started at off (`gpu/vk/astc_opaque.h`, `vk_menu.cpp` `tex_bc1_at_start`) | | `RX_VK_TEX_BC1=0/1` | Opaque ASTC textures kept as BC1 instead of BC3 for one run, or not, over the Debug tab's Opaque textures as BC1 row (on by default since options_version 5): a texture the renderer keeps as BC3 whose every block the GPU decoder writes opaque is encoded as BC1, half the video memory, the same texels (`gpu/vk/tex_bc1.h`, `gpu/vk/astc_opaque.h`). One whose memory later holds blocks with alpha, or that a copy on the GPU writes, becomes BC3 for good; the log counts both | +| `RX_VK_ASTC_BC_ALL=1` or `0` | Large decoded ASTC textures are kept as BC3, a quarter of RGBA8: the coarse blocks (6 x 6 texels and up) from 2 MB as RGBA8 (`RX_VK_ASTC_BC_MB`), the finer ones (UI, characters) only from 64 MB. Where the driver allows under 8.5 GB the finer ones go from 2 MB too (`gpu/vk/astc_bc_policy.h`). (Triforce, 11-22, the performance audit's TEX-12, after Source A's) And on every integrated GPU, whatever it is allowed: its allowance is a share of the system's RAM (a 16 GB handheld reported 10,361 MB), so it kept those textures as RGBA8 in RAM the game needs too. The log says `all large ASTC textures kept as BC3`, with `an integrated GPU` there. `1`: on any card; `0`: the finer ones as RGBA8 on any card, which is an integrated GPU allowed 8.5 GB or more as it was before | +| `RX_VK_BC_SMALL_LAYERS=` | (Triforce, 11-21, the performance audit's TEX-09) The layers one run of the ASTC-to-BC3 re-encode takes for a level of 1024 x 1024 or less, 1 to 8, through a small scratch of its own (1024 x 1024 x 8, 32 MB, and 4 MB more of output buffer). Unset: 8, and 1 on AMD's cards until the picture is checked there (one layer a run was for dark lines between Hyrule Field's ground tiles on an RX 6600). A terrain array of 90 layers and 11 levels was about 990 runs of a decode, an encode, a copy and three full barriers; its levels from the second down are now about 210. Each layer of a run is copied by a region of its own. No small scratch at reduced or low texture detail. `1`: none, every run one layer, as before (`gpu/vk/bc_scratch.h`, `vk_texture.cpp` `bc_small_layers`, `bc_encode_into`) | +| `RX_VK_BC_LAYERS=`, `RX_VK_BC_SCRATCH_FULL=1` | The layers one run of the GPU's ASTC-to-BC3 re-encode decodes and encodes, 1 to 8 (1: with more, an array's layers past a run's first came out wrong in the smaller levels, dark lines between Hyrule Field's ground tiles). (Triforce, 11-21, the performance audit's TEX-10) The 2048 x 2048 scratch, its output buffer and the second scratch are made with that many layers, 20 MB a layer (36 with the second), not with 8 whatever a run used: 140 of 160 MB were held and never written. `RX_VK_BC_SCRATCH_FULL=1`: 8 layers, as before (`gpu/vk/bc_scratch.h`, `vk_texture.cpp` `bc_scratch_layers`) | +| `RX_VK_TARGET_POOL_MB=`, `RX_VK_POOL_IDLE=`, `RX_VK_TARGET_POOL_TRIM=0` | (Triforce, 11-21, the performance audit's TEX-29) The render targets' pool gives a dead target's image to the next target of its shape. Its bound is a twelfth of what the driver allows, 256 to 768 MB (`` sets it; 0: no pool); it was a fixed 256, which starved at 2560 x 1440. It is trimmed oldest first, whatever the shape. A shape nothing has taken from the pool goes after 8 presents (`RX_VK_POOL_IDLE`), one that was taken before after 120. Above 90% of the allowance it holds half its bound, above 97% nothing, unless a floor of the targets' blocks is kept. `RX_VK_TARGET_POOL_TRIM=0`: emptied above 90%, as before; with `RX_VK_TARGET_POOL_MB=256 RX_VK_POOL_IDLE=120` the pool as it was. The 600-present line `vk: render target images` has the reuse, the bound and the tier (`gpu/vk/target_pool.h`, `vk_texture.cpp` `pool_trim`) | +| `RX_VK_STAGING_BURST_S=` | (Triforce, 11-22, the performance audit's TEX-28, after Source A's) The uploads stage their bytes in host-visible arenas of 64 MB, and each present gives the idle ones back beyond what is kept: 256 MB, 64 MB where the driver allows 6.5 GB or less (`RX_VK_STAGING_KEEP_MB=`), and any nothing staged into for 30 s (`RX_VK_STAGING_IDLE_S=`, 0: kept). An arena is taken again only once the GPU is done with the submission that read it, so frames that stage in a row use two in turn, and with 64 MB kept the one that had its turn was freed at every second present of a load and made again the present after. An idle arena staged into in the last 2 seconds now stays while the arenas add up to 256 MB (`gpu/vk/staging_keep.h`), unless VRAM is over 90% of the allowance. Nothing changes where 256 MB are kept anyway. The log's `vk: memory:` line counts the arenas made so far and their time. `0`: as before | +| `RX_VK_WROTE_REGIONS=0`, `=2` | (Triforce, 11-21, the performance audit's TEX-02) Every sampled, uploaded texture is asked once an epoch whether the GPU wrote into its storage since its upload, which took a lock and walked the map of written ranges; nearly every answer is no. One bit a 2 MB region of GPU addresses, set where a GPU write ever reached, answers no without the lock when the storage touches no set region; a set bit asks the map as before. `0`: the lock and the walk each time; `2`: a check, both asked and compared, the count of disagreements (which must be 0) logged (`gpu/vk/wrote_regions.h`, `vk_texture.cpp` `texture_gpu_wrote`, `gpu_wrote_into`) | +| `RX_VK_PREDICT_BG_STREAM=0` | (Triforce, 11-21, the performance audit's PIPE-83) The builder of the predicted pipelines behind the game (`predict_background`: the starts after a translator or driver update, resting while the game is in play) keeps its plan as record numbers and reads a pipeline's two notes from `triforce_pipelines.bin` when it builds it, each worker through its own handle; it held every note (~27 KB) and a full note of every predicted pipeline until it ended, about 1 GB. The same programs and pipelines are built. `0`: all of them held, as before (`gpu/vk/predict_stream.h`, `vk_draw.cpp` `predict_background`) | +| `RX_VK_PREDICT_CHECK=0`, `=2` | (Triforce, 11-22, the performance audit's PIPE-68, after Source B's and Source A's; `gpu/vk/predict_check.h`) On unless `0`. The predicted pipelines are built once for the driver's own cache and a record says which programs are done; a driver can lose them all the same (NVIDIA's cache cap, a cleared cache folder, AMD's Windows driver keeping a third). At a start whose record says all are built, the builder's idle thread, after its delay and only while the frame shown is a light one, samples up to 12 of them spread over the plan: each built for the driver's cache alone and timed, then once more as the warm reference; cold is a first build over 4 times the second and 3 ms more, and the sample ends once its majority is decided. Cold: the record is removed, so they are all built again behind the game at light frames, and `.predicted.check` takes a strike for these shaders and this driver; cold again after that, the log says this driver does not keep them and no later start checks. Warm: the strike is forgotten. `2`: checked at every start, a recorded stop too. Also counted: the builds in play of programs the plan predicts that took over 20 ms (`vk: predicted pipelines: N builds in play of predicted programs took over 20 ms`, at most every 10 s while it grows). `0`: neither the check nor the counter, as before (`gpu/vk/vk_draw.cpp` `predict_check_cold`, `predict_background`, `predicted_play_build`) | +| `RX_VK_PREDICT_TYPES=1` or `0` | (Triforce, 11-22, the performance audit's PIPE-44, after Source A's; `gpu/vk/predict_types.h`) Unset: each predicted pipeline is built once for its program, by its key as play keys it (two predicted layouts that differ only in what is dynamic, a stride, gave the same pipeline twice): the same pipelines, fewer builds. `1`: that, and the plan drops a predicted vertex layout whose attributes definitely differ from the program's vertex inputs (an integer input against a float attribute, or the other signedness, read from the vertex module's SPIR-V; scaled formats count as integers): such a pipeline is invalid. Off by default because it changes which pipelines are built, and it applies to a plan made after it is set (a plan is made once for the shaders). `0`: every predicted pipeline built, as before. The predicted lines say how many were left out as built already, how many programs the programs' file did not lend, the notes of variants it has no shaders for and the layouts that did not fit (`gpu/vk/vk_predict.cpp` `predict_pipelines`, `gpu/vk/vk_draw.cpp` `predict_first`) | +| `RX_VK_DRAWN_PAGE_MAP=0` | (Triforce, 11-21, the performance audit's TEX-21) A texture in a virtual pool is checked through its mapping, 64 KB a page, and its page map was recorded at its first check: one the GPU drew first had none, so once the game finalized it the first check took its whole storage as mapped since and uploaded it from memory over what the GPU drew (HUD labels and render targets blank after an area change, up to 15 MB an upload). The map is recorded at the first draw that says who drew it, and the later check gives only the pages mapped anew or written by the CPU. Not for a write that gives no reason (a clear, the movie upscaler). The log counts them. `0`: at the first check, as before (`gpu/vk/drawn_page_map.h`, `vk_texture.cpp` `texture_written`) | +| `RX_VK_DEAD_SUM=0`, `RX_VK_DEAD_POOL_NOW=0`, `RX_VK_DEAD_MIN_UNUSED=3` | (Triforce, 11-21, the performance audit's TEX-30) The frame's transient render targets, drawn over from another start: the other targets' parts over one are added up, each byte once, so one covered at its head by one target and at its tail by another, each under an eighth, is let go too (it was kept for good); it is dead after 1 unused present, not 3; and its image goes to the render targets' pool in the pass that lets it go, once the GPU is done with it, not at the next present. `RX_VK_DEAD_SUM=0`: judged one by one; `RX_VK_DEAD_POOL_NOW=0`: pooled at the next present; `RX_VK_DEAD_MIN_UNUSED=3`: three presents: each as before (`gpu/vk/target_overlap.h` `ShiftedOver`, `vk_texture.cpp` `texture_sweep`) | +| `RX_VK_ASTC_REUSE=2` | (Triforce, 11-21, the performance audit's TEX-14, its first step; off when not set) When the game comes back to an area it makes its textures again, mostly at other addresses, and the same ASTC blocks are decoded and encoded again while the image made of them the first time may still be in VRAM. With `2` a new ASTC texture's blocks are hashed at its first upload (128 bits), the textures the game finalizes that were uploaded from memory and never drawn into are listed, and each new texture with a listed one's blocks and shape is counted: the log's `decoded ASTC used again` line says how many MB of images a copy would have stood in for, and what the hashing cost. Every texture is decoded as before. The copy itself is not built: the walk it was measured on would have reused 0 to 11 MB, and the audit asks for 500 MB on a tour before it is (`gpu/vk/astc_reuse.h`, `vk_texture.cpp` `reuse_upload`) | +| `RX_VK_KEEP_OBSOLETE=1` | (Triforce, 11-21, the performance audit's TEX-43) A texture the GPU drew in one frame is kept after the game finalizes it, for being made again over that storage and sampled undrawn (the terrain's tiles; `RX_VK_KEEP_FINALIZED_MB`). One of 32 KB or more, in no virtual pool, whose storage the CPU has written three quarters of since the release holds nothing that memory holds any more: the sweep lets it go whatever the budget says, where only a draw over it did, and the game's own writes kept each for good (about 200 render target images an hour in players' logs). The log counts them. `1`: kept, as before (`gpu/vk/finalized_obsolete.h`, `vk_texture.cpp` `texture_sweep`) | +| `RX_VK_SLOT_RETIRE=` | (Triforce, 11-21, the performance audit's TEX-76) The game draws each character's shadow map at a fixed address, sized from the camera every frame, and each size is a texture of its own: the ones before it, drawn over, waited 120 presents unused before the sweep took them. A depth target of one level and layer whose start a newer target of the same format and another size was drawn over since its last use goes once unused for `` presents, 16 when not set; nothing is written back, and its image goes to the render targets' pool. The log counts them. `0`: after the sweep's own wait, as before (`gpu/vk/slot_retire.h`, `vk_texture.cpp` `texture_sweep`) | +| `RX_VK_COUNT_BY_TEXELS=1` | (Triforce, 11-22, the performance audit's TEX-86, after Source A's) A texture the GPU samples in the game's own compressed blocks (BC, and ASTC where the log says `vk: ASTC sampled natively`) is counted by its whole blocks (`gpu/vk/tex_count.h`): a level is its width and height rounded up to blocks, times the block's bytes and the layers. It was counted by texels: a byte a texel for BC (twice BC1's and BC4's size) and the 16 bytes of an ASTC block for every texel (16, 64 and 144 times the size at 4 x 4, 8 x 8 and 12 x 12), so the texture budget let textures go for memory nothing held, and the first size estimate left out the top levels of arrays that fit. The count is still lifted to what the allocator gave. Decoded ASTC (NVIDIA's and AMD's desktop drivers) and the renderer's own BC3 and BC1 are counted as before. The log says once which count is on. `1`: by texels, as before | +| `RX_VK_EVICT_STALE_FLUSH=1` | (Triforce, 11-22, the performance audit's TEX-R404, after Source A's) Over the budget, the eviction writes each chosen texture the GPU drew back into the game's memory before it goes. The game draws its HUD labels inside the storage of a 1920 x 1080 target it no longer uses; when that target went, its old texels landed over the labels, which were then uploaded from them as noise. A write-back is now left out when another image the GPU drew overlaps the texture's storage and was drawn in a later frame than the texture's own last drawing (`gpu/vk/evict_reason.h`; one drawn in the same frame keeps it); the texture goes all the same. The log names the first 20 (`vk: texture #... goes without its write-back`) and counts them on the `written back into their memory before eviction` line. Nothing changes while no pass writes anything back (a card under its budget). `1`: every chosen texture is written back, as before | | `RX_VK_COMPACT=1`, `RX_VK_COMPACT_PCT=<50..90>`, `RX_VK_COMPACT_MB=` | VRAM compaction (off by default): the textures that are not render targets in a pool of their own, moved out of sparsely used blocks a few a present so that the blocks go back to the driver, and the texture budget holding the driver's count at that share of what it allows (85); the free space that starts a run (512; 192 while the count is over 90%) | +| `RX_VK_EVICT_EXCESS=0` | (Triforce, on by default; `=0`, or the lab's knob `noexcess`, as before) With the compaction on, what the 90% and 97% rules ask of the hold's pass is their excess (`gpu/vk/vram_compact.h` `rule_target`, `gpu/vk/vk_texture.cpp` `texture_frame`): their budget less 64 MB, where the pass went down to 7/8 of it. Those budgets are the textures less the excess, not sizes: 62 MB over the 90% line on an 8 GB card took about 400 MB of textures, all on screen again within the camera's turn, made again, and over the line again. Over the budget's size (the fixed one, or the live line's) the pass still goes down to 7/8 of that size; with both, the lower | +| `RX_VK_COMPACT=1`, `RX_VK_COMPACT_PCT=<50..90>`, `RX_VK_COMPACT_MB=` | VRAM compaction (off by default): the textures that are not render targets in a pool of their own, moved out of sparsely used blocks a few a present so that the blocks go back to the driver, and the texture budget holding the driver's count at that share of what it allows (85); the free space that starts a run (192 while the count is over 90%). `RX_VK_COMPACT_MB` unset: (Triforce, 11-22, the performance audit's TEX-38, after Source A's) a sixteenth of what the driver allows, 64 to 512 MB (`vram_compact.h` `default_min_free`): 512 on a card that allows 8 GB or more, as before; 327 on a 6 GB laptop that allows 5,233 MB; 512 before the first look. `RX_VK_COMPACT_MB=512`: 512 on every card, as before. `tests/test_compact_start.py` | +| `RX_VK_MEMORY_PRIORITY=1` or `,` | (Triforce, 11-22, the performance audit's TEX-47, after Source A's; off when not set) When a card is short of video memory, Windows moves some of the game's device memory to system memory, read over the bus from then on, and picks what goes. With the switch and a driver that has `VK_EXT_memory_priority`, the game ranks its memory: the render targets' blocks 1.0 (kept first), the sampled textures' 0.25 (moved out first), everything else the driver's own 0.5 (`gpu/vk/memory_priority.h`); `,` sets the first two, each from 0 to 1. The textures then come from their own pool also where it is not compacted (`RX_VK_COMPACT`), and a sparse texture's chunks name the textures' rank. The log says `vk: memory priorities on` with the three numbers, or that the driver has no such extension. Source A measured it on an RTX 4090 with the video memory taken by other programs: with 1.3 GB moved out, a median GPU frame of 24 to 29 ms against 109 to 179; with room on the card, no difference. Not run on a small card, and a driver may ignore it: off by default, and without the switch the extension is not asked for. `0`, `off`, `no` or a value that cannot be read: off | | `RX_VK_SKIP_COMPRESSED_OVER_RT=1` | A compressed (ASTC, BC, ETC2/EAC) texture made over storage render targets used is not uploaded, as before: bosses and new enemies all blue the first time | | `RX_VK_NO_WRITEBACK` (set) | GPU writes to mirrored pools are not copied back to guest memory (an experiment, for A/B) | +| `RX_VK_MSYNC_PASS=0`, `=1` | (Triforce, 11-21, the performance audit's MEM-48) A mirror sync's pages that come from a capture's buffer go to the pool's VRAM mirror by the write-back diff's compute pass, one dispatch a sync, where they were copy regions of a page or a few each (700 to 1,300 a frame, each read over PCIe after the one before). A run not in whole words, a GPU without subgroup votes in compute and a sync whose chunk table finds no room keep the copies; a mirror written twice in one upload command buffer, once by the pass, gets a full barrier between. On for NVIDIA when not set; `1`: on every GPU; `0`: copy regions, as before. The log's `mirror syncs' pass (600 submits)` line counts dispatches, chunks, fallbacks and barriers; `RX_VK_LAB_AB=nomsyncpass` is the in-run comparison (`gpu/vk/msync_pass.h`, `vk_memory.cpp` `msync_pass_add`, `msync_pass_flush`) | +| `RX_VK_WB_FORGET_SORT=1` | (Triforce, 11-21, the performance audit's MEM-61) At the end of each part with write-backs the write-back diff clears, for every page written since, the bit that says its pool's shadow knows the page. The three lists of such pages (the game's writes, the collection's, the replay's) were sorted first, thousands of pages a part, only so that the pool found for one page serves the next; the bits and the counts do not depend on the order, and the lists hold ascending runs as they come. They are no longer sorted. The log's `forgetting the written pages took` line gives the time. `1`: sorted, as before (`vk_memory.cpp` `shadows_forget_written`) | | `RX_VK_WB_DIFF=0` | Write-backs copy every byte again instead of only what changed against a VRAM shadow (the diff is on by default; v2.0.9 and v2.0.10 had it off, issue 1161) | | `RX_VK_LAZY_WB=1`, `=2` or `=0` | The lazy write-back for one run, over the Debug tab's Lazy write-back row (`lazy_write-back=1`; on by default since options_version 5; `docs/lazy-writeback.md`, `gpu/vk/wb_lazy_state.h`): a whole page of what shaders write into a mirrored pool that the CPU leaves alone is no longer copied back at every submission; its page has no access, and whoever touches it (the game's code or the host's, by a fault; a host call that hands it to the kernel; the GPU through the import) has it copied down first. Windows only. On AMD's driver the row alone leaves it off, as NX2navi did after games ended in play there; `=1` turns it on anyway. `=2` is the self-check: every copy is still made, the pages are protected and paid as if not, and each payment compares what it would have copied with guest memory: a `MISMATCH at guest` line is a page it would have got wrong. The log says `vk: lazy write-back on` (or why off) at the start, and every 600 submits `vk: lazy write-back (600 submits)`: the MB not copied, the pages owed, the pages paid by a fault, a host call, a GPU read, a mirror given up and a recording's submit, the payments and their time, and what failed | | `RX_VK_LAZY_PAYERS=<1..4>` | (Triforce, 4 by default; `=1` as before) How many of the lazy write-back's payments copy at once (`gpu/vk/wb_lazy_payers.h`, NX2navi's `wb_lazy_unlocked`). A payment's copy goes onto the queue after everything submitted so far and its thread waits for it (0.5 to 3 ms each in the owner's sessions, up to 21 ms at a load); with one at a time, a second thread that touched another owed page, or the replay giving a mirror up, waited for the first one's copy to come back before its own went onto the queue. Each payment holds a slot of its own (a command buffer and a fence); the pages of two payments are never the same ones (`gpu/vk/wb_lazy_state.h`), and each copy still goes onto the one queue after everything before it. The 600-submit line `vk: lazy write-back (600 submits)` ends with the payments that waited for a free slot and the most slots in use at once | | `RX_VK_GEN_SEED=` | The page write generations (`gpu/vk/write_gen.h`) start at `n` and not at 1 (decimal, or hex with `0x`; 2^62 at most). They are 64 bits; the 32-bit ones of earlier builds ran out after days of play, and textures and mirrors then kept old contents. `RX_VK_GEN_SEED=0xFFFF0000` takes a play test across 2^32 within seconds: the log says `the write generations start at` and, at the crossing, `this run has made 4294967296 write generations`, and nothing in the picture should change there | +| `RX_VK_UPSCALE_OFF=1` | (Triforce) The upscaler Off for one run: the game's gameplay upscale networks are not dispatched, their input is stretched linearly into their output (`gpu/vk/upscale_off.h`). Softer than the networks below 100%; only while the game's own upscaler draws (DLSS and FSR take the dispatch first). Not set: as before, the networks run, and on cards that run them on software multiplies the first one is stood in for. The lab's `upoff` phases switch it within a run | +| `RX_VK_STUB_SAME_SIZE=0` | (Triforce, 11-22, the performance audit's GPUWORK-21, after Source B's and Source A's; `gpu/vk/stub_same_size.h`) On unless `0`. A gameplay upscale network that cannot run (left out or not translated: `RX_VK_NO_F16`, a card or driver that cannot build it) is stood in for by its input drawn into its output (`gpu/vk/vk_draw.cpp` `upscale_stub`) also where the two are one size: `9dc98871eb7fc101`, the network of Render resolution 100%, was declined as "no larger than its input" and skipped whole, a black 3D under the HUD. Both gameplay networks now take the typed upscale first (it reads the scene's shared-exponent picture and undoes the network's square-root encoding; it was the 3/4 network's alone), then the blit, which warns of wrong colours for both, then nothing. The picture is the upscaler Off's: softer, none of the network's anti-aliasing. Only where the network cannot run. `0`: the decline, and the 3/4 network alone in the typed upscale, as before. `RX_VK_NO_UPSCALE_STUB=1` still turns the whole stub off; `RX_VK_FORCE_UPSCALE_STUB=2` sends every network through it, for a look | | `RX_VK_DLSS=0/1/2`, `RX_VK_DLSS_PRESET=`, `RX_VK_DLSS_LOG=1`, `RX_VK_REFLEX`, `RX_VK_REFLEX_LOG=1`, `RX_VK_ANTILAG=0/1` | NVIDIA DLSS and Reflex, and AMD Anti-Lag 2, for one run (`=-` on Reflex or Anti-Lag 2: its extensions not even enabled): the readme's NVIDIA DLSS section. `RX_VK_DLSS_TRACE=` logs one frame's large draws every that many frames, `RX_VK_DLSS_DEBUG=motion` shows the motion vectors, `=alternate` the game's own upscale every other frame | | `RX_VK_FSR4=0/1/2`, `RX_VK_FSR_DIR=` | AMD FSR 4 for one run (0 never, 1 FSR 4, 2 its Native AA), and another folder for AMD's DLLs (not from `hookshot_env.txt`): the readme's AMD FSR 4 section. `RX_VK_FSR4=1/2` and `RX_VK_DLSS=1/2` go over the Upscaler row, a saved FSR 3.1 too where the engine they ask for runs; `RX_VK_FSR=1` goes over them (one upscaler at a time; `=2`: FSR 3.1 as its Native AA, the 3D at full size). The DLSS trace and debug switches above apply to FSR 4 too, on the frame chain they share | | `RX_VK_FSR4_DRS=1`, `RX_VK_FSR4_AUTO_EXPOSURE=0` | (Triforce) AMD FSR 4's context as before, for comparing (`gpu/vk/vk_fsr4.cpp` `make_context`, `gpu/vk/temporal_rules.h` `plan_context`): `RX_VK_FSR4_DRS=1` one context with FSR's dynamic-resolution flag for every render size from the start, where a fixed Render resolution otherwise gets a context of its own size without it (and the game's dynamic resolution one with it once the size first changes); `RX_VK_FSR4_AUTO_EXPOSURE=0` its HDR contexts (Upscale before bloom, or an RGBA16F picture with RenoDX's HDR Native) without FSR's auto exposure. A creation or evaluation of DLSS or FSR 4 that fails leaves the game's upscale for 2 s and is tried again; the fourth failure in a row turns that engine off (the Upscaler row then shows Game and says so) until the row is set again | @@ -273,19 +376,61 @@ case-sensitive, and an unknown one is ignored. | `TRIFORCE_MENU_AHEAD=0` | (Triforce) What the options' rows ask of NVIDIA's and AMD's libraries (whether DLSS, FSR 4 and FSR 4's frame generation run here: NGX's start, AMD's loader) asked at the options' open on the presenting thread, as before, and not once on a thread of its own as soon as the game runs (`gpu/vk/vk_menu.cpp` `menu_ahead`; its log line says how long each took). For comparing: the first open waited about 2 s for them in test11 | | `RX_VK_AA=<0..3>`, `RX_VK_AA_KEEP=1` | The anti-aliasing for one run, over the Insert menu's Anti-aliasing row: 0 the game's own SMAA, 1 none, 2 FXAA, 3 SMAA at its Ultra preset in its place (`gpu/vk/aa_rows.h`, `gpu/vk/vk_aa.cpp`). Unset, the row decides; with DLSS, DLAA or FSR drawing from the tone mapped picture (Upscale before bloom off) the game's SMAA is left out (none: they want jittered frames no other anti-aliasing has been over), and before the bloom it runs as the game's; the switch goes over both, for an A/B (`RX_VK_AA=0`: the game's SMAA under the upscaler). The game's SMAA is three compute programs known by their keys (game build `e0f3ccfa`): with another choice in effect its composition is replaced and its edge and weight passes are left out, and `RX_VK_AA_KEEP=1` keeps those two running. The log says `vk: aa:` when a composition is first replaced, why one was left to the game, and when none was found for a minute | | `RX_VK_ASYNC=1` | The replay on a thread of its own instead of the game's submitting thread (off by default). The game still presents only once the GPU has run its frame, so nothing is released early. (Triforce, `gpu/vk/async_rules.h`; not yet run in the game) With it on, each with a switch of its own to put it back: a signal of the CPU's takes its capture on the game's thread only when a submission handed over before it may still wait for that event, which the host tells at the submit (`RX_VK_ASYNC_EVENT_NOTES=0`: at every signal while the thread has work, as before; the `vk: the replay thread: ... signals of the CPU's` line counts both); the submit's capture shares its copies with the copy helpers and has them all made before the hand-over (`RX_VK_ASYNC_CAPTURE_BEHIND=0`: every copy on the game's thread); the game's submit keeps its virtual core for the hand-over (`HOS_ASYNC_KEEP_CORE=0`: given up and taken again); the game's present poll waits for the replay thread asleep and looks at its event between yields only once the GPU has the frame (`HOS_PRESENT_POLL=0`: the game's own sleeps); a present runs the jobs behind it until the frame's stopped work has gone on and then makes go only what was already stopped at the present before (`RX_VK_PRESENT_DRAIN=1`: every present forces every stopped queue past its wait, as before; `=0`: none does; `RX_VK_PRESENT_WAIT_MS`, 8: how long it runs the jobs behind); and the thread looks for its next job for 100 µs before it sleeps (`RX_VK_ASYNC_SPIN_US=<0..1000>`). Still open, and why it stays off: after a ten-area tour the player's character is drawn black for one frame, 0 to 3 times a minute at the Market (`gpu/vk/async_rules.h` has what the lab ruled out). Two diagnostics for it: `RX_VK_LOG_READBACK_ZERO=1` logs each readback of GPU results that is all 0 where it was not at its readback before, and `RX_VK_REGION_TRAP=1` (with `RX_VK_PASS_TRACE=1` and the frame log) logs the passes of the last frames and writes their render conditions to `region_cond_.txt` when the picture's bottom middle goes dark for a frame, eight times a run. `RX_VK_ASYNC_FORCED=1`: the thread as it was before 11-14, every catch-up forcing | -| `RX_VK_OCC_QUERY=0`, `RX_VK_OCC_QUERY_SKIP=[,...]`, `RX_VK_OCC_COND=0`, `RX_VK_OCC_LOG_COND=1` | (on by default since 11-17: +2.7 to +4.6% fps in every scene measured, 10-12% fewer draws, no flicker added; `RX_VK_OCC_QUERY=0` turns it off) The game's samples-passed reports (which objects show; a count of 0 is not drawn) get their real counts: the replay still writes each report as it passes it with 4096 (`RX_VK_OCC_FIXED`, everything visible), and the draws from its reset are counted by Vulkan occlusion queries (precise where the card has them); once the GPU is done with the recording that holds them the count is written into the report's record over the 4096, as NX2navi writes it at completion. The done recordings are found with one look at the recordings' timeline (nothing waits) as a game's fence wait ends (the game reads its counts after it), at each recording's begin and each present, at the latest when the recording's slot comes round. The game's report records rotate (a ring: an address is another object's at its next report), so a count is written only while its report is still the newest at its address, and only when it is whole (every query available, no draw left out, reset and report in one recording, read back within 8 presents); otherwise the record keeps its 4096. The game also predicates draws and clears on these words (render conditions), decided as NX2navi decides them: a word whose newest report is a samples-passed one whose count is still to come is unknown and the condition renders; a word whose count was written (or for which none comes) is read from memory as before, where a written 0 skips the draws and clears it gates. A count goes in only over the value the replay left in the word (a compare-and-write: never over the game's CPU re-arming the record), and a record rewritten in the stream after its report (a small buffer clear, a report of another counter type) gets no count. A parallel replay's segments signal the recordings' timeline and are sealed as they are submitted, so their counts are written too. The count word is written as the replay writes a report (`guest_cpu_write`, and the word kept out of the mirrors' write-backs). With hostQueryReset (Vulkan 1.2) a recording's queries are reset on the CPU once read, only those it used. `RX_VK_OCC_QUERY_SKIP`: report addresses that keep the fixed answer (the log's `occlusion query sites` lines list the first 64 addresses with their counts). The log's `occlusion queries` line every 600 presents: the reports, how many were counted whole and why not, the counts written at completion (and of them 0), those not written and why (lost, superseded by a later report at the address, late, a render condition's), the mean latency in presents and how many were written after the next present, where the done recordings were found, and the render conditions on samples-passed words (rendered because a count was pending, rendered or skipped from a written count, from memory with no count to come, as before in the knob's off phases) with the draws and clears they skipped. `RX_VK_OCC_COND=0`: a condition on a samples-passed word always renders (the counts are still written), to tell the counts from the skips. `RX_VK_OCC_LOG_COND=1`: one such condition in 499 logged (300 lines), with both words, their sources (pending, written, memory) and the present. Lab: the knob `occq` in `RX_VK_LAB_AB` runs the queries the whole run and writes the counts of the reports made in its phases (`gpu/vk/occ_answer.h`, `vk_replay.cpp` "occlusion queries") | +| `RX_VK_OCC_QUERY=0`, `RX_VK_OCC_QUERY_SKIP=[,...]`, `RX_VK_OCC_COND=0`, `RX_VK_OCC_LOG_COND=1`, `RX_VK_OCC_GATE=0` | (on by default since 11-17: +2.7 to +4.6% fps in every scene measured, 10-12% fewer draws, no flicker added; `RX_VK_OCC_QUERY=0` turns it off) The game's samples-passed reports (which objects show; a count of 0 is not drawn) get their real counts: the replay still writes each report as it passes it with 4096 (`RX_VK_OCC_FIXED`, everything visible), and the draws from its reset are counted by Vulkan occlusion queries (precise where the card has them); once the GPU is done with the recording that holds them the count is written into the report's record over the 4096, as NX2navi writes it at completion. The done recordings are found with one look at the recordings' timeline (nothing waits) as a game's fence wait ends (the game reads its counts after it), at each recording's begin and each present, at the latest when the recording's slot comes round. The game's report records rotate (a ring: an address is another object's at its next report), so a count is written only while its report is still the newest at its address, and only when it is whole (every query available, no draw left out, reset and report in one recording, read back within 8 presents); otherwise the record keeps its 4096. The game also predicates draws and clears on these words (render conditions), decided as NX2navi decides them: a word whose newest report is a samples-passed one whose count is still to come is unknown and the condition renders; a word whose count was written (or for which none comes) is read from memory as before, where a written 0 skips the draws and clears it gates. A count goes in only over the value the replay left in the word (a compare-and-write: never over the game's CPU re-arming the record), and a record rewritten in the stream after its report (a small buffer clear, a report of another counter type) gets no count. A parallel replay's segments signal the recordings' timeline and are sealed as they are submitted, so their counts are written too. The count word is written as the replay writes a report (`guest_cpu_write`, and the word kept out of the mirrors' write-backs). With hostQueryReset (Vulkan 1.2) a recording's queries are reset on the CPU once read, only those it used. `RX_VK_OCC_QUERY_SKIP`: report addresses that keep the fixed answer (the log's `occlusion query sites` lines list the first 64 addresses with their counts). The log's `occlusion queries` line every 600 presents: the reports, how many were counted whole and why not, the counts written at completion (and of them 0), those not written and why (lost, superseded by a later report at the address, late, a render condition's), the mean latency in presents and how many were written after the next present, where the done recordings were found, and the render conditions on samples-passed words (rendered because a count was pending, rendered or skipped from a written count, from memory with no count to come, as before in the knob's off phases) with the draws and clears they skipped. `RX_VK_OCC_COND=0`: a condition on a samples-passed word always renders (the counts are still written), to tell the counts from the skips. `RX_VK_OCC_LOG_COND=1`: one such condition in 499 logged (300 lines), with both words, their sources (pending, written, memory) and the present. (11-22) `RX_VK_OCC_GATE=0` turns off (on by default since its lab A/B: no fps lost at the Market): a count over a draw that a written count's condition left out (between the reset and its report) is the skip's 0, not the object's, and a condition on it renders; the count is still written for the game's CPU. Why: the game's CPU reads a count after its frame wait and draws a hidden object's proxy instead of the object, but most counts land after it has read the 4096, so it goes on drawing the object under the condition, the condition reads the skip's 0 and the object stays hidden (tickets 11-21: items and trees in the Forest Temple, black interiors; `RX_VK_OCC_QUERY=0` showed them). The line's second half counts these counts (counted whole, written, of them 0), the conditions that read one and those the switch rendered, on or off. Cost with it on: a hidden object under such a condition is drawn every other frame instead of never. Lab: the knob `occq` in `RX_VK_LAB_AB` runs the queries the whole run and writes the counts of the reports made in its phases (`gpu/vk/occ_answer.h`, `vk_replay.cpp` "occlusion queries") | +| `RX_VK_OCC_DEPTH_ONLY=1` | (Triforce, 11-22, the performance audit's GPUWORK-01, after Source A's; off unless set) Only the occlusion windows whose draws write no colour are counted. The game's occlusion tests draw depth only, but a query here opens at a window's first draw in the pass and stays open to the report or the pass's end, so it can enclose draws that write colour, and a query's end waits for the work it encloses. With the switch, a window's first draw that writes a colour target gives its count up: the open query ends before that draw, no query begins again until the next reset, and the report is incomplete, so its record keeps the 4096 placeholder and what it gates is drawn. Fewer objects are culled with it (Source A: 4% fewer for 0.25 ms of GPU time), never one missing; whether that pays here is for a measurement to say, which is why it is off. The lab's knob `occdepth` (`RX_VK_LAB_AB`) is the switch in its phases. Counted either way, in a third `vk: occlusion queries` line every 600 presents: the counting windows that met a colour draw, the colour draws recorded under an open query, the counts given up and the reports kept 4096 for it (`gpu/vk/occ_answer.h` `Counter::colour_draw`; `gpu/vk/vk_replay.cpp` `occ_draw_colour`; `gpu/vk/vk_draw.cpp` `draw`) | +| `RX_VK_OCC_LEAN=3` | (Triforce, 11-22, the performance audit's DESC-82, after Source B's: its first step, the census; off unless set) The game's occlusion proxies write no colour, yet each runs its whole fragment shader and fills its fragment descriptors. With `3`, each draw an open occlusion query counts that writes no colour is classed (`gpu/vk/occ_lean.h`): lean, or the first refusal of a pipeline without the fragment stage (no fragment stage, depth out, sample mask, demote, stores, image writes, atomics, global memory, alpha to coverage, alpha test), and its fragment descriptors added to its class; every 600 presents `vk: occlusion proxies (RX_VK_OCC_LEAN=3` gives each class's draws a frame and descriptors a draw. Nothing else changes: no pipeline, key or descriptor. The entry's lean pipelines (`1`) and their check (`2`) are built only if the census finds more than one fragment descriptor a proxy to save; they are not built, and read as off. `tests/test_occ_lean.py` | | `RX_VK_GATE_WAIT_MS=<1..100>` | (Triforce, off unless set, after NX2navi's `gate_wait_ms`, whose default is 10; not yet run in the game) A submission whose first command buffer begins (after state only) with a wait for an event the game signals from the CPU, unmet in the replay's order, waits at its submit for that signal, at most this long, before its capture and its replay (`gpu/vk/gate_wait.h`, `gpu/vk/vk_replay.cpp` `gate_wait_signal`): the capture then holds what the game wrote up to its signal, and the submission is replayed in the submit call on the submitting thread, instead of stopping at the wait and going on from the order thread (`sync_watch`) with a second capture once the signal came. The wait holds no lock and looks at the event's word in the game's memory. One that runs out goes as before; after 4 that ran out, once they are more than a tenth, it is off for the run (logged). A `vk: RX_VK_GATE_WAIT_MS` line every 600 waits gives their mean and how many ran out; the order line's `captures at a CPU event's wait` and `made to go by presents` say how often the old way was taken | +| `RX_VK_WAIT_HOLD_PAUSE_US=` | (Triforce, 11-22, the performance audit's THREAD-11, after Source B's; off unless set, not measured here) A wait of the game's for a fence whose work is stopped at an event's wait holds up to `RX_VK_WAIT_HOLD_US` (3 ms) for another queue to let it on before it forces it, looking again and again: a flush, the timeline's value, the order lock to ask whether the stopped work is still needed, and a yield of the processor. With `` (2 is Source B's; 1000 at most) a gap of PAUSEs that long stands between two looks instead of the yield, and the order lock is taken only when one of its sections has left since the waiter last asked, which is the only time its answer can change; the fence's own value is still read at every look, and the bound and the forced drain stay. Source B measured 59.2 to 59.9 fps and a 1% low of 40.0 to 42.1 with such a gap on a loop that runs far more often than this one (about 15 holds a second here). `0` or not set: as before. Either way the `vk: queues ordered by their events` report is followed by a `vk: fence waits held for stopped work` line: the holds, their looks, the time held, the takes of the order lock and how the holds ended (`gpu/vk/wait_hold.h`, `gpu/vk/vk_replay.cpp` `wait`) | +| `RX_VK_WAIT_HOLD_PAUSE_US=` | (Triforce, 11-22, the performance audit's THREAD-11, after Source B's; off unless set, not measured here) A wait of the game's for a fence whose work is stopped at an event's wait holds up to `RX_VK_WAIT_HOLD_US` (3 ms) for another queue to let it on before it forces it, looking again and again: a flush, the timeline's value, the order lock to ask whether the stopped work is still needed, and a yield of the processor. With `` (2 is Source B's; 1000 at most) a gap of PAUSEs that long stands between two looks instead of the yield, and the order lock is taken only when one of its sections has left since the waiter last asked, which is the only time its answer can change, or while a signal of the CPU's (`nvnEventSignal` without the replay thread) is queued and no section has acted on it yet; the fence's own value is still read at every look, and the bound and the forced drain stay. Source B measured 59.2 to 59.9 fps and a 1% low of 40.0 to 42.1 with such a gap on a loop that runs far more often than this one (about 15 holds a second here). `0` or not set: as before. Either way the `vk: queues ordered by their events` report is followed by a `vk: fence waits held for stopped work` line: the holds, their looks, the time held, the takes of the order lock and how the holds ended (`gpu/vk/wait_hold.h`, `gpu/vk/vk_replay.cpp` `wait`) | +| `RX_VK_INDEX_REBASE=0` | (Triforce, 11-21, the performance audit's DESC-12) A direct indexed draw binds its index buffer at offset 0 and gives its offset as the draw's first index, so draws at different offsets of one buffer bind it once: bound at each draw's own offset, nearly every indexed draw called `vkCmdBindIndexBuffer`. Only where the offset is a whole number of indices; indirect indexed draws are as before. `RX_VK_PROF_FINE=1` counts the index binds looked up and those made. `0`: every draw binds at its own offset (`gpu/vk/index_rebase.h`, `vk_draw.cpp` `draw`, `bind_index`) | +| `RX_VK_RECORDER_OWN=0` | (Triforce, 11-21, the performance audit's REPLAY-04) With the recorder thread, a recording slot's descriptor pools are reset, its extra pools destroyed and its grown streams given back in `draw_begin_recording`, after the recording's first packets, as before: each is a boundary the replay waits at until the recorder has run what came before. The default makes that reset in `begin_recording` right after the slot's command pools are reset, where the recorder has nothing held. The 120-present line `a frame's recorder thread` counts the replay's waits (`gpu/vk/vk_draw.cpp` `draw_slot_reset`, `vk_replay.cpp` `begin_recording`) | +| `RX_VK_DESC_POOL_GROW=0`, `RX_VK_DESC_POOL_DIV=<1..32>`, `RX_VK_STREAM_MB=` | (Triforce, 11-22, the performance audit's DESC-33, after Source A's; `gpu/vk/desc_pool_grow.h`) Each recording slot (48 at most) keeps a descriptor pool for the programs over the push limit, and the driver may keep host memory for all of it 48 times. A slot's pool is made at half size (4096 sets, 16384 uniform buffers and sampled images, and so on) and made whole (8192 sets) at the slot's first reset after a recording ran out of it; a recording that runs out gets a whole extra pool on the spot, as it always did. `RX_VK_DESC_POOL_GROW=0`: whole pools from the start, as before. `RX_VK_DESC_POOL_DIV=` divides both sizes by n. `RX_VK_STREAM_MB=` is the size of each slot's first upload stream (2 by default; a recording that needs more is moved to a bigger one, kept for the slot's next 120 recordings; 24 with `RX_VK_STREAM_GROW_OFF=1`, which skips draws instead). Every 3000 recordings the log's two "the recording streams" lines say the most one recording held of its stream, how many slots are still at half, how many were made whole, the extra pools made, and the most sets and descriptors of each type one recording took: what a smaller default is judged by (the stream's stays 2 MB until those numbers are read in play) (`gpu/vk/vk_draw.cpp` `make_pool`, `slot_reset_locked`, `draw_begin_recording`) | +| `RX_VK_CAP_SPARE_FIT=0`, `RX_VK_CAP_STEP_MB=` | (Triforce, 11-22, the performance audit's MEM-33; the first on by default, the second off unless set; not yet run in the game) `RX_VK_CAP_SPARE_FIT=0`: a capture with no spare buffer makes its own with the spares' lock held, takes any spare that fits, and a spare made ahead is made once, as before. Without it (`gpu/vk/cap_spare.h`, `gpu/vk/vk_memory.cpp` `cap_buf_get`) the buffer is made with the lock let go, so the replay's take does not wait behind a submitting thread's allocation to give a buffer back; where spares are not capped at 32 MB a capture leaves alone a spare of 32 MB or more that is over twice its size class, for the load that wants it; the spare a capture of a class of 32 MB or more takes is made again on a worker; a spare is made ahead only if it fits what spares may hold together with the ones held and being made, and a buffer given back counts the ones being made; when the allocation fails, any spare that fits, then the exact size. `RX_VK_CAP_STEP_MB=`: where spares are capped at 32 MB (a card of 6.5 GB or less), a capture over 16 and up to 32 MB gets a buffer in steps of n MB, not the whole 32 MB (Source A: 4; its worst frames were worse in two pairs, so off here). The 600-take line counts the buffers made on a submitting thread and their ms | +| `RX_VK_MIRROR_PIECE_IDLE=` | (Triforce, 11-22, the performance audit's MEM-26, its first step only; off unless set; not yet run in the game) A mirror in pieces keeps a piece's memory until the whole mirror goes, also where no game pool is left in its import. Giving such pieces back is NOT ported (the unbind and the free must wait for the GPU and for the bind timeline). What it would find is counted (`gpu/vk/piece_idle.h`): every 1200 submissions, after the `mirrors in pieces` line, the MB of pieces no game pool covers; with this switch each binding given a mirror in pieces also notes the submission in its pieces, and the line says how many of those MB no binding reached for that many submissions (600 is the wait the release would use). Under about 100 MB at the end of a tour on a 6 GB card (`RX_VK_TEST_CARD_MB=6144`) the release is not worth porting | +| `RX_VK_RANGE_WALK=0` | (Triforce, 11-22, the performance audit's MEM-42; on by default; not yet run in the game) The capture and collection loops (`gpu/vk/vk_memory.cpp` `memory_collect_written`, `capture_make`'s staging and behind loops, `take_capture`'s loops over the texture storage's ranges) ask of every page whether it lies in a sorted range list with a binary search, as before. Without it each loop has a cursor that only moves forward (`gpu/vk/range_walk.h`), one pass over the ranges for the loop; an address below the last one asked searches again, so any order gives the search's answers (`tests/test_range_walk.py` compares them). The `RX_VK_LAB_AB` knob `norangewalk` searches in its phases | +| `RX_VK_TABLE_STREAM=0` | (Triforce, 11-22, the performance audit's SYNC-04; on by default; not yet run in the game) The guest page table's changes of a recording (the table the shaders' pointers go through, `gpu/vk/vk_memory.cpp` `memory_table_flush`) are flushed as before: a (page, value) pair for every page changed, all of them sorted, the last of each page kept: 16 bytes a page and a sort on the recording thread, a few ms when a pool of hundreds of MB is imported ahead or merged. Without it the changes are streamed (`gpu/vk/table_stream.h`): runs in page order and apart are appended one after the other, any other set is swept by page with the newest change owning each page, through one reused batch of 8,192 entries. The transfers are the same ones in the same order (`tests/test_page_table_stream.py` compares the two, entry for entry). A flush over 1 ms is logged with its pages, changes and transfers (the first 16) | | `RX_VK_CAPTURE_BEHIND=0` | (Triforce, on by default; `=0` as before) The submit-time capture (`gpu/vk/vk_memory.cpp` `capture_make`) takes the write marks on the game's submitting thread and copies there only the pages read at once (the late pages and the texture storage); the rest of its copies, in parts of 32 pages, go to the capture's copy helpers while that thread replays the submission (`gpu/vk/capture_behind.h`). A read of a page as submitted copies its part itself when no helper has taken it yet, or waits for the helper; every part is done and flushed before a recording is submitted and before the thread goes back to the game. Only with the replay on the submitting thread (not `RX_VK_ASYNC=1`), with copy helpers (`RX_VK_CAPTURE_THREADS`, not `RX_VK_LOW_SPEC`), at least 128 pages behind and at most 512 ahead (a scene's load: copied whole as before). At the Market the old copy was ~0.25 ms of each submit's ~0.35 ms capture, six submits a frame. The 120-present line `a frame's captures at the game's submits` gives their time on the game's threads (the scan, the copies made there, and the copies behind made or waited for by the replay), the MB copied and behind, and who copied the parts; the stall line names the captures when they took 1 ms or more | +| `RX_VK_CAP_FREE_ASYNC=0` | (Triforce, on by default; `=0` as before) A retired capture buffer the spares cannot keep (past 12 buffers or 768 MB, 32 MB on a card of 6.5 GB or less) is destroyed on a thread of its own (`gpu/vk/vk_memory.cpp` `cap_buf_free`, `destroy_buffer_later`), not in the capture's take on the replay's thread with its locks held: freeing a 64-256 MB host-cached buffer makes the driver unmap and release every page, the `retires` part of a `vk: slow capture take` line. Only buffers the GPU is done with get there; the memory comes back a moment later. The 600-take line counts the buffers and MB handed over. With no thread to be had they are destroyed where they were retired | | `RX_VK_PARALLEL=<0..8>`, `RX_VK_PARALLEL_AUTO=<2..8>` | The parallel replay: a frame's large command buffers recorded on that many threads, always, or wherever short trials as the game runs find it faster than the replay thread alone; `RX_VK_PARALLEL=0`: never. The Debug tab's Replay helper threads row is the second without an environment to set (`replay_helper_threads=` 1 to 7 for 2 to 8 threads); either switch goes over the row. The row's Automatic (8), which it is unless the options file says otherwise, is the count `gpu/vk/replay_threads.h` gives the PC: none with four cores or fewer or on AMD's driver for RDNA 1 and 2, 2 threads with five to seven cores, 3 with eight or more. The log says at the start what was chosen (`replay helper threads:`) and at each trial which way is in use (`parallel replay, trial`). Between trials the tuner's rule (`gpu/vk/replay_threads.h` `rule`) judges every 120 frames on the replay thread alone: where the replay is plainly the frame's limit (70% of the frame or more) and the helper threads would take at least 1 ms and a tenth of the frame off it, never counting below the GPU's time (the game waits for each frame's GPU work), they are turned on without a trial; where they cannot pay (no command buffer large enough to cut, the GPU's time about as long as the replay's, or the replay under 60% of the frame: the game's own threads are its limit) no trial is made while that holds; trials decide the rest, and a trial that takes the helpers back off holds the rule's "on" for 3,600 presents. The log says so (`parallel replay, the rule`) when its verdict changes. `RX_VK_PARALLEL_RULE=0`: trials only, as before. With DLSS or FSR on, for upscaling or frame generation, the dry pass tells their hooks in the draws in stream order, and each command buffer from the first draw or dispatch that takes what they noted (the upscale, or the engine's point before the bloom) on is recorded in order after the segments before it (`gpu/vk/stream_order.h`); `RX_VK_PARALLEL_TEMPORAL=0`: the replay thread records alone then, as before. The frame statistics' `a frame's parallel replay:` line says in how many frames the threads recorded, the command buffers in segments and alone (and why), and the time in them; the next one (`its parts`) where that time went: the setup before the threads begin, how soon each command buffer's first segment reached the GPU, the helper threads' wake-up, each command buffer's longest segment, the replay thread's wait, the submits, the recording lock's waits, and the segments' GPU time (the `GPU time` line counts the segments too; it left them out). After each trial's line a `trial N's parts` line gives its three parts' frame, replay thread's time, GPU time, the rest of the frame, full barriers and draws, and the helper threads' parts. That header lists what an audit of the parallel replay found and fixed, and what is left. `RX_VK_NO_DRY_COPIES=1`: the dry pass does not mark what copies and texture clears write, as before. The replay thread records the next segment to submit itself when no helper thread has begun it (it only waited before, on the core kept for it); `RX_VK_PARALLEL_OWN=0`: it waits for the helper threads, as before. `RX_VK_PARALLEL_SEG=<32..>`: the draws and dispatches a segment is cut for (160); `RX_VK_PARALLEL_SEGS=<2..12>`: the most segments a command buffer is cut into (6), without which a smaller `RX_VK_PARALLEL_SEG` cuts no finer once a command buffer has six segments' worth | | `RX_VK_PARALLEL_BACKOFF=0` | The tuner's trials as before: one every 1,800 presents after one that kept the way, each with 104 frames the other way. With the back-off (the default, `gpu/vk/replay_threads.h` `Tuner`) each trial that keeps the way, the other way longer than both of its own parts, puts the next one off twice as long (3,600, 7,200, then 14,400 presents at most; a margin of 10% or more counts twice), one too close to tell keeps the old interval (the first such one after a change of scene or way: 900 presents), and once the way has held so the other way is tried for 64 frames, not 104; a short trial that finds the other way shorter changes nothing itself, a full trial follows at once. The next trial comes early (900 presents after the last one at the soonest, never while the rule says no trial) when the scene clearly changes: two windows of 120 frames in a row whose draws moved by a quarter (200 at least) or whose replay or GPU time moved by 30% (1 ms at least) from the first window after the way was set, or a trial whose first and last parts did. In the lab's steady Market a trial came every ~2,090 presents, each ~100 frames 1.9 ms longer; the trial lines now end with when the next may come, and `parallel replay, the scene changed` says when one was brought forward | -| `RX_VK_RECORDER=thread`, `=defer`, `=sync` | (off unless set; not yet measured in the game) `thread`: the replay thread's `vkCmd*` calls packed with copies of their arguments into a ring and made by a recorder thread of their own, in order (`gpu/vk/vk_recorder.h`); the replay thread waits for it at every command buffer begin, end and reset, submit, wait, readback and destruction. Beside the Replay helper threads too (no longer refused with `RX_VK_PARALLEL`): a parallel replay's segments make their calls themselves, after the ring has been run (`segments_begin`), and the recorder is idle while they record. With it on, the frame statistics' `a frame's recorder thread` line says how many of the replay's calls it made a frame and in how long, and how long the replay thread waited for it; what it saves is the replay time line's `commands` with it against without it. `defer` and `sync` are its test steps (held and run on the same thread). `RX_VK_RECORDER_RING_MB` (8), `RX_VK_RECORDER_SPIN_US`, `RX_VK_RECORDER_STATS=0`, `RX_VK_RECORDER_POISON=1`, `RX_VK_RECORDER_PIN` (`next`, `sibling` or a mask) | +| `RX_VK_TUNER_SETTLED=0` | The tuner's rule judges every window and a trial the scene changed under decides, as before. With settled scenes only (the default, `gpu/vk/replay_threads.h` `Tuner`, after Kaefergeneral's #dev-fixes post: on his 5900X and RTX 3090 the rule turned the helper threads on in an area's load and the first trial after the arrival kept them, and Automatic was 6 to 10% slower than Off) the rule judges a window of 120 frames only if the one before it describes the same scene (its draws within a quarter, 200 at least, its replay and GPU times within 30%, 1 ms at least: the back-off's test), so a load's window and the first after it get no verdict and the rule's "on" comes a window after a scene settles; a trial whose first and last parts moved so changes nothing, either way, and the next comes 120 presents after a window like the one before it, not 900 or more later. The trial's line says so (`the scene changed under it`), and `parallel replay, the scene settled` when the next is set | +| `RX_VK_RECORDER=thread`, `=defer`, `=sync`, `=off` | (unset: `thread` on an NVIDIA GPU with 12 logical processors or more, and on an AMD GPU with 6 physical cores or more in the process's affinity but for AMD's Windows driver for RX 5000 and RX 6000 (`gpu/vk/recorder_auto.h`); else off. The log's `vk: recorder thread on/off by default` line says which rule applied; `RX_VK_LAB_AB=norec` makes the calls directly in its phases, to measure it) `thread`: the replay thread's `vkCmd*` calls packed with copies of their arguments into a ring and made by a recorder thread of their own, in order (`gpu/vk/vk_recorder.h`); the replay thread waits for it at every command buffer begin, end and reset, submit, wait, readback and destruction. Beside the Replay helper threads too (no longer refused with `RX_VK_PARALLEL`): a parallel replay's segments make their calls themselves, after the ring has been run (`segments_begin`), and the recorder is idle while they record. With it on, the frame statistics' `a frame's recorder thread` line says how many of the replay's calls it made a frame and in how long, and how long the replay thread waited for it; what it saves is the replay time line's `commands` with it against without it. `defer` and `sync` are its test steps (held and run on the same thread). `RX_VK_RECORDER_RING_MB` (8), `RX_VK_RECORDER_SPIN_US`, `RX_VK_RECORDER_STATS=0`, `RX_VK_RECORDER_POISON=1`, `RX_VK_RECORDER_PIN` (`next`, `sibling` or a mask) | | `RX_VK_REPLAY_PRIO=0` or `1` | (Triforce, on by default, after NX2navi's `recorder_prio`; `=0` as before) A thread runs at the highest priority while it replays the game's frames (the submitting thread's replay and its capture, the present, the stopped work the order thread lets on: `gpu/vk/vk_replay.cpp` `ReplayPin`) and gets its own priority back after, so the game's code it runs between replays keeps the game's priority; the core kept for the replay is kept from the game's threads only, and the helper threads and the capture's copy helpers (above normal) outranked a replay at the game's priority there. A thread at that priority or above already is left alone. The replay thread of `RX_VK_ASYNC=1` and the recorder thread of `RX_VK_RECORDER=thread` take the level for good. `=1`: above normal. The log's `vk: a thread that replays runs at` line says which (`gpu/vk/replay_prio.h`) | +| `RX_VK_REPLAY_PIN_SOFT=0`, `=2` | (Triforce, 11-22, the performance audit's THREAD-09, after Source A's; Windows) A thread is pinned while it replays to the core the host keeps for the replay, and that core is kept from the game's threads, not from other programs': where another program's high-priority thread holds it, the pinned replay waits for it, ready to run (Source A, a 6+8-core hybrid processor with an overlay service at priority 15: 14 to 37 frames over 40 ms in a 40 s run, none with this). Not set: on a processor with cores of more than one kind the pin is soft: the thread may run on the reserved core and on the fastest-kind cores among its own processors, the reserved core's last logical processor is its ideal one, and the core it is on when the pin starts is left out so that it moves at once; its affinity and ideal processor come back after the replay. The replay thread of `RX_VK_ASYNC=1` gets the same processors for good. A processor of one kind of core keeps the strict pin, and nothing changes there. `0`: the reserved core alone everywhere, as before; `2`: soft everywhere (the lab; the knob `pinsoft` of `RX_VK_LAB_AB` does it by phases). A thread of another program that wakes on the core every millisecond moves the replay again and again (Source A: 54.3 against 44.6 fps, one run each): `0` is for that. The log's `vk: a replay's pin is` line says which (`gpu/vk/replay_pin.h`, `gpu/vk/vk_replay.cpp` `ReplayPin`, `replay_pin_soft`) | | `HOS_VCORES=0`, `HOS_CORES=<1..8>`, `HOS_RESERVE_CORE=0` | No virtual cores; the cores the game sees (6); no core kept for the replay (one is, with 8 or more logical processors) | +| `HOS_VCORES=0`, `HOS_CORES=<1..8>`, `HOS_RESERVE_CORE=0`, `=1` | No virtual cores; the cores the game sees (6); no core kept for the replay (one is, with 8 or more logical processors). (Triforce, 11-22, the performance audit's THREAD-49, after Source A's) On the cache die of a two-die X3D processor with SMT off none is kept unless `HOS_RESERVE_CORE=1` asks for it or the replay has a thread of its own (`RX_VK_ASYNC=1`): the game then has all 8 of the die's cores (Source A, a 9950X3D with SMT off: 59.8-59.9 fps with none kept against 56.9-57.4 with one). The log's `cpu: no core kept for the replay on the cache die` line says so. With SMT on and on every other processor nothing changes (`host/hos_x3d.h` `hos_x3d_keep_replay_core`) | +| `HOS_AV1=mf`, `HOS_AV1_MISSING=1` | (Triforce, 11-22, the performance audit's IO-32; Windows) The game's AV1 movies on Media Foundation first (Microsoft's AV1 Video Extension), as before, and on dav1d only where the PC has no such decoder. Without it dav1d goes first where the FFmpeg the build links has libdav1d, as on Linux, and Media Foundation where it has not or where libdav1d does not open (`host/video_av1.h`, `host/nn_video.c` `dec_init_sized`): Media Foundation's path decodes in software on every GPU (`host/mf_av1.c` takes the synchronous decoder). `HOS_AV1=dav1d`, the old way to ask for dav1d first, is the default now. `HOS_AV1_MISSING=1`: Media Foundation as if the extension were missing. Each movie logs one line at its Stop, Flush or its decoder's end: its pictures, the decoder, the mean and the worst time to decode a submission, and the pictures skipped to catch up (`host/video_movie.h`). The two decoders were not timed against each other in the game | +| `HOS_VIDEO_RGBA=1` | (Triforce, 11-22, the performance audit's IO-36) Every planar 4:2:0 movie picture (libdav1d's) is converted to RGBA when it is decoded, as before: a swscale pass, an 8 MB allocation and an 8 MB row copy for a 1080p picture, on the movie thread. Without it such a picture is kept as NV12 alone, as Media Foundation's NV12 pictures already were, and its pool makes the RGBA only if the game reads RGBA, which this game's movie library does not; planar 4:2:0 with alpha is converted at once either way. The NV12 buffer is zeroed only for an odd size or a format that is not copied into it, and U and V are interleaved 16 pairs a step with SSE2 (`host/video_nv12.h`, `host/nn_video.c` `queue_frame`). The movie's line in the log (IO-32) has the time a picture took with either. Not timed in the game | +| `HOS_VIDEO_UNBOUNDED_CATCHUP=1`, `HOS_VIDEO_NO_CATCHUP=1` | (Triforce, 11-22, the performance audit's THREAD-82) The movie decoder's acquire skips the pictures behind the movie's clock and decodes the next ones to skip them too. `HOS_VIDEO_UNBOUNDED_CATCHUP=1`: with no bound but a guard of 600 steps, as before: with a decoder slower than the movie (19.5 ms a picture against 16.7) each picture decoded to catch up puts the clock further ahead, so an acquire decoded up to about 300 pictures with its lock held, threw them away, and the next started further behind. Without it each decode that gives a picture is timed, about the last 8 kept as one number, and the acquire stops decoding ahead when that number is more than the gap between two pictures (from the pictures' own times) or once it has decoded for one gap, and shows the late picture it has (`host/video_catchup.h`, `host/nn_video.c` `dec_acquire`): such a movie falls behind its sound instead of stalling, and its line in the log says "slower than it plays". A decoder faster than the movie catches up as before, a few pictures an acquire. `HOS_VIDEO_NO_CATCHUP=1`: no picture skipped at all (the movie in slow motion under 60 fps). Not run in the game | +| `HOS_VIDEO_INPUT_LIMIT=` | (Triforce, 11-22, the performance audit's IO-34) How many submissions the movie decoder holds, not decoded yet or decoded and not taken, before it refuses one: 16 when unset; `0`: no limit, every submission taken, as before. The game submits a whole movie at once on the thread that also takes the pictures, so at a movie's start the pictures that came due while it submitted were skipped. With the limit a submission is refused at once with the console's result for a full decoder (0x1c266f) and the game submits it again later; the stream's last submission is never refused, and a decoder with no codec behind it refuses nothing. Each decoder has its own "ready for a submission" event, clear from a refusal until an acquire, a Stop or a Flush makes room (it was one event for all, always set); the movie's line in the log counts the refusals (`host/video_input.h`, `host/nn_video.c` `dec_submit`, `input_room`). If a movie ever hangs on a black screen, try `=0` first. Not run in the game | +| `HOS_MOVIE_PACE=0` | (Triforce, 11-22, the performance audit's PRESENT-30) The frame rate cap is kept while a movie plays, as before. Without it, at a cap of exactly 60 and with the Steady frame rate row off, the frames are paced by 240 while a movie player has updated in the last quarter second, and the game's own limiter and its time step follow as they do at a cap of 240 (`host/movie_pace.h`, `host/nn_vi.c` `movie_pace_decide`, `pace_fps`, `realtime_step`): the game's movie player takes one picture an update at most and its updates follow the frame loop, so at 60 every frame that came late was a picture the decoder skipped. Game time stays real time; the limiter is put back where it was when the movie is over, and the log has a line at the movie's start and one at its end. The cap itself (the Steady row, the vsync event, the hand-off to RivaTuner or Reflex) does not move. Movies use more CPU and power with it. Not run in the game | +| `A64_NO_VBYTES=1`, `A64_NO_VMOVES=1` | (Triforce, 11-21, the performance audit's LIFT-02; read when the game is translated, and in the lift key) The translator writes TBL with a one-register table and EXT as loops of single bytes through the stack, a 128-bit vector load as two halves and DUP by element as a loop, each followed by a 16-byte read of the register that cannot be forwarded from those stores. `nx2/fastvec.py` rewrites each on the lifted text as one instruction's helper (PSHUFB; PALIGNR or two shifts; one 16-byte load; one broadcast), the same bytes in the same registers (`tests/test_vbytes.py` runs every shape both ways). `A64_NO_VBYTES=1` keeps the translator's text for TBL and EXT, `A64_NO_VMOVES=1` for the loads and DUP. A build with them needs a hot profile of its own (`RX_PGO=gen`) | +| `A64_DMB_LIGHT=1` | (Triforce, 11-22, the performance audit's LIFT-47, after Source B's; read when the game is translated, and in the lift key; off by default) The translator writes every DMB and DSB as `recomp_barrier()`, an out-of-line full fence. x86-64 keeps load-load, load-store and store-store order in hardware, so with this set the eight DMBs that ask only for those (ISHLD, ISHST and the OSH, NSH and full-system forms) are lifted as `A64_DMB_LDST()` (`nx2/insn.py` `dmb_light`), which is nothing but a barrier for the compiler on x86-64 and the full fence on any other host (`nx2/rtgen.py`). A full DMB, every DSB, ISB and STLR's locked store keep theirs. Unset, empty or `0`: the kit's text. `tests/test_dmb_light.py` runs every such word both ways, alone and between a store and a load. Not counted in this game's code and not measured in the game: how many of its barriers are of these kinds is not known, and what other threads see cannot be shown by a test on one thread. With it set the hot functions that hold such a DMB no longer match the shipped hot profile: such a build needs one of its own (`RX_PGO=gen`) | +| `HOS_FMA_FAST=0`, `A64_NO_FMA_SOFT=1` | (Triforce, 11-22, the performance audit's LIFT-06, after Source A's; only in a build without FMA3, x86-64-v2: with FMA3 none of it is compiled) Without a hardware FMA every double-precision FMADD, and each single-precision one its binary64 fast path cannot prove, ran the translator's exact body, integer code of many words (about 180 ns a call and 13.7% of a core on an i7-3770, Source A's figures). `nx2/fma_soft.py` puts a short way out at the head of `recomp_fpe_fma64` and `recomp_fpe_fma32` for the plain case: default FPCR (FZ or not), three normal operands and a normal result, by an exact product and sum in binary64, rounded to odd, then rounded once, FPSR.IXC set when inexact; everything else goes on into the body. `tests/test_fma_soft.py` compares both bodies with it and without, value and every byte of the context. `HOS_FMA_FAST=0` (read once when the game runs): the exact bodies only, as before. `A64_NO_FMA_SOFT=1` (read when the game is translated, and in the lift key): the translator's runtime text as it was. Not measured in a game on a CPU without FMA3 | +| `A64_NO_FMA_STEPS=1` | (Triforce, 11-22, the performance audit's LIFT-05, after Source A's; read when the game is translated, and in the lift key; only in a build with FMA3, x86-64-v3) The single-precision FRECPS and FRSQRTS steps (2 - a*b and (3 - a*b) / 2, the Newton-Raphson steps of a reciprocal and a reciprocal square root) were computed in binary64 on x86-64 (two widenings, a multiply, a subtraction, a narrowing) with a test for a binary32 midpoint, which went to the exact body. `nx2/fma_steps.py` gives their fast paths the guard the fused multiply-add's already had: with FMA3 one fused instruction, the step rounded once as on the console; the tests that decide whether the result is kept are unchanged (`tests/test_fma_steps.py` runs every form both ways and against the exact body). `A64_NO_FMA_STEPS=1`: the translator's text for the two steps. The compile define `-DA64_FPX_NO_FMA32` returns all three (FMADD/FMLA and the two steps) to the binary64 form. Source A measured the steps alone at 23% to 29% fewer cycles; not measured in the game, and how often this game runs them is not counted | +| `A64_VECFP=1` | (Triforce, 11-22, the performance audit's LIFT-03, after Source A's; read when the game is translated, and in the lift key; off by default) The translator's passes turn straight FADD, FSUB and FMUL .4S and the .4S by-element multiplies into one SSE op; other float forms stay a helper call a lane or exact integer code a lane. With this set `nx2/vecfp.py` lifts each of them as `if (!a64_vfpx_q_
(c, ...)) { the C as before }`: the helper does the op in SSE and writes the register only when the scalar fast paths' rules prove the result is the console's with no new FPSR bit. `1`: every group; or a list of `elem` (FMUL, FMLA, FMLS .2S by element), `vec` (FMUL .2S; FMLA, FMLS and FDIV .2S and .4S; the multiply-adds need FMA3 and decline without it), `minmax` (FMAX, FMIN, FMAXNM, FMINNM: scalar S, .2S, .4S; kept unless a NaN or, under FZ, a subnormal is involved) and `fcvt` (FCVT S to D and D to S, FCVTL, FCVTL2, FCVTN). Unset, empty or `0`: the translator's text. `tests/test_vecfp.py` runs 137 guest words three ways (with it, without it, exact). Source A measured +3.5% to +5.1% fps where the game's update bounds the frame, for 3.7% more compile time; not measured in this game, which is why it is off. A build with it needs a hot profile of its own (`RX_PGO=gen`) | +| `HOS_HEAP_COMMIT=1` | (Triforce, 11-21, the performance audit's MEM-09) On Windows the game's heap (~6.9 GB asked for at its start) is left reserved and committed as it is touched, 64 KB at a first touch, by the handler that commits pages nothing handed out; committed whole, all of it counted against RAM plus the page file, touched or not. A fault another thread's commit answered is tried again, a commit refused at Windows' commit limit is tried again for 5 s, and a file read or write reads each page of its buffer once before the kernel's call, which faults nothing in. The log says `heap: left reserved` and counts the heap's first touches. `1`: all of it committed at the start, as before; so it is in large pages (`HOS_LARGE_PAGES`) and on Linux (`host/heap_commit.h`, `mkw_glue.c` `commit_on_touch`, `hos_mem.c` `SetMemoryHeapSize`) | +| `RX_VK_DMABUF_NVIDIA=1`, `RX_VK_NO_DMABUF=1`, `RX_VK_DMABUF_PIECE_MB=` | (Linux) Where the game's memory is a memfd, each pool of it is offered to the driver as a dma-buf made by udmabuf, larger pools as several of at most udmabuf's limit bound into one sparse buffer; the log's `vk: guest memory as dma-bufs:` line says yes or why not. `RX_VK_NO_DMABUF=1`: host-pointer imports only. `RX_VK_DMABUF_PIECE_MB=`: pieces of at most n MB. (Triforce, 11-22, the performance audit's OS-08, after Source A's; not yet run on Linux) NVIDIA's proprietary driver refuses every such dma-buf and the pool ends as a host-pointer import anyway, after a refused import each (up to 198 ms for one during a load) and a kernel message at each start: with that driver none is offered, `/dev/udmabuf` is not opened, and the line says `no (NVIDIA's driver refuses them: host pointers)` (`gpu/vk/dmabuf_gate.h`). NVK and RADV keep their dma-bufs. `RX_VK_DMABUF_NVIDIA=1`: offered to NVIDIA's driver too, as before | +| `HOS_COMMIT_WAIT_MS=` | (Triforce, 11-22, the performance audit's OS-04, after Source A's and Source B's; 5000 unless set) How long a refusal at Windows' commit limit (RAM and the page file full, error 1455) is waited out, tried again every 50 ms: such a refusal often ends in moments, as a page file set to grow grows or commit is given back. The game's own commits waited so already (the row above); now the host's own tables do too (a failed allocation there ended the game at once: `host/host_alloc.h`, `hos_xmalloc` in `host/mkw_glue.c`), and the renderer's failed `new`, which was an uncaught `std::bad_alloc`: its new handler sleeps 50 ms and lets the allocation be tried again, up to this long a thread, but only while the system has less than 4 GiB of commit left (a `new` refused with more free is not the limit's and throws at once), and not for the few allocations that have their own answer to a refusal (`gpu/vk/new_wait.h`, `gpu/vk/vk_main.cpp` `new_refused`). The log has `refused ... made after N ms` for the first rescues. `0`: no wait anywhere, a refusal fails at once; 60000 at most | +| `HOS_TEXELS_KEEP_CORE=1` | (Triforce, 11-21, the performance audit's THREAD-20) `nvnTextureReadTexels` and `nvnTextureWriteTexels` reach the back end, which waits for all the GPU work submitted and then for the replay's lock; the game calls them from its loading threads at an area's arrival and at boot. The calling thread gives its virtual core up for that call, as it does around a readback, so the game threads allowed only on that core do not wait with it. A write kept for a renderer not started yet never waits and keeps the core. `1`: kept, as before (`host/rx_backend.c` `rx_nvn_read_texels`, `rx_nvn_write_texels`) | +| `HOS_LOG_SYNC=1`, `HOS_LOG_BATCH_MS=` | The log's lines are written by a thread of their own, not by the thread that logs: a write that blocks (a console, a busy disk) holds no game thread. The writer waits a little after a line and writes the lines of that time in one write (`HOS_LOG_BATCH_MS`, 250 unless set, at most 2000; 0: the lines there are at each wake); a queue half full goes out at once, and every way out of the game writes the queue first. `HOS_LOG_SYNC=1`: each line written by the thread that logs it. (Triforce, 11-22, the performance audit's THREAD-56, after Source A's; not yet run on Linux) The writer is Linux's and the Steam Deck's too now: there every `hos_log` wrote stderr in the calling game thread, its virtual core held, so `HOS_LOG_SYNC=1` is Linux as it was. And `nvnCommandBufferEndRecording`'s two lines (the sweep's, the 50,000-recording line) are written after the recordings' lock is released, the virtual core given up, on every system (`host/hos_thread.c` `log_queue`, `host/nvn_native.c`) | +| `RX_VK_INVALIDATE_QUICK=1` | (Triforce, 11-22, the performance audit's THREAD-25, after Source A's; off unless set. Source A has it on; here it waits for a run in the game, as a wrong answer skips the wait a readback of GPU results needs, the exposure flashes' kind of fault) The game invalidates each shape's 64-byte occlusion slot, per view, and the host gives the calling thread's virtual core up before every such call and queues for it again after, though nearly none has anything to wait for. With `1` the back end is asked first, with no lock and no wait: a range with nothing to invalidate, or one below 512 GB with no fast fence poll's order ticket still to wait for, no live texture's storage in it, no texture the GPU wrote ever listed there and no write-back ever made into its pages, is one `invalidate` returns from at once, so the call is left out and the core kept. The two "ever" answers are bits a page that are only set, before their tables change (`gpu/vk/page_marks.h`); a page written back once has the call made for good. Source A measured Hyrule Field at 50.5 to 51.4 fps and its 1% low up by two to four frames a second, its busiest worker keeping the core 31,444 times against 2,604. Every 10 s a `vk: invalidates:` line counts the calls left out and made. With `RX_VK_LOG_INVALIDATE`, `RX_VK_LOG_READBACK` or `RX_VK_INVALIDATE_WAIT` set every call is made; the lab's knob `noinvq` makes them all in its phases (`gpu/vk/vk_replay.cpp` `invalidate_quick`, `host/rx_backend.c` `rx_nvn_invalidate`) | +| `HOS_OPEN_KEEP_CORE=1` | (Triforce, 11-21, the performance audit's IO-24) nn::fs::OpenFile gives its virtual core up only around the host's own open, as before. The default gives it up before the path is worked out too: that reaches the button prompts' builder, which builds a file at its first open (a quarter of a second and more) while the other openers wait for it, and none of them holds a core meanwhile. Every return takes the core back before it answers the game (`host/nn_fs.c` OpenFile, `host/prompts.c` `hos_prompts_font_file`) | +| `HOS_SHORT_SLEEP_TIMER=1`, `=0` | (Triforce, 11-21, the performance audit's OS-31) How sleeps under a millisecond wait. Not set: with 6 or fewer logical processors in the process's mask (Windows, one processor group) the game's own short sleeps wait on the high-resolution timer, which wakes about 0.5 ms late, and the host's own yield until their time; with more, all yield, as before. The game's idle job workers sleep 20 us and look again about 7,600 times a second, and a yielding sleeper keeps a processor busy. `1` (any value but 0, off, no, false): every short sleep on the timer, as the switch did before; `0`: every short sleep yields. (Triforce, 11-22, the performance audit's OS-32, after Source A's; not yet run on Linux) Off Windows the processors are not counted and, not set, every short sleep yields, as before; `HOS_SHORT_SLEEP_TIMER=game` puts the game's own short sleeps on the timer and leaves the host's yielding at any processor count, on any system: Source A's default off Windows (on Linux the timer wakes tens of microseconds late, not half a millisecond), which was never measured in the game by itself and is to be tried on a Steam Deck before `SHORT_SLEEP_ELSEWHERE` in `host/short_sleep.h` makes it what Linux does when the switch is not set. The log's `sleep:` line says which (`host/short_sleep.h`, `host/hos_thread.c` `sleep_for`) | +| `HOS_DISCARD_KEEP_CORE=0`, `=1` | (Triforce, 11-22, the performance audit's OS-40, after Source A's; not set: as before, which is NOT Source A's default. It reverses a choice made here on purpose, changes when a game thread waits, and was not measured here) A discard of memory the game let go is milliseconds in the kernel for a region of some MB, and before it the lazy write-back's pages there are paid and pages a GPU import holds are zeroed; the game threads allowed only on the freeing thread's virtual core wait all the while. Not set: the core is given up for the discard's own kernel call, and only with the memory limit on (`HOS_MEM_LIMIT`), as before. `0`: a freed region's thread gives its core up before the whole of the discard and takes it back once, after the region is unlisted and never under the regions' lock; the discard's own call gives it up without the limit too (`FreeMemoryPages`). Source A at its Market: core waits of 50 ms and more 287 to 19, the longest 177 to 45 ms, frames unchanged; the freeing thread once waited 124 ms for its core back, in a load. `1`: the core kept through every discard, with the limit too. With `0` the log counts such frees and the longest take-back every 1,000 (`host/hos_mem.c` `discard_core`, `hos_region_free`, `discard_pages`) | +| `HOS_SHORT_SLEEP_TIMER=1`, `=0` | (Triforce, 11-21, the performance audit's OS-31) How sleeps under a millisecond wait. Not set: with 6 or fewer logical processors in the process's mask (Windows, one processor group) the game's own short sleeps wait on the high-resolution timer, which wakes about 0.5 ms late, and the host's own yield until their time; with more, all yield, as before. The game's idle job workers sleep 20 us and look again about 7,600 times a second, and a yielding sleeper keeps a processor busy. `1` (any value but 0, off, no, false): every short sleep on the timer, as the switch did before; `0`: every short sleep yields. The log's `sleep:` line says which (`host/short_sleep.h`, `host/hos_thread.c` `sleep_for`) | +| `HOS_SHORT_SLEEP_POLLERS=`, `=auto` | (Triforce, 11-22, the performance audit's OS-33, after Source A's; off unless set: it changes how the game's threads wait and was not measured here) A cap on the game threads that yield through a sleep under a millisecond at once. Where those sleeps yield (more than 6 logical processors, `HOS_SHORT_SLEEP_TIMER` not set) every idle job worker keeps a processor busy; with the cap, a game thread that would be one more than `` yielding waits on the high-resolution timer for that sleep instead, about 0.5 ms late. `auto`: a quarter of the process's logical processors (2 with 8, 4 with 16, 6 with 24), Source A's rule, whose provisional numbers are 2.7% less CPU a frame at the same frame rate with 16 processors, and 1.7% of the frame rate lost in a 24-thread run held to 2. The game's MainThread and Presentation Thread and the host's own sleeps are never counted and always yield. Not set, `0`, `off`: no limit, as before. The start's `sleep: at most` line says the cap, and with `HOS_SLEEP_LOG=1` a `sleep:` line every 5 s counts the sleeps that took the timer for it (`host/short_sleep.h` `short_sleep_pollers`, `short_sleep_poll_enter`, `host/hos_thread.c` `sleep_for`) | +| `HOS_OS_YIELD=0` | (Triforce, 11-22, the performance audit's THREAD-41, after Source A's; off unless set: it changes how the game's threads wait and was not measured here) The game's `YieldThread` hands its virtual core to a waiting game thread of equal or better priority and then also gives the processor up (`SwitchToThread`, a call into the kernel, thousands of times a second from the game's job system). `0`: only the virtual core's hand-over, the console's yield; Windows still takes the processor at the end of a time slice. Source A, the game held to 4 processors: half of its six executors' CPU time was inside that call, and without it the mean frame was 0.26 to 0.34 ms shorter; with 16 processors no difference. The risk is a thread yielding in a loop for one that waits on its processor, on PCs with 2 to 4 logical processors. Not set or `1`: both, as before. With `HOS_VCORES=0` the processor's yield stays whatever this says; `SleepThread(0)` and the short sleeps keep theirs (`host/hos_thread.c` `os_yield`, `YieldThread`) | | `HOS_VCORE_SPIN_US=`, `HOS_VCORE_SPIN_MAX=` | (Triforce, on by default; `HOS_VCORE_SPIN_US=0` as before) A game thread that waits for a virtual core it asked a holder to give up spins for it, up to 250 us, before it sleeps (`host/hos_thread.c` `vc_spin`, after NX2navi's `core_take`): the holder gives the core up at its next checkpoint, mostly within microseconds, and the waiter then runs without an OS wake. Who gets a core does not change, only how soon. At most `HOS_VCORE_SPIN_MAX` threads spin at once and the rest sleep at once: by default a quarter of the logical processors beyond 8, at most 4 (none with 8 or fewer, 1 with 12, 2 with 16). The `vcores:` line (every 5 s) counts the waits that ended in the spin and those that slept, the time spun, the spins left out for the cap, and how soon a granted core ran after a spin and after a sleep; the start's `vcores: a waiter spins up to` line says what is in effect. A yield with no thread waiting for a core takes no lock (NX2navi's yield fast path) | +| `HOS_VIDEO_ACQUIRE_TRIES=1`, `HOS_VIDEO_KEEP_CORE=1` | (Triforce, on by default; each `=1` as before) A movie's Acquire (`host/nn_video.c` `dec_acquire`) waits for a picture up to 100 ms by the clock, and not at all after a Stop or Flush with nothing submitted since: the old wait, 50 sleeps of 2 ms that each took 4-5 ms on Windows, ran out only after ~250 ms once a movie had stopped, a frame that long at every load from the title. It also decodes without the calling thread's virtual core, taken back only after the decoder's lock is let go, so other game threads run meanwhile. `HOS_VIDEO_ACQUIRE_TRIES=1`: the 50 sleeps; `HOS_VIDEO_KEEP_CORE=1`: the core held throughout | +| `HOS_VCORE_WIDE=1` | (Triforce, 11-22, the performance audit's GAME-22, after Source B's; off unless set, never measured there or here) A game thread whose core mask names two or more cores takes any free virtual core, not only those. Source B's reason: the game's two main threads may run on two of three cores only, where per-core job workers of better priority preempt them while the other cores stand idle. A thread with a one-core mask, which guards the game's lock-free per-core data, or with no mask, stays where it was; no two threads ever run under one core number either way. The risk is the one `HOS_VCORE_ANY=1` (every thread on any core, which crashed) has, for fewer threads: a thread that reads its core number, blocks and goes on under another. The log's `vcores: ... asks for ideal core ... and core mask` lines name every thread whose mask has two or more cores (all masks when the switch is set to anything): with no such line the switch changes nothing (`host/hos_thread.c` `vc_allowed`, `core_mask_note`) | | `RX_VK_HELPER_SPIN_US=` | (Triforce, on by default; `=0` as before) The parallel replay's spins (`gpu/vk/helper_spin.h`, after NX2navi's replay workers): a helper thread with no segment to record spins up to 100 us for the next before it sleeps, and the replay thread spins as long for a segment a helper records before it sleeps for it. The frame statistics' `its spins` line, after the parallel replay's parts, says how often each ended in the spin or slept and the time spun | +| `HOS_X3D=0` | (Windows) The game's threads and the replay's core are not kept on the die with the largest L3 cache. By default they are (the replay's core only where one is kept there: `HOS_RESERVE_CORE`), on a processor whose L3 caches differ in size and whose largest serves 8 or more cores (AMD's two-die X3D parts: a 7950X3D's or 9950X3D's 3D V-Cache die; `host/hos_x3d.h`), and the log's `cpu: logical processors ... share the largest L3 cache` line says so. On every other processor nothing changes and the log has no such line | | `HOS_X3D=0` | (Windows) The game's threads and the replay's core are not kept on the die with the largest L3 cache. By default they are, on a processor whose L3 caches differ in size and whose largest serves 8 or more cores (AMD's two-die X3D parts: a 7950X3D's or 9950X3D's 3D V-Cache die; `host/hos_x3d.h`), and the log's `cpu: logical processors ... share the largest L3 cache` line says so. On every other processor nothing changes and the log has no such line | +| `HOS_SIM_CPU=/` | (Triforce, 11-22, the performance audit's OS-49, after Source B's; a lab switch, off unless set; Windows) A PC with fewer cores, simulated: the process is held to `` of the PC's physical cores (the die with the largest L3 first, then the fastest class, then the last ones, so that the replay's kept core is among them; one core always stays out; `host/hos_sim_cpu.h`), set before any thread, and the renderer sizes its pools and thread counts by `` (`gpu/vk/cpu_count.h`: an affinity mask alone does not change `hardware_concurrency`). `6/12` and `4/8` play as those PCs' pools do; `6` alone is 6/6. The log says `cpu: HOS_SIM_CPU=` with the mask, or that it was not applied (then the variable is cleared and nothing changes). Not run in the game. `tests/test_sim_cores.py` | +| `RX_VK_SIM_GPU_SLOW=` | (Triforce, 11-22, the performance audit's OS-49, after Source B's; a lab switch, off unless set) A slower GPU, simulated (`gpu/vk/sim_gpu.h`): for each ms of work the recordings measure, (factor - 1) ms more, from over 1 to 10 (`2.5` is read). A one-invocation compute loop (`shaders/sim_gpu_burn.comp`) ends the replay's own recordings between full barriers, inside the GPU time they are measured over, so the frame's GPU time and Resolution Auto see the slower card; a third timestamp a recording measures what the burn took and the difference is carried (owed time held between -50 and 100 ms, burns of 1 to 16 ms, the loop's time calibrated 0.8/0.2). A line every 10 s: work, added, the factor reached, dropped. It repeats the fast card's relative answer only: no bandwidth, driver or upscaler cost. Not run in the game. `tests/test_sim_cores.py` | | `HOS_POWER_THROTTLING=1` | (Windows) Windows' power throttling of a window not in front left as it is (for A/B runs). By default every run, the shader preparation's too, opts out of it at the start: full speed, not the efficiency speed (EcoQoS) Windows 11 gives such a process, and the 1 ms timer resolution kept, which Windows 11 may otherwise ignore for it (Windows 10: full speed only). The log's `power:` line says what Windows took | -| `HOS_HOST_ZSTD=1` or `2` | The game's files (`.zs`) decoded by the host's zstd (`third_party/zstd`) instead of the game's own translated decoder; off unless set, and nobody has played with it on. `1`: the host decodes what `host/host_zstd.h` takes (whole buffers, and the loading threads' streams with their content size and room for it), the game's code the rest, errors included. `2`: check, the game's code decodes everything as before and the host beside it, and the log's `zstd check: ... MISMATCH` lines name every difference (its `zstd:` lines count what was compared): the way to try it before `1`. Game build `e0f3ccfa` only. The lift needs both of its calls (`nx2/oot_patches.py` `patch_host_zstd`): a build log that says `the host's zstd decoder's calls were found at 1 of 2 places` has one only, and the switch then covers only that function | +| `HOS_HOST_ZSTD=0` or `2`, `HOS_HOST_ZSTD_AWAY=0/1/2` | The game's files (`.zs`) are decoded by the host's zstd (`third_party/zstd`) instead of the game's own translated decoder: the host decodes what `host/host_zstd.h` takes (whole buffers, and the loading threads' streams with their content size and room for it), the game's code the rest, errors included. (Triforce, 11-21, the performance audit's IO-01) On when not set: the host is given the content's exact size to write into (libzstd may park literals in spare room past a block, where the game's decoder never writes), and check mode ran clean in the game. `0`: the game's own decoder for everything, as before. `2`: check, the game's code decodes everything and the host beside it, and the log's `zstd check: ... MISMATCH` and `wrote past the end` lines name every difference (its `zstd:` lines count what was compared). `HOS_HOST_ZSTD_AWAY`: which decodes give their virtual core up: `1` (not set) whole buffers with 128 KB of input or room, streams never; `0` none; `2` whole buffers and stream feeds from 64 KB, as it was. Game build `e0f3ccfa` only. The lift needs both of its calls (`nx2/oot_patches.py` `patch_host_zstd`): a build log that says `the host's zstd decoder's calls were found at 1 of 2 places` has one only, and the switch then covers only that function | ### Building @@ -295,18 +440,32 @@ A build reads and writes tens of gigabytes. When the game folder or the build fi | Switch | What it does | |---|---| | `HOOKSHOT_JOBS=` (or `--jobs`) | Compilers at once (default one per CPU thread, each started when the free memory holds it; `HOOKSHOT_MEM_GATE=0` turns that check off) | +| `HOOKSHOT_LOW_PRIORITY=1` | (Triforce, 11-22, the performance audit's OS-57) (Windows) The builders start the build tools (`make_game.py` and everything it runs) below normal priority, as every build was before. The default is normal priority: Source A reports, without a measurement, that below normal Windows kept the steps that are one process (the read, the lift) on a hybrid Intel CPU's efficiency cores while the performance cores idled, and a first build took over an hour. The compilers then compete with the desktop like any program (`HOOKSHOT_JOBS` bounds them). Read by Triforce Setup (`setup/Setup.cs`), the launcher's build (`launcher/Logic/Build/BuildRunner.cs`) and the two older builders (`builder/HookshotBuilder/Runner.cs`, `builder/py/hookshot_builder/core.py` `build_priority`); Linux sets no nice value either way. The shaders' preparation at a build keeps its workers below normal in the normal-class process (`gpu/vk/vk_draw.cpp` `prepare_priority`) | | `HOOKSHOT_LIFT_JOBS=` | Processes that write the lifted C. Forked on Linux and macOS: default the cores, at most 16, one per 2 GB of free memory. Spawned on Windows (`nx2/emit_pool.py`), where they also cut the big chunks and share the page marks: default half the processors, at most 8, one per GB of free memory. `1`: one process, on every system; the C is the same whatever the count | | `HOOKSHOT_KEEP_BUILD_FILES=1` | Keeps the lifted C and the objects after a build, so that the next one compiles only what changed | +| `HOOKSHOT_LIFTED_C_REGEX=1` | (Triforce, 11-22, the performance audit's IO-40) The lifted functions' definitions are found by the patterns' own `finditer`, tried at every line of the lifted C, as before. By default `nx2/lifted_c.py`'s `definitions` and `noted_starts` jump from one `"\nvoid f_"` to the next and try the pattern only there, for `build.py`'s memory estimate, `chunks.py`'s costs and cuts and `split_hot.py`'s pieces: the same matches (`tests/test_lifted_c_scan.py`), so the same files. `lifted_c.py`, `chunks.py` and `split_hot.py` are in the lift key: the first build with this change translates the game once more. The other half of the entry, translating while the package's romfs is still being extracted, is not here | +| `HOOKSHOT_KEEP_LIFT=0` | (Triforce, 11-22, the performance audit's LIFT-27) The translated game is kept as little as before: its key (what decides whether a build translates the game again) hashes every `A64_`, `OOT_` and `RX_` variable and the folders' paths, and `nx2/released_translation_stamp.json`, and any failed build forgets it. By default the key takes of the environment only what shapes the C, the lifter's `A64_` switches and `RX_TARGET` (`tests/test_lift_key_env.py` holds the lift's files to that), and leaves out the release's shader translation stamp, which `build.py` alone reads; and a build that failed in its compile, only in the program's own files (`host/`, the renderers) and in nothing the lift wrote, keeps the key once, so that the retry compiles the rest without translating the game again (a second such failure in a row forgets it, as a generated header cut short fails the program's own files too). The key changes once with this, and the two values of the switch have different keys. Not here yet: keeping the translated game after a player's finished build, which is the owner's decision (`nx2/make_game.py` `lift_key`, `keep_lift_after_failure`) | | `HOOKSHOT_EARLY_COMPILE=0` | (Windows) Not compiled beside the lift: libopus and the two renderers, which need no translated code. By default `make_game.py` starts `build.py` with `RX_EARLY=1` for them while the game is translated (the lift's own processes take at most half the processors there), and the build proper finds their objects made | | `HOOKSHOT_MEM_GIANT=1` or `0` | Whether the memory gate counts a function of 1.2 MB or more by the square of its size (a chunk that is one 5 MB function takes 6.2 GB, not the 1.9 GB it is counted at otherwise). By default only on a PC with less than 14 GB free when the compile starts: with more, counting the giants in full halves the compilers running at once for no need | | `HOOKSHOT_MEM_LOG=1` | For each file compiled, `@@mem `: the most memory its compiler was seen holding, beside what the memory gate counted it at (`mem_need` in `nx2/build.py`). For fitting the gate after a change to the lift or the compiler | +| `HOOKSHOT_PROMPTS_FOLDER_KEY=0` | (Triforce, 11-21, the performance audit's IO-43) A game built from romfs and exefs folders has its button prompts drawn at every build and update, as before. The default keeps them when nothing they are made from changed: with no unpack stamp to tell the dump by, the key is the content of the game's files the tool reads (the font, the textures, the layouts, the camera's pack, the size table, the dictionaries and every language's messages). Every game's key also holds `host/prompts.c` and the tools' release (`nx2/make_game.py` `prompts_key`, `prompt_game_files`) | +| `HOOKSHOT_PROMPTS_WIDEN_EACH=1` | (Triforce, 11-22, the performance audit's IO-42) The button prompts' tool widens the "lines that stay" image (a 9 by 9 maximum, what a leader line pays to cross them) at every route, as before. By default the widened array is kept for the last image asked, and every route of one line is given the same image: 4 or 5 widenings a controller face where there were 35 or 37, each a filter over a 2-megapixel image (the five faces' lines on blank stand-in images: 21 s where they took 42). The lines drawn are the same (`nx2/prompt_assets.py` `guide_widened`) | +| `HOOKSHOT_DUMP_LINKS_LSTAT=1` | (Triforce, 11-21, the performance audit's IO-41) A build from romfs and exefs folders checks them for links with an `os.lstat` for every entry, as before. The default reads each entry's kind and attributes from the folder's own listing (`os.scandir`), 1.3 to 1.5 s where the one-by-one check took 36 to 49 s over the game's 186,583 files, and stops the build at a folder it cannot list, which the walk passed over. The copy that follows makes each folder once and lets `os.link` say that a file is there already (`nx2/make_game.py` `refuse_dump_links`, `link_or_copy`) | +| `HOOKSHOT_NUP_IN_PLACE=0` | (Triforce, 11-22, the performance audit's IO-51) A `.nup` package's unpack writes each file to `.part` and renames it, and resolves the output folder and looks at every folder above the file for every file, as before. The default writes each file straight to its name (one `os.lstat`, then a create that fails if anything is there) and looks at each folder once: under a virus scanner on NTFS the temporary's create and rename and the repeated looks were most of a small file's cost over the game's 186,583 files. A file that is there whole is kept, one of another size is unlinked and written again, one whose write fails is deleted; the files written are the same (`nx2/nup.py` `extract`, `Destination`) | +| `HOOKSHOT_FINISH_COPY_ALL=1` | (Triforce, 11-22, the performance audit's IO-45) The end of a build copies every file into the game folder as `.new` and swaps it in, as before, and keeps no note. By default the finish notes in the game folder (`.triforce_installed.json`), for each file it installed, the SHA-1 of the source and the installed copy's size, modification time, file number and mode, and the next finish passes over a file whose source hashes the same and whose installed copy is still as noted, without reading it: `image.bin` and the prebuilt DLLs at an update that did not change them. The exe and the renderer are linked again at every build with the link's time in them, so they are still copied. A file not noted, changed by hand, on a file system without file numbers (FAT, exFAT), or under a note that does not read is copied; deleting the note copies everything (`nx2/make_game.py` `INSTALLED_NOTE`, `installed_same`) | +| `HOOKSHOT_LINK_REPRO=1` | (Triforce, 11-22, the performance audit's IO-45) (Windows) The game's exe and the renderer are linked with `/Brepro`: lld writes a hash of the file where it otherwise writes the time of the link, so two links of the same objects are the same bytes, and the finish above passes an unchanged exe over instead of copying its 460 MB. Off unless set: it changes the bytes of every linked file once (`nx2/build.py` `repro_link`) | | `HOOKSHOT_MINGW=`, `HOOKSHOT_MSVC=1` | The llvm-mingw to build with; the Visual Studio route instead | +| `HOOKSHOT_WORK_LOCK=0` | (Triforce, 11-22, the performance audit's IO-47) `make_game.py` takes no lock on its work folder, as before. By default it holds an OS lock on `/make_game.lock` for as long as it runs (`msvcrt.locking` on Windows, `flock` elsewhere), taken after an earlier build left running is stopped and before the stale objects go: a second build on the same folder stops at once with the first one's process id instead of compiling the same objects again and racing on the staged game folder, which nothing prevented on Linux (two builders, a builder beside the launcher, `--cli`). The lock ends with its process, so a build that crashed never blocks the next; an error other than "held by another" is said and the build runs without the lock (`nx2/make_game.py` `lock_work`) | | `HOOKSHOT_SINGLE_INSTANCE=0` | (Triforce) Hookshot Builder takes no lock on its build tools, as before. By default a second builder for the same tools folder (`builder/HookshotBuilder/Instance.cs`, named after `Tools.Root`), its `--cli` included, waits up to 5 s for one that is ending, then says so and ends: two of them built into one work folder and stopped each other's build. Triforce Setup has its own lock (`setup/Guard.cs` `Instance`) | | `TRIFORCE_LEGACY_PATHS=1` | (Triforce) Triforce Setup keeps .NET Framework's legacy path checks, which refuse paths of 260 characters and more whatever its manifest says (an exe csc builds is taken as written for .NET 4.0). By default `setup/Setup.cs` `Main` sets the two switches 4.6.2 defaults to (`Switch.System.IO.UseLegacyPathHandling`, `Switch.System.IO.BlockLongPaths`) before any path is used, so that with Windows' own LongPathsEnabled on Setup reads and writes deep paths | +| `HOOKSHOT_SEGMENT_HEAP=0` | (Triforce, Windows; on by default since 11-22: the lab's play A/B, 81.4 against 80.6 fps) The exe's manifest asks for Windows' segment heap (`heapType SegmentHeap`, Windows 10 2004 and newer) for the whole process, its DLLs and the graphics driver too, in place of the NT heap, whose lock AMD's driver's compile threads waited on through the shader preparation (`nx2/win_manifest.py`, `tests/test_segment_heap.py`). The heap is fixed when the process starts: a build with it and one without are the two arms. `0`: the NT heap, as before. The log's `the shaders' preparation runs on` line names the process heap (`segment` or `NT`) | | `RX_CPM=`, `WGPU_NATIVE=` | The CPM cache (Vulkan and SPIR-V headers, VMA, FFmpeg) and wgpu-native | | `RX_TARGET=windows` | On Linux: the Windows game, with llvm-mingw's Linux build in `HOOKSHOT_MINGW` | | `RX_RELEASE=1` | A release build (`make_game.py` sets it): no console window, the shaders prepared before the title | +| `RX_VK_PGO=gen`, `=`, `=0`; `RX_VK_PGO_FILE=` | (Triforce, 11-21, the performance audit's LIFT-10) A play profile for the renderer's own code (`gpu/vk/*.cpp`; not the vendored recompiler, ImGui or zstd). `gen`: a build whose renderer counts its branches; run it with `RX_VK_PGO_FILE=`, make `.reset` once the scene to record is loaded and `.dump` at its end, then `llvm-profdata merge -o work\triforce_vk.profdata `. ``: built with that profile (its contents are in the object keys, its path is not). Not set: `make_game.py` passes `work\triforce_vk.profdata` when the tools have one and the compiler can read it; `0`: none. A profile from another version of the renderer is taken function by function, the changed ones built without it. Record it again for each release, and compare against `RX_VK_PGO=0` before shipping one (`nx2/build.py` `vk_pgo_flags`, `host/pgo_rt.c`) | | `RX_PGO=gen` or `RX_PGO=` | The hot code counting its branches for a profile, or built with one (`make_game.py` passes the shipped profile) | +| `A64_HOT_STICKY=0` | (Triforce, 11-22, the performance audit's LIFT-30, after Source A's; read when the game is translated, and in the lift key) `nx2/split_hot.py` deals the hot functions out afresh and gives every hot piece code_hot.c's whole preamble, as before. By default `\image\hot_pieces.txt` keeps each hot function's piece: the next translation leaves the functions it knows where they were and puts a new one in the piece it shares the most calls with, else in the cheapest, unless that makes the costliest piece more than 1.15 times a fresh split's (then the split is the fresh one); and each piece keeps only the declarations of `f_` and `hos_` names it uses. An update that changes a few hot functions then compiles one or two hot pieces of about ten MB again, not up to eight. The functions' text is the same either way, and the hot profile goes by function name, so it fits both. The file is not in the lift key (`tests/test_sticky_hot_pieces.py`) | +| `RX_COLD_OPT=O2` or `=`, `RX_COLD_PART_MB=` | (Triforce, 11-22, the performance audit's LIFT-17, after Source A's; opt-in) The hot pieces are compiled at -O2 with the play profile and every other lifted chunk at -O1. `RX_COLD_OPT=O2`: every other lifted chunk at -O2 too; `=`: only the chunks the file lists, one file name a line (`code_012.c`). Unset, empty or `O1`: as before. Each -O2 chunk is compiled as parts of `RX_COLD_PART_MB` MB by the lift's own estimate (6 when unset; `0`: no cut), cut only where a function begins and written under `/split` with a key file a chunk, so a second build writes nothing (`nx2/cold_parts.py`, `tests/test_chunk_parts.py`): -O2's time grows with the square of a function's size. The lifted files themselves are not touched. Source A's figures: +4.4% and +5.5% fps where the game's update bounds the frame, for about twice the lifted compile's processor time (23 minutes against 5 without the cut); not built or measured with this game. The names begin with `RX_`, so setting them translates the game again once | | `RX_MARCH=` | The CPU level to build for (default this PC's: `x86-64-v3`, else `x86-64-v2`, else `x86-64`) | | `RX_VK_ONLY=1` | Only the Vulkan back end, with no lift | | `OOT_WORK`, `OOT_DUMP`, `OOT_EXEFS`, `OOT_ROMFS` | The work folder and the dump's folders, over `work/dump.txt` | diff --git a/gpu/render/rx_texture.cpp b/gpu/render/rx_texture.cpp index c7d424a..948db9d 100644 --- a/gpu/render/rx_texture.cpp +++ b/gpu/render/rx_texture.cpp @@ -15,6 +15,7 @@ #include #include "common/alignment.h" +#include "../vk/cpu_count.h" // ((Triforce, 11-22, OS-49) the threads to size by: HOS_SIM_CPU #include "common/div_ceil.h" #include "video_core/texture_cache/format_lookup_table.h" #include "video_core/textures/astc.h" @@ -766,7 +767,7 @@ std::unique_ptr decode(const TICEntry& tic, const TexInfo& info_in, uint6 copy_layers >= 4 && level_size.depth == 1 && (size_t)level_size.width * level_size.height * 4 <= (8u << 20)) { const Extent3D block = AdjustMipBlockSize(tiles, li.block, level, li.num_levels); const u32 stride_alignment = StrideAlignment(tiles, info.block, gob, li.bpp_log2); - const unsigned nthreads = std::max(1u, std::min(copy_layers, std::thread::hardware_concurrency() - 1)); + const unsigned nthreads = std::max(1u, std::min(copy_layers, cpu_count() - 1)); std::atomic next_layer{0}; static std::atomic t_swz, t_dec, t_up; auto work = [&] { diff --git a/gpu/vk/achievement_rows.h b/gpu/vk/achievement_rows.h new file mode 100644 index 0000000..c19e440 --- /dev/null +++ b/gpu/vk/achievement_rows.h @@ -0,0 +1,145 @@ +// (Triforce) Achievements on the menu's side: the list the host names (host/achievements.c; vk_api.h HOS_OPT_ACH_*), which of +// them are earned, the one a notice shows, and the Achievements page's choice and pages. vk_menu.cpp draws them. +// +// The achievements are the host's: it reads the lists, asks the save data and keeps what was earned. The menu is told +// each one's name, the line about it, its icon and the day it was earned, a byte a call as the save rows' lines are, and +// later each one earned as it is. An icon is drawn, not a picture file: a shape and a tone (host/achievements.h has the +// same in the same order, ACH_SHAPES and ACH_TONES; tests/test_achievements.py holds the two together). +// +// The host's calls (text, add, done, earn) come on its threads, one at a time (its lock); the list is drawn on the +// renderer's: names, lines and icons are written before count says they are there and never after, a day before its +// achievement's earned says so. +// +// No ImGui and no Vulkan: tests/test_achievements.py compiles it alone. +#pragma once +#include +#include +#include + +namespace achievement_rows { +constexpr int kSlots = 128; // (host/achievements.h ACH_MAX) +constexpr int kName = 41, kAbout = 161, kDay = 16; +constexpr int kEarned = 256; // (vk_api.h HOS_ACH_EARNED) +enum Shape { kStar, kSword, kGem, kMedal, kNote, kToken, kShield, kShapes }; +enum Tone { kGold, kGreen, kRed, kBlue, kPurple, kOrange, kYellow, kSilver, kTones }; +constexpr int shape_of(int icon) { return (icon & 15) < kShapes ? icon & 15 : kStar; } +constexpr int tone_of(int icon) { return (icon >> 4 & 15) < kTones ? icon >> 4 & 15 : kGold; } + +struct List { + char names[kSlots][kName] = {}; + char abouts[kSlots][kAbout] = {}; + char days[kSlots][kDay] = {}; + unsigned char icons[kSlots] = {}; + std::atomic earned[kSlots] = {}; + std::atomic count{0}; // (the achievements shown: 0 until the host's list is whole) + std::atomic fresh{-1}; // (the one earned now, until the notice takes it) + // the host's side + char in[kName + kAbout + kDay + 8] = {}; + int in_n = 0, added = 0; + + /// HOS_OPT_ACH_TEXT: the next byte of what is being named + void text(int byte) { + if (byte > 0 && byte < 256 && in_n + 1 < (int)sizeof in) in[in_n++] = (char)byte; + } + /// `from` copied into a field of `cap` bytes, cut at a whole letter + static void put(char* to, int cap, const char* from) { + int n = (int)strlen(from); + if (n > cap - 1) { + n = cap - 1; + while (n && ((unsigned char)from[n] & 0xC0) == 0x80) n--; + } + memcpy(to, from, (size_t)n); + to[n] = 0; + } + /// HOS_OPT_ACH_ADD: the bytes so far are the next achievement's name and, each after a line feed, the line about it and + /// the day it was earned; bits: its icon, and kEarned + void add(int bits) { + in[in_n] = 0; + in_n = 0; + if (count.load(std::memory_order_acquire) || added >= kSlots) return; // (shown already, or no room left) + const int slot = added++; + char* about = strchr(in, '\n'); + char* day = about ? strchr(about + 1, '\n') : nullptr; + if (about) *about++ = 0; + if (day) *day++ = 0; + put(names[slot], kName, in[0] ? in : "(no name)"); + put(abouts[slot], kAbout, about ? about : ""); + put(days[slot], kDay, day ? day : ""); + icons[slot] = (unsigned char)(bits & 255); + earned[slot].store((bits & kEarned) != 0, std::memory_order_relaxed); + } + /// HOS_OPT_ACH_COUNT: the list is whole: shown from now on + void done(int n) { + in_n = 0; + if (count.load(std::memory_order_acquire)) return; + count.store(n < 0 ? 0 : n < added ? n : added, std::memory_order_release); + } + /// HOS_OPT_ACH_EARNED: the achievement at that place was earned now, the bytes so far the day. false: no such place + bool earn(int slot) { + in[in_n] = 0; + in_n = 0; + if (slot < 0 || slot >= shown()) return false; + if (!earned[slot].load(std::memory_order_relaxed)) put(days[slot], kDay, in); + earned[slot].store(1, std::memory_order_release); + fresh.store(slot, std::memory_order_release); + return true; + } + int shown() const { return count.load(std::memory_order_acquire); } + bool is_earned(int slot) const { return slot >= 0 && slot < kSlots && earned[slot].load(std::memory_order_acquire) != 0; } + int earned_count() const { + int n = 0; + for (int i = 0, end = shown(); i < end; i++) n += is_earned(i); + return n; + } + /// The one earned since the last ask, for the notice; -1 none + int take_fresh() { return fresh.load(std::memory_order_relaxed) < 0 ? -1 : fresh.exchange(-1, std::memory_order_acq_rel); } +}; + +/// The page's cards: `columns` to a row, `capacity` to a page (both 1 at least), n of them +struct View { + int choice = 0, page = 0; + + static int pages(int n, int capacity) { return n <= 0 ? 1 : (n + (capacity < 1 ? 1 : capacity) - 1) / (capacity < 1 ? 1 : capacity); } + /// The choice kept to the list, and the page shown to the pages there are + void keep(int n, int capacity) { + choice = n <= 0 ? 0 : choice < 0 ? 0 : choice >= n ? n - 1 : choice; + const int last = pages(n, capacity) - 1; + page = page < 0 ? 0 : page > last ? last : page; + } + /// The page that shows the choice + void reveal(int capacity) { page = choice / (capacity < 1 ? 1 : capacity); } + /// The arrows (cards: Left and Right, rows: Up and Down), Page Up and Down (turns), Home and End. true: the choice moved + bool move(int cards, int rows, int turns, bool home, bool end, int n, int columns, int capacity) { + if (n <= 0) return false; + const int was = choice; + const int step = cards + rows * (columns < 1 ? 1 : columns) + turns * (capacity < 1 ? 1 : capacity); + choice = home ? 0 : end ? n - 1 : choice + step; + keep(n, capacity); + if (choice == was) return false; + reveal(capacity); + return true; + } + /// The wheel, or a page arrow that leaves the choice where it is: the page shown turned. true: it changed + bool turn(int by, int n, int capacity) { + const int was = page; + page += by; + keep(n, capacity); + return page != was; + } + /// A click on a card. true: the choice changed + bool choose(int slot, int n) { + if (slot < 0 || slot >= n || slot == choice) return false; + choice = slot; + return true; + } +}; + +/// How far a notice has come in, 0 (out of sight) to 1 (in place): in over the first 250 ms of its `shown_ms`, out over +/// the last 300 +inline float notice_in(int64_t age_ms, int64_t shown_ms) { + if (age_ms < 0 || age_ms >= shown_ms) return 0; + const float in = (float)age_ms / 250.0f, out = (float)(shown_ms - age_ms) / 300.0f; + const float k = in < out ? in : out; + return k > 1 ? 1 : k; +} +} // namespace achievement_rows diff --git a/gpu/vk/astc_bc_policy.h b/gpu/vk/astc_bc_policy.h index 09bdd26..ccbe8b7 100644 --- a/gpu/vk/astc_bc_policy.h +++ b/gpu/vk/astc_bc_policy.h @@ -13,4 +13,12 @@ inline bool all_by_vram(uint64_t allowed) { return allowed && allowed < (8704ull << 20); } +// (Triforce, 11-22, the performance audit's TEX-12, after Source A's) And every integrated GPU, whatever it is allowed. Its +// allowance is a share of the system's RAM (a 16 GB handheld reported 10,361 MB), so by the allowance alone it kept the +// fine-block textures of 2 to 64 MB as RGBA8, in RAM the game needs too, inside a texture budget capped at 2,560 MB. +// Source A measured the difference on a discrete card (RX_VK_ASTC_BC_ALL 0 against 1, Kokiri Forest: decoded ASTC +// 2,496-2,532 MB against 1,579-1,587, the GPU's frame the same); it was not run on an integrated GPU there either. +// The shape of tex_detail::by_vram(allowed, integrated). RX_VK_ASTC_BC_ALL=0: the finer ones as RGBA8, as before +inline bool all(uint64_t allowed, bool integrated) { return integrated || all_by_vram(allowed); } + } // namespace astc_bc_policy diff --git a/gpu/vk/astc_opaque.h b/gpu/vk/astc_opaque.h index 9e1612e..6b08b47 100644 --- a/gpu/vk/astc_opaque.h +++ b/gpu/vk/astc_opaque.h @@ -18,12 +18,28 @@ // 65535 / 65535 = 1. Modes 4, 5, 10, 12 and 13 carry alpha, and 2, 3, 7, 11, 14 and 15 are the HDR ones, which take the // decoder's other path: no for all of those, without decoding them (such a block may be opaque all the same). // One partition has its mode in bits 13 to 16. Several partitions share one mode when bits 23 and 24 are zero (the mode is -// bits 25 to 28); when they do not, the modes' other bits sit below the weights and are not looked for: no. +// bits 25 to 28). +// - (Triforce, 11-21, the performance audit's TEX-50, after Source B's) When bits 23 and 24 are not zero each partition has a +// mode of its own, read as the decoder reads it: 2, 5 or 8 more bits for 2, 3 or 4 partitions sit right below the weights +// (at 128 less the weights' bits less their own count); with bits 25 to 28 under them they are a class bit a partition, +// then two mode bits a partition, and a partition's mode is (bits 23 and 24 as a number, less 1, plus its class bit) * 4 +// plus its two bits. Yes only when every partition's mode is one of 0, 1, 6, 8 and 9. Those bits count in the decoder's +// last error too: 96 bits of weights, the 29 read and 5 or 8 more wrap, and the block is an error. Such blocks were +// refused unread, and a texture with one of them stayed BC3. parts_too = false (RX_VK_TEX_BC1_PARTS=0): refused, as before. #pragma once +#include #include namespace astc_opaque { +/// Whether a block with a mode a partition is looked at (above); false: refused unread, as before. Set once, at the start +inline bool parts_too = true; +/// How many such blocks were found opaque (the log's) +inline std::atomic parts_passed{0}; + +/// Whether the decoder gives both endpoints of a colour endpoint mode the alpha 255: the modes without alpha, low dynamic range +constexpr bool opaque_mode(unsigned cem) { return cem == 0 || cem == 1 || cem == 6 || cem == 8 || cem == 9; } + /// `n` bits (16 at most) of a block from bit `at`, the lowest first, as the decoder's StreamBits reads them inline uint32_t bits(const uint8_t* b, unsigned at, unsigned n) { uint32_t v = 0; @@ -88,17 +104,32 @@ inline bool block(const uint8_t b[16], unsigned bw, unsigned bh) { range = range - 2 + (layout != 9 && (mode & 0x200) ? 6 : 0); const unsigned wbits = weight_bits(weights, range); if (wbits < 24 || wbits > 96) return false; - unsigned read, cem; + unsigned read, cem = 0, extra = 0; + uint32_t base = 0; if (parts == 1) { cem = bits(b, 13, 4), read = 17; } else { - const uint32_t base = bits(b, 23, 6); - if (base & 3) return false; // (a mode a partition: not looked for) - cem = base >> 2, read = 29; + base = bits(b, 23, 6); + read = 29; + if (base & 3) { // (a mode a partition: its other bits are below the weights) + if (!parts_too) return false; + extra = parts == 2 ? 2 : parts == 3 ? 5 : 8; + } else { + cem = base >> 2; + } } - // (the decoder's last error, its count of the remaining bits wrapping. With one shared mode it cannot: 96 + 29 + 2) - if (wbits + read + (dual ? 2u : 0u) > 128) return false; - return cem == 0 || cem == 1 || cem == 6 || cem == 8 || cem == 9; + // (the decoder's last error, its count of the remaining bits wrapping. With one shared mode it cannot: 96 + 29 + 2. With a + // mode a partition it can: 96 + 29 + 8) + if (wbits + read + extra + (dual ? 2u : 0u) > 128) return false; + if (!extra) return opaque_mode(cem); + // (the decoder's own order: (extra << 6 | base) >> 2 is a class bit a partition, then two bits a partition) + const uint32_t all = ((bits(b, 128 - wbits - extra, extra) << 6) | base) >> 2; + for (unsigned i = 0; i < parts; i++) { + const unsigned cls = (base & 3) - 1 + ((all >> i) & 1); + if (!opaque_mode((cls << 2) | ((all >> (parts + 2 * i)) & 3))) return false; + } + parts_passed.fetch_add(1, std::memory_order_relaxed); + return true; } /// Whether every one of `n` blocks (16 bytes each, in any order) is one the decoder writes opaque diff --git a/gpu/vk/astc_reuse.h b/gpu/vk/astc_reuse.h new file mode 100644 index 0000000..30783c6 --- /dev/null +++ b/gpu/vk/astc_reuse.h @@ -0,0 +1,133 @@ +// (Triforce, 11-21, the performance audit's TEX-14, after Source A's) Decoded ASTC used again. +// +// The game's textures are ASTC, decoded on the GPU and for the most part encoded again as BC3 or BC1: tens of ms of GPU +// time and the decoder's scratch for each. When the game leaves an area it finalizes its textures, and when it comes back +// it makes them again, mostly at other addresses: the same blocks are decoded and encoded again while the image made of +// them the first time may still be in VRAM, kept until the cache needs its room. Source A hashes an ASTC texture's blocks +// at its first whole upload, lists a texture the game finalizes under that hash, and gives a new texture with the same +// blocks and the same shape that image by a GPU copy, the old one going after it. Its tours of two areas four times at an +// 8 GB budget: peak decoded ASTC 3,603 to 2,562 MB, evictions 23,378 to 16,109, the slowest frame 637 to 487 ms. +// +// What keeps a hit right: 128 bits of hash over the blocks as the decoder is given them; every field of the image that +// the decode made and of the storage it was decoded from (Shape); and a source that was uploaded from memory and never +// drawn into, still listed (used again at its own address or gone, it is unlisted). +// +// RX_VK_ASTC_REUSE: 0 or not set, off (the audit: off until measured in play); 2: the blocks are hashed, the finalized +// textures listed and each hit counted, and every texture decoded as before: what a reuse would save, for no risk; +// 1: the hit's image is copied. +// +// No Vulkan here: tests/test_astc_reuse.py compiles it. +#pragma once +#include +#include +#include +#include + +#include "upload_same.h" // (content_hash: the first of the two lanes) + +namespace astc_reuse { + +enum Mode { kOff = 0, kCopy = 1, kCount = 2 }; +/// RX_VK_ASTC_REUSE's mode: 1 and 2 as they are, anything else off +inline Mode mode(const char* asked) { + if (!asked || !*asked) return kOff; + const int v = atoi(asked); + return v == 1 ? kCopy : v == 2 ? kCount : kOff; +} + +/// A second hash of n bytes, of other constants and another mix than upload_same::content_hash's: eight chains over 64 +/// bytes a step, as there, so that both read at memory's speed +inline uint64_t lane2(const void* p, uint64_t n) { + const uint8_t* b = static_cast(p); + const auto rotl = [](uint64_t x, int r) { return (x << r) | (x >> (64 - r)); }; + uint64_t h = 0xD6E8FEB86659FD93ull + n * 0x9FB21C651E98DF25ull, i = 0; + if (n >= 512) { + uint64_t c[8]; + for (int k = 0; k < 8; k++) c[k] = h + 0xA0761D6478BD642Full * (uint64_t)(2 * k + 1); + for (; i + 64 <= n; i += 64) { + uint64_t w[8]; + std::memcpy(w, b + i, 64); + for (int k = 0; k < 8; k++) c[k] = rotl(c[k] + w[k] * 0xE7037ED1A0B428DBull, 29) * 0x8EBC6AF09C88C6E3ull; + } + for (int k = 0; k < 8; k++) h = rotl(h ^ c[k], 23) * 0x589965CC75374CC3ull + 0x1D8E4E27C47D124Full; + } + for (; i + 8 <= n; i += 8) { + uint64_t w; + std::memcpy(&w, b + i, 8); + h = rotl(h + w * 0xE7037ED1A0B428DBull, 29) * 0x8EBC6AF09C88C6E3ull; + } + for (; i < n; i++) h = rotl(h ^ b[i], 11) * 0x589965CC75374CC3ull; + h ^= h >> 32, h *= 0xD6E8FEB86659FD93ull, h ^= h >> 32; + return h; +} + +/// The blocks' hash, 128 bits, made of pieces given in order (a big texture's storage is read 64 MB at a time): each piece +/// goes through both lanes kStep bytes at a time, while they are in the cache, and is folded in with its length +struct Hash { + uint64_t a = 0, b = 0; + bool operator==(const Hash& o) const { return a == o.a && b == o.b; } + explicit operator bool() const { return (a | b) != 0; } +}; +inline constexpr uint64_t kStep = 256u << 10; +struct Hasher { + Hash h; + uint64_t bytes = 0; + void add(const uint8_t* p, uint64_t n) { + for (uint64_t at = 0; at < n; at += kStep) { + const uint64_t m = n - at < kStep ? n - at : kStep; + h.a = (h.a ^ upload_same::content_hash(p + at, m)) * 0x9E3779B97F4A7C15ull + m; + h.b = (h.b + lane2(p + at, m)) * 0xC2B2AE3D27D4EB4Full ^ (m << 1 | 1); + } + bytes += n; + } + /// The hash, never all zero (zero is "not hashed") + Hash done() const { return (h.a | h.b) ? h : Hash{1, 0}; } +}; + +/// What has to be the same besides the blocks for one image to stand for another: the image the decode made (its format, +/// size, levels and layers, the top levels left out and whether by the card's texture detail, kept as BC3 or BC1 or +/// RGBA8, BC1 given up for BC3, resized for AMD) and the storage it was decoded from (the NVN format, the key's layout) +struct Shape { + uint32_t format, width, height, depth, levels, layers, skip, nvn_format, layout; + bool bc, bc_resized, bc1, bc1_no, detail_skip; + bool operator==(const Shape& o) const { + return format == o.format && width == o.width && height == o.height && depth == o.depth && levels == o.levels && + layers == o.layers && skip == o.skip && nvn_format == o.nvn_format && layout == o.layout && bc == o.bc && + bc_resized == o.bc_resized && bc1 == o.bc1 && bc1_no == o.bc1_no && detail_skip == o.detail_skip; + } +}; + +/// The finalized textures whose image may serve again, by their blocks' hash. T is the cache's texture; one is in the list +/// at most once: listed and unlisted by its pointer +template +class Freed { +public: + void list(const Hash& hash, T* t) { by_hash.emplace(hash.a, Entry{hash.b, t}); } + void unlist(const Hash& hash, T* t) { + auto [a, b] = by_hash.equal_range(hash.a); + for (auto it = a; it != b; ++it) + if (it->second.t == t) { + by_hash.erase(it); + return; + } + } + /// One listed with this hash, both lanes, whose shape is `shape` (shape_of(T*) gives each one's) and that is not + /// `self`; null: none + template + T* find(const Hash& hash, const Shape& shape, const T* self, ShapeOf shape_of) const { + auto [a, b] = by_hash.equal_range(hash.a); + for (auto it = a; it != b; ++it) + if (it->second.b == hash.b && it->second.t != self && shape_of(it->second.t) == shape) return it->second.t; + return nullptr; + } + size_t size() const { return by_hash.size(); } + +private: + struct Entry { + uint64_t b; + T* t; + }; + std::unordered_multimap by_hash; +}; + +} // namespace astc_reuse diff --git a/gpu/vk/barrier_rules.h b/gpu/vk/barrier_rules.h index f2ff7f6..a87d50c 100644 --- a/gpu/vk/barrier_rules.h +++ b/gpu/vk/barrier_rules.h @@ -71,6 +71,28 @@ inline bool split_in_pass(const char* env, uint32_t vendor, bool every_draw) { return false; // helper threads record most frames there, and the rule stands aside for them) and no gain (90.6 / 89.2 fps // against 92.8 / 90.3 with the old barriers) } +/// ((Triforce, 11-22, the performance audit's REPLAY-14, after Source A's) Two workarounds for Intel's drivers at a submission's +/// splits (vk_replay.cpp maybe_split and the dry pass's points). No other vendor has either unless its switch asks. +/// RX_VK_SPLIT_BEFORE_DRAW: a part ends only right before a plain draw. On an Arc B580 a command buffer that begins with mesh +/// draws drew its later plain draws as exploded triangles (Source A: 40 shard frames of 1,121 before, 0 of 1,101 after). Unset: +/// on for Intel's GPUs (vendor 0x8086); 0 off; anything else on, on any GPU +inline bool split_before_draw(const char* env, uint32_t vendor) { + if (env && *env) return std::strcmp(env, "0") != 0; + return vendor == 0x8086; +} +/// RX_VK_SPLIT_UBFLUSH: whether a part's end writes the submission's uniform updates so far into the game's memory. Without +/// mirrors they land in the memory later parts' other reads and the game's CPU see, mid-submission, where an unsplit submission +/// writes them once at its end: white blocks and flashes on Intel's integrated GPUs (Source A: 1 run of 3 before, 0 of 3 +/// after). Unset: on, as before, but off on Intel's GPUs without mirrors; 0 off; anything else on +inline bool split_ub_flush(const char* env, uint32_t vendor, bool mirrors) { + if (env && *env) return std::strcmp(env, "0") != 0; + return !(vendor == 0x8086 && !mirrors); +} +/// A part may end right before this op: a plain draw (NOP_DRAW, _IDX, _INDIRECT: not a mesh draw, not a dispatch) that will +/// draw (rendering on, no rasterizer discard) +constexpr bool cut_ok(bool plain_draw, bool render_enabled, bool raster_discard) { + return plain_draw && render_enabled && !raster_discard; +} /// Whether a part's split goes on in the same pass: the rule on, a pass open at the split, nothing recorded at the part's end (quiet: /// the work count as it was after the pass's end) and no event's write held then (written behind a full barrier before the part's /// write-backs, which copy its word to the game's memory: the GPU drained there all the same). ((Triforce, P4) no parallel replay's @@ -85,6 +107,25 @@ inline bool split_start_full(bool carry, bool recorded_between) { return !carry || recorded_between; } +/// ((Triforce, 11-22, the performance audit's SYNC-66, after Source B's) RX_VK_CHUNK: how many draws and dispatches a part of a +/// long submission has (vk_replay.cpp maybe_split). 120 on every card since the parts were made: the GPU starts early, and each +/// split costs a queue submit, two full barriers and the part's bookkeeping, about 27 times a frame in a serial replay. Source B +/// cuts at 128 where the GPU idles while the CPU records and at 4096 on desktop drivers; what pays here is not measured on any +/// driver, so the default by driver is 120 for each until the lab picks one (S1 and S2 with RX_VK_PARALLEL=0 and the lab knob +/// chunk:). vendor: the card's PCI vendor; radv: Mesa's RADV +constexpr int64_t kPartDraws = 120; +inline int64_t part_draws_default(uint32_t vendor, bool radv) { + (void)vendor, (void)radv; + return kPartDraws; +} +/// The size in force: RX_VK_CHUNK's number when the variable is set (0 or less: whole submissions, never split); else +/// knob_or_default, the lab knob chunk:'s number in its phases and the driver's default outside them +inline int64_t part_draws(bool env_set, int64_t env_value, int64_t knob_or_default) { + return env_set ? env_value : knob_or_default; +} +/// As the size was read before: the variable's number, 120 unset +inline int64_t part_draws_before(const char* env) { return env ? std::atoll(env) : 120; } + /// RX_VK_EVENT_BARRIER: "late" the rule on any card; "full" (or any other value) as before; unset or empty: with the pass barrier's /// game mode (pass_barrier::game: NVIDIA's driver unless RX_VK_PASS_BARRIER says otherwise) inline bool event_late(const char* env, bool pass_game) { @@ -139,5 +180,18 @@ inline bool batch_flush(Batch m, bool added_held, size_t held, size_t cap) { if (m == Batch::Parts && !added_held) return true; return held >= cap; } +/// ((Triforce, 11-22, the performance audit's SYNC-09, after Source A's) RX_VK_BATCH_IDLE: a submit that joins the held batch +/// sends it at once when the GPU has finished every batch sent. A held batch otherwise waits for a wait, a present, the 2 ms +/// poll or the cap of 32 while the GPU sits idle beside it, until the game waits mid-frame at a capture's late pages; while the +/// GPU is busy, submits still group. Source A has it on wherever batching is (RADV), off where guest memory stayed host-pointer +/// imports (each submit there pays amdgpu's page check, 15-45 ms). HERE: on only with a number other than 0; unset it is OFF, +/// on every driver, until a Linux RADV machine has run it (the audit's lab proof; with NVIDIA's driver and AMD's and Intel's on +/// Windows nothing is held). No guard for host-pointer imports is built: with the switch off unless asked there is nothing for it to guard +inline bool batch_idle_on(const char* env) { return env && std::atoi(env) != 0; } +/// Whether a held batch is looked at for an idle GPU after a submit joined it: the switch on, Hold mode only (Parts sends by +/// its own rule, when the game's submit returns), and something held (the cap's flush may just have sent it all) +inline bool batch_idle(Batch m, bool held, bool on) { + return on && m == Batch::Hold && held; +} } // namespace mkwvk::barrier_rules diff --git a/gpu/vk/bc_scratch.h b/gpu/vk/bc_scratch.h new file mode 100644 index 0000000..6311896 --- /dev/null +++ b/gpu/vk/bc_scratch.h @@ -0,0 +1,66 @@ +// (Triforce, 11-21, the performance audit's TEX-10, after Source A's) How many layers the BC3 scratch has. +// +// Big ASTC textures are decoded into a 2048 x 2048 RGBA8 scratch and encoded again as BC3 on the GPU, into an output +// buffer. Both were made with 8 layers, 128 + 32 = 160 MB (and the second scratch of the textures encoded at another +// size, 128 MB more), whatever a run used; and a run decodes one layer (RX_VK_BC_LAYERS, the dark-line fix), so 140 MB +// were held and never written, for good once made (RX_VK_BC_IDLE_S is 0). Sized by the layers one run uses they are +// 16 + 4 = 20 MB. Source A measured 139 MiB less GPU memory on a Steam Deck, the median of 7 pairs, at the same fps. +// RX_VK_BC_SCRATCH_FULL=1: all 8 layers, as before. +// +// The rule alone, no Vulkan: tests/test_bc_scratch_layers.py compiles it. +#pragma once +#include +#include +#include + +namespace bc_scratch { + +inline constexpr uint32_t kDim = 2048, kMost = 8; // (the largest level a BC3 texture may have; the most layers a run may take) + +/// The layers one run decodes and encodes: RX_VK_BC_LAYERS's value, 1 to kMost; not set, or not a number, 1 +inline uint32_t run_layers(const char* asked) { + return asked ? std::clamp((uint32_t)std::max(atoi(asked), 0), 1u, kMost) : 1u; +} +/// The scratch's layers: a run's, or all of them when full (RX_VK_BC_SCRATCH_FULL) +inline uint32_t layers(uint32_t run, bool full) { return full ? kMost : std::clamp(run, 1u, kMost); } +/// The output buffer's bytes: BC3's 16 bytes a block of 4 x 4, every layer +inline uint64_t out_bytes(uint32_t layers) { return (uint64_t)(kDim / 4) * (kDim / 4) * layers * 16; } +/// What the scratch holds, in MB: the image and its output buffer, and the second image when there is one +inline uint64_t mb(uint32_t layers, bool second) { + const uint64_t image = (uint64_t)kDim * kDim * 4 * layers; + return (image + out_bytes(layers) + (second ? image : 0)) >> 20; +} + +// ---- (Triforce, 11-21, the performance audit's TEX-09, after Source A's) the small scratch ---- +// A run is a decode, an encode, a copy and three full barriers, and one layer a run made a 1365 x 1365 x 90 terrain array +// of 11 levels about 990 of them: the GPU spent most of that time between the runs (Source A, Hyrule Field's entry on an +// RX 9070 XT: 50 ms of GPU an array at one layer a run, 16-17 ms at 8; the lab's own array took 114.8 ms of an RTX 5090 in +// one upload command buffer). Levels of kSmallDim or less each way, the arrays' second level down, go small_layers() layers +// a run through a scratch of their own, kSmallDim x kSmallDim x 8: 32 MB, and the output buffer 8 MB for 4. One layer a +// run was for dark lines between Hyrule Field's ground tiles on an RX 6600, layers after a run's first wrong at the small +// levels, cause never found: so on AMD's cards the default stays 1 until the picture is checked there, and each layer of a +// run is copied by a region of its own, at the encoder's own layer stride. A level of a texture encoded at another size +// keeps the first scratch. RX_VK_BC_SMALL_LAYERS=: the layers a run, 1 to 8; 1: no small scratch, as before. +inline constexpr uint32_t kSmallDim = 1024; + +/// The small scratch's layers, a run's of a small level: RX_VK_BC_SMALL_LAYERS, 1 to kMost; not set, kMost, and 1 on AMD's +/// cards (vendor 0x1002) +inline uint32_t small_layers(const char* asked, uint32_t vendor) { + if (asked && *asked) return std::clamp((uint32_t)std::max(atoi(asked), 0), 1u, kMost); + return vendor == 0x1002 ? 1u : kMost; +} +/// Whether a level decoded at w x h goes through the small scratch: there is one, the level fits it, and the texture is not +/// one encoded at another size (which has the second scratch) +inline bool small_level(uint32_t w, uint32_t h, bool have_small, bool resized) { + return have_small && !resized && w <= kSmallDim && h <= kSmallDim; +} +/// The small scratch's part of the output buffer, and its image, in bytes +inline uint64_t small_out_bytes(uint32_t layers) { return (uint64_t)(kSmallDim / 4) * (kSmallDim / 4) * layers * 16; } +inline uint64_t small_image_bytes(uint32_t layers) { return (uint64_t)kSmallDim * kSmallDim * 4 * layers; } +/// Where layer k of a run begins in the encoder's output: host_shaders/bc_encoder.comp writes block +/// (z * blocks_y + y) * blocks_x + x, 16 bytes a block, 8 where only the colour half is written (an image kept as BC1) +inline uint64_t layer_offset(uint32_t k, uint32_t w, uint32_t h, bool bc1) { + return (uint64_t)k * ((w + 3) / 4) * ((h + 3) / 4) * (bc1 ? 8 : 16); +} + +} // namespace bc_scratch diff --git a/gpu/vk/built_check.h b/gpu/vk/built_check.h new file mode 100644 index 0000000..0e0c022 --- /dev/null +++ b/gpu/vk/built_check.h @@ -0,0 +1,164 @@ +// Whether the caches still have what the light precompile's record says it built (no game, no GPU: the rules; vk_draw.cpp +// built_check_cold, precompile_run, built_play_build). After lib_check.h, which asks the same of the stage libraries, and +// predict_check.h, which asks it of the predicted pipelines. +// +// (Triforce, 11-22, the performance audit's PIPE-21, after Source A's and Source B's.) A light precompile (little RAM: +// precompile_keeps_all) builds the noted pipelines for the caches, lets them go, and records their keys in .built, +// the compute programs' as one key. A later start trusts that record on its stamp alone: what it lists is left to its first +// draw or dispatch, to be built from the caches there. When the caches lost it (the pipeline cache file deleted or refused, +// the play file lost, a driver's own cache cleared or past its cap), each of those first draws compiles in play while the +// record still says built: a draw left out or waiting for a whole build, a dispatch waiting 50 ms and then left out. Source A +// measured a cleared NVIDIA cache: 702,312 draws left out and 21.3 s of draw waits in 140 s of play, against 456 and none +// once everything was built again before the title. +// +// The check. Before the record is trusted, up to kGraphics of the recorded notes (spread over them from another place each +// start) are asked of Triforce's own caches as a draw asks, and up to kComputes of the listed compute programs as a dispatch +// asks, where the device can ask without compiling (FAIL_ON_PIPELINE_COMPILE_REQUIRED): the answer is the caches' own. +// Not a scratch cache of the check's own: AMD's Windows driver keeps no cache of its own (pcache_bound.h), so a question +// put to the driver alone would always be cold there. Without the flag, a pipeline is built twice and timed, as lib_check +// does: the second comes from the cache by construction, and a first over kRatio times it and kSlackMs more was compiled. +// Under kAnswers answers the sample says nothing (the record is trusted, as before); at half of them or more cold it is cold. +// +// Cold: the record is not trusted this start, everything noted and every listed compute program is built again before the +// title (as for a fresh record), and the record is written anew after it. .built.check keeps, for the stamp, that the +// check built them again (kStrikes 0): cold again right after that, the caches do not keep them on this PC (often a driver's +// shader cache that is too small), and they are left to their first draws, as before this check, with a hint in the log. A +// warm sample forgets that. +// +// In play, a counter of what the check is for: recorded pipelines whose first build took kSlowUs or more, and recorded +// compute programs the dispatch's ask did not find, logged at most every kSayMs while it grows. +// +// RX_VK_BUILT_CHECK: 1 by default; 0 no check and no counter, as before; 2 always cold (the lab's and the tests'). +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace built_check { + +inline constexpr int kGraphics = 24; // (recorded notes asked: Source A asks 48, Source B six computes) +inline constexpr int kLook = 2 * kGraphics; // (notes looked at for them: one may have lost its program) +inline constexpr int kComputes = 6; // (listed compute programs asked) +inline constexpr int kAnswers = 16; // (fewer answers say nothing) +inline constexpr double kRatio = 4.0; // (without the flag: a first build against the second, compiled above) +inline constexpr double kSlackMs = 3.0; // (and this much more) +inline constexpr int kStrikes = 0; // (built again once, and cold again right after: left to the draws) +inline constexpr int64_t kSlowUs = 40000; // (a play build of a recorded pipeline over this was compiled) +inline constexpr int64_t kSayMs = 10000; // (the counter's line: at most this often) + +/// RX_VK_BUILT_CHECK: 0 off, 1 on (unset too), 2 always cold +inline int mode(const char* env) { + if (!env || !*env) return 1; + const int v = std::atoi(env); + return v <= 0 ? 0 : v >= 2 ? 2 : 1; +} + +/// One pipeline asked. asked: asked of the caches only; answered_cold: they said a compile is required; made: it was made +/// (both times, not asked). One the driver refuses says nothing of the caches +struct Probe { + bool asked = false, answered_cold = false, made = false; + double first_ms = 0, again_ms = 0; +}; +inline bool counts(const Probe& p) { return p.answered_cold || p.made; } +inline bool cold(const Probe& p) { + return p.answered_cold || (!p.asked && p.made && p.first_ms > kRatio * p.again_ms + kSlackMs); +} + +struct Tally { + int got = 0, cold = 0, answered = 0; +}; +inline Tally tally(const std::vector& probes) { + Tally t; + for (const Probe& p : probes) + if (counts(p)) t.got++, t.cold += cold(p), t.answered += p.answered_cold; + return t; +} + +enum class Verdict { Unknown, Warm, Cold }; +/// Under kAnswers answers nothing; half of them or more cold: cold +inline Verdict verdict(const Tally& t) { + if (t.got < kAnswers) return Verdict::Unknown; + return t.cold * 2 >= t.got ? Verdict::Cold : Verdict::Warm; +} + +/// k places spread over n, the first at seed's place in the first step: each start asks other pipelines +inline std::vector spots(size_t n, size_t k, uint64_t seed) { + std::vector at; + if (!n || !k) return at; + const size_t step = std::max(1, n / k); + for (size_t i = (size_t)(seed % step); i < n && at.size() < k; i += step) at.push_back(i); + return at; +} + +// ---- .built.check: "built-check " + +struct State { + bool found = false; + uint64_t stamp = 0; + int strikes = 0; +}; +inline State state_read(const std::string& path) { + State s; + FILE* f = fopen(path.c_str(), "rb"); + if (!f) return s; + char line[96]{}; + const size_t n = fread(line, 1, sizeof line - 1, f); + fclose(f); + line[n] = 0; + unsigned long long stamp = 0; + int strikes = 0; + if (sscanf(line, "built-check %16llx %d", &stamp, &strikes) == 2 && strikes >= 0) + s.found = true, s.stamp = stamp, s.strikes = strikes; + return s; +} +/// Written whole beside the old one, then put in place (a start that ends in the middle leaves the old file) +inline bool state_write(const std::string& path, uint64_t stamp, int strikes) { + char line[96]; + const int n = snprintf(line, sizeof line, "built-check %016llx %d\n", (unsigned long long)stamp, strikes); + const std::string tmp = path + ".tmp"; + FILE* f = fopen(tmp.c_str(), "wb"); + if (!f) return false; + bool ok = fwrite(line, 1, (size_t)n, f) == (size_t)n; + ok = fclose(f) == 0 && ok; + if (ok) { + remove(path.c_str()); // (rename does not replace on Windows) + ok = rename(tmp.c_str(), path.c_str()) == 0; + } + if (!ok) remove(tmp.c_str()); + return ok; +} + +enum class Do { Nothing, Again, Stop }; +/// What a start does with the verdict, the record of the last starts and the stamp now. always: mode 2, built again at every +/// start whatever was recorded. *next: the strikes to record once they are built again (Again only) +inline Do decide(Verdict v, const State& had, uint64_t stamp, bool always, int* next) { + if (always) return *next = 0, Do::Again; + if (v != Verdict::Cold) return Do::Nothing; + const bool same = had.found && had.stamp == stamp; // (built again for this stamp, and cold again since) + if (same && had.strikes >= kStrikes) return Do::Stop; + *next = same ? had.strikes + 1 : 0; + return Do::Again; +} + +// ---- the counter: play builds of recorded pipelines the caches did not have + +/// Whether a build of that many microseconds was compiled rather than found +inline bool slow(int64_t us) { return us >= kSlowUs; } +/// Whether the counter's line is due: said_ms, when it was last written (0: never) +inline bool say(int64_t now_ms, int64_t said_ms) { return !said_ms || now_ms - said_ms >= kSayMs; } +/// keys sorted without repeats, for has() +inline std::vector sorted(std::vector keys) { + std::sort(keys.begin(), keys.end()); + keys.erase(std::unique(keys.begin(), keys.end()), keys.end()); + return keys; +} +inline bool has(const std::vector& sorted_keys, uint64_t key) { + return std::binary_search(sorted_keys.begin(), sorted_keys.end(), key); +} + +} // namespace built_check diff --git a/gpu/vk/cap_spare.h b/gpu/vk/cap_spare.h new file mode 100644 index 0000000..380fc16 --- /dev/null +++ b/gpu/vk/cap_spare.h @@ -0,0 +1,64 @@ +// (Triforce, 11-22, the performance audit's MEM-33, after Source A's) Which spare capture buffer a capture takes, how +// much the spares and the ones being made ahead may hold together, and a capture's size in steps. +// +// A capture's buffer comes from the spares (retired buffers, and the 32 to 192 MB ones made ahead on a worker), or is +// made new on the game's submitting thread. Three things were so before: +// - the new buffer was made with cap_free_mu held (AMD's driver: about 0.8 ms a MB), and the replay's take, holding +// pools_mu, waits on that mutex to retire a buffer; +// - any spare that fit was taken, so a 2 MB capture could sit in a load's 192 MB spare for the 24 submissions a capture +// is kept, and the load then made its own on the game's thread; +// - a spare made ahead was made once: the one a load took was not made again. +// The rules here are the decisions alone (no Vulkan, no lock): tests/test_capture_spare_reuse.py compiles them, and +// vk_memory.cpp's cap_buf_get, cap_prewarm_class and cap_buf_free ask them. RX_VK_CAP_SPARE_FIT=0: as before. +#pragma once +#include +#include + +namespace cap_spare { + +constexpr uint64_t kMB = 1ull << 20; +constexpr uint64_t kBig = 32 * kMB; // (a load's spare: the classes made ahead start here) +constexpr uint64_t kHeld = 768 * kMB; // (the most the spares hold, whatever the card) +constexpr size_t kMost = 12; // (and the most there are) + +/// A capture of size class `cls` leaves the spare of `size` bytes alone: a big one (32 MB or more) over twice its class. +/// any_size: no spare is left alone (spares capped at 32 MB, the switch off, or the fallback after a failed allocation) +constexpr bool skips(uint64_t size, uint64_t cls, bool any_size) { return !any_size && size >= kBig && size > 2 * cls; } + +/// The spare a capture of `need` bytes (class `cls`) takes of n spares, size_of(i) the i-th one's bytes: the smallest that +/// fits and is not left alone, the first of equals. n: none +template +constexpr size_t pick(size_t n, const SizeOf& size_of, uint64_t need, uint64_t cls, bool any_size) { + size_t best = n; + for (size_t i = 0; i < n; i++) { + const uint64_t s = size_of(i); + if (s >= need && !skips(s, cls, any_size) && (best == n || s < size_of(best))) best = i; + } + return best; +} + +/// Room for one more spare of `size` bytes to be made ahead: with what the spares hold and what is being made (reserved) +/// it stays within what spares may hold (spare_max: the card's, cap_spare_max) and 768 MB +constexpr bool room(uint64_t held, uint64_t reserved, uint64_t size, uint64_t spare_max) { + const uint64_t limit = spare_max < kHeld ? spare_max : kHeld; + return held + reserved <= limit && size <= limit - (held + reserved); +} + +/// A buffer of `size` bytes given back is kept as a spare: fewer than 12 with the ones being made (workers), none over 256 +/// MB (keep_old, RX_VK_CAPTURE_KEEP_OLD: 64), and with the bytes being made (reserved) within the limit. With no workers +/// and nothing reserved this is the rule as it was +constexpr bool keeps(size_t spares, size_t workers, uint64_t held, uint64_t reserved, uint64_t size, uint64_t spare_max, bool keep_old) { + const uint64_t limit = keep_old ? spare_max : (spare_max < kHeld ? spare_max : kHeld); + return spares + workers < kMost && size <= (keep_old ? 64 * kMB : 256 * kMB) && held + reserved <= limit && + size <= limit - (held + reserved); +} + +/// (RX_VK_CAP_STEP_MB=, 0: off) a capture over 16 and up to 32 MB in steps of step_mb MB, not the whole 32 MB class, +/// where spares are capped at 32 MB (a card of 6.5 GB or less). cls: its class as it is (cap_buf_size) +constexpr uint64_t stepped(uint64_t need, uint64_t cls, uint64_t step_mb, bool capped) { + if (!step_mb || !capped || cls != kBig || need <= 16 * kMB || need > kBig) return cls; + const uint64_t step = step_mb * kMB, v = (need + step - 1) / step * step; + return v < cls ? v : cls; +} + +} // namespace cap_spare diff --git a/gpu/vk/capture_tables.h b/gpu/vk/capture_tables.h new file mode 100644 index 0000000..9224e9c --- /dev/null +++ b/gpu/vk/capture_tables.h @@ -0,0 +1,110 @@ +// (Triforce, 11-21, the upscaler audit's UPS-02) The stored draws' two tables, kept from one frame to the next. +// +// While NVIDIA DLSS, AMD FSR 4 or AMD FSR 3.1 upscales, every plain draw into the scene stores its positions and is drawn +// again into the motion vectors (vk_temporal.cpp capture_motion). Two tables go with that, both filled again every frame: +// the draws of a model so far this frame (which of them a draw is: the n-th this frame is the n-th the frame before), and +// the frame before's draws by their key (where each one's positions of that frame are). Both were std::unordered_map, +// emptied and filled each frame: a node from the heap for every entry and back to it a frame later, about 1,400 of each +// a frame in the owner's log of 11-21 ("captured motion ... of 1444 stored a frame ... 0.650 ms CPU a frame"), on the +// thread that records the frame. Here they keep their memory: the count in one open-addressed array (flat_map.h) that is +// emptied in place, the frame before's draws in two lists that take turns, found through such an array. A frame of about +// the size of the one before takes nothing from the heap. +// +// RX_VK_CAPTURE_TABLES=0: std::unordered_map again, a node an entry, as before (kept()). +// +// No Vulkan: tests/test_capture_tables.py compiles this and counts the heap's calls. +#pragma once +#include +#include +#include +#include +#include +#include + +#include "flat_map.h" + +namespace capture_tables { + +/// RX_VK_CAPTURE_TABLES (value, null when unset): kept unless it is 0 +inline bool kept(const char* value) { return !value || !*value || std::atoi(value) != 0; } + +/// An array emptied for the next frame: in place, so the frame fills it without the heap; given back when it is far +/// larger than what the frame put in it (a frame of many draws once, then few), so emptying it does not walk all of that +template +inline void empty_for_next(FlatMap& m) { + if (m.capacity() > 8 * (m.size() + 512)) m.clear(); + else m.reset(); +} + +/// The room a frame's list is given at its start: the frame before's count (kept), so it grows once and not a dozen times +/// from nothing; 0, as before, with the switch off +inline size_t room(size_t taken, bool keep) { return keep ? taken : 0; } + +/// How many of a model's draws came so far this frame +class Nth { +public: + explicit Nth(bool keep = true) : keep_(keep) {} + /// The draws of `model` before this one, which is counted + int next(uint64_t model) { return keep_ ? flat_[model]++ : node_[model]++; } + void clear() { + if (keep_) empty_for_next(flat_); + else node_.clear(); + } + size_t size() const { return keep_ ? flat_.size() : node_.size(); } + +private: + bool keep_; + FlatMap flat_; + std::unordered_map node_; +}; + +/// A frame's entries by key, and the frame before's: turn() makes what was put since the last turn the frame before's, +/// put() notes one of this frame's (the last with a key stands), before() finds the frame before's +template +class Before { +public: + explicit Before(bool keep = true) : keep_(keep) {} + void turn() { + if (!keep_) { + node_old_ = std::move(node_now_); + node_now_.clear(); + return; + } + std::swap(now_, old_), std::swap(now_at_, old_at_); + now_.clear(); // (its memory kept) + empty_for_next(now_at_); + } + void put(uint64_t key, const V& v) { + if (!keep_) { + node_now_[key] = v; + return; + } + uint32_t& at = now_at_[key]; // (the entry's place in now_, plus one: 0 is none yet) + if (at) now_[at - 1] = v; + else now_.push_back(v), at = (uint32_t)now_.size(); + } + /// The frame before's entry of `key`, null when it had none: good until the next turn() or clear() + const V* before(uint64_t key) { + if (!keep_) { + const auto it = node_old_.find(key); + return it == node_old_.end() ? nullptr : &it->second; + } + const auto* s = old_at_.find(key); + return s ? &old_[s->second - 1] : nullptr; + } + /// Nothing of either frame (the chain let go: its memory too) + void clear() { + std::vector().swap(now_), std::vector().swap(old_); + now_at_.clear(), old_at_.clear(); + node_now_.clear(), node_old_.clear(); + } + size_t size() const { return keep_ ? now_.size() : node_now_.size(); } + +private: + bool keep_; + std::vector now_, old_; + FlatMap now_at_, old_at_; + std::unordered_map node_now_, node_old_; +}; + +} // namespace capture_tables diff --git a/gpu/vk/checkpoint_kinds.h b/gpu/vk/checkpoint_kinds.h index ba69584..41ffb12 100644 --- a/gpu/vk/checkpoint_kinds.h +++ b/gpu/vk/checkpoint_kinds.h @@ -23,6 +23,7 @@ inline const char* name(char c) { case 'u': return "upload of texture"; case 'g': return "GPU unswizzle of texture"; case 'w': return "write-back diff pass, chunks"; + case 's': return "mirror syncs' pass, chunks"; // ((Triforce, 11-21, MEM-48)) case 'r': return "recording begun, value"; case 'v': return "uploads of the recording, value"; case 'z': return "recording ended, value"; @@ -45,6 +46,9 @@ inline const char* gap(char c) { return "that pool's write-back copies (its mirror -> guest memory, and the shadow's fill), then the write-back diff pass if " "one follows (w): with no w after it, the end of the recording (z)"; case 'u': case 'g': return "that texture's upload or unswizzle itself"; + case 's': + return "the mirror syncs' pass (one compute dispatch in the recording's uploads, reading a capture's buffer and writing " + "the pool's VRAM mirror by device address; RX_VK_MSYNC_PASS=0: copy regions), then the sync's copies from guest memory"; case 'w': return "the write-back diff pass (one compute dispatch over every pool's chunks, reading mirrors and shadows and writing guest " "memory by device address), then its barrier and the end of the recording (z)"; diff --git a/gpu/vk/commit_room.h b/gpu/vk/commit_room.h new file mode 100644 index 0000000..5c28d80 --- /dev/null +++ b/gpu/vk/commit_room.h @@ -0,0 +1,120 @@ +// (Triforce, 11-22, the performance audit's OS-05, after Source A's) The textures give commit back near Windows' commit limit, +// and the page files' room to grow counts as commit still to come. +// +// Windows charges every block of a discrete card's memory to the process's commit, as it does the game's own memory. Where the +// system's commit runs out (RAM and the page files full) an allocation is refused, and the game ends unless the refusal passes +// (OS-04's wait, HOS_COMMIT_WAIT_MS, helps only if commit comes back). Source A takes the shortfall under a reserve of free +// commit (2,560 MB) off the texture budget on any card, so that eviction and compaction give blocks back. Its first rule fired +// while a page file Windows manages could still grow (292,526 textures retired in 90 minutes, black textures); the fix counts +// each page file's room to grow, capped by its drive's free space, as commit still free. +// +// The rules here, apart from Windows so that tests/test_commit_room.py can run them: +// - reserve_of: RX_VK_TEX_COMMIT_RESERVE_MB, the reserve. Off (0) unless set: the entry's 2560 is not measured on this tree +// - growth_on: RX_VK_TEX_COMMIT_GROWTH, whether the growth is counted: 0 never, another number always, unset where a reserve is +// - low_line, read_growth: the growth is read only while less is free than the higher of the reserve and the line of +// tex_pressure::commit_low (whose fall-back to the fixed size then gets the growth too: vk_texture.cpp commit_now) +// - cap: the textures at most what they hold less the shortfall under the reserve, never under 1 GB (Source A has no floor, +// and its false cut went to 0) +// - growth: the page files' room to grow, from the records of NtQuerySystemInformation's class 0x90 (undocumented: each +// record is checked against the buffer, and one that is not inside it ends the walk), a drive's page files capped together +// by its free space. vk_texture.cpp page_file_growth asks Windows, at most once a second, and answers 0 on any failure. +// Written from the audit's description: Source A's file is not on this PC. +#pragma once + +#include +#include +#include +#include +#include + +namespace commit_room { + +inline constexpr uint64_t kFloor = 1ull << 30; // (the textures are never capped under 1 GB) +inline constexpr int64_t kMostMb = 65536; // (the reserve at most) + +/// RX_VK_TEX_COMMIT_RESERVE_MB in bytes: 0 (unset, empty, no number, 0 or less): no cut, as before; at most kMostMb MB +inline uint64_t reserve_of(const char* env) { + if (!env || !*env) return 0; + const long long mb = std::atoll(env); + return mb <= 0 ? 0 : (uint64_t)std::min(mb, kMostMb) << 20; +} + +/// RX_VK_TEX_COMMIT_GROWTH: unset or empty, counted where a reserve is set; 0 (or no number), never; another number, always +inline bool growth_on(const char* env, uint64_t reserve) { + if (!env || !*env) return reserve != 0; + return std::atoi(env) != 0; +} + +/// tex_pressure::commit_low's line in bytes: want_mb MB; -1 (less than 0) the larger of 4 GB and a tenth of the limit; 0, or a +/// limit not known (0), none +inline uint64_t low_line(uint64_t limit, int64_t want_mb) { + if (!limit || !want_mb) return 0; + return want_mb > 0 ? (uint64_t)want_mb << 20 : std::max(4096ull << 20, limit / 10); +} + +/// Whether the page files' growth is read at this look: counted (on), and less free (avail) than the higher of the reserve and +/// commit_low's line (low) +inline bool read_growth(bool on, uint64_t avail, uint64_t reserve, uint64_t low) { return on && avail < std::max(reserve, low); } + +/// The textures' cap: what they hold (total) less the shortfall of what is free (avail, the growth in it) under the reserve, +/// never under kFloor. UINT64_MAX (no cap) with no reserve, or with the reserve free +inline uint64_t cap(uint64_t total, uint64_t avail, uint64_t reserve) { + if (!reserve || avail >= reserve) return UINT64_MAX; + const uint64_t shortfall = reserve - avail; + return std::max(kFloor, total > shortfall ? total - shortfall : 0); +} + +/// NtQuerySystemInformation's class for the page files' records (SystemPageFileInformationEx) +inline constexpr unsigned kPageFileClass = 0x90; + +/// One record of that class as 64-bit Windows lays it out; sizes in pages +struct PageFileRecord { + uint32_t next; // (bytes from this record to the next; 0: the last) + uint32_t total_pages; // (its size now) + uint32_t in_use_pages, peak_pages; + uint16_t name_bytes, name_room; // (the name: UTF-16, such as \??\C:\pagefile.sys, in the same buffer) + const uint16_t* name; + uint32_t min_pages, max_pages; // (the size it may grow to) +}; +static_assert(sizeof(void*) != 8 || sizeof(PageFileRecord) == 40, "the record as 64-bit Windows lays it out"); + +/// The drive letter of a page file's name ('C' for \??\C:\pagefile.sys), 0 where it has none +inline char drive_of(const uint16_t* name, size_t chars) { + for (size_t i = 0; i + 1 < chars; i++) { + const uint16_t c = name[i]; + if (name[i + 1] == ':' && ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'))) return (char)(c >= 'a' ? c - 32 : c); + } + return 0; +} + +/// The page files' room to grow in bytes, from the records in buf[0, len): a record's (max - size) pages of `page` bytes, the +/// records of one drive summed and capped by that drive's free space (free_of('C'), bytes). A record that is not wholly inside +/// the buffer, whose name is not, or whose next one would overlap it, ends the walk: what came before it counts +template +inline uint64_t growth(const uint8_t* buf, size_t len, uint64_t page, FreeOf free_of) { + uint64_t by_drive[26] = {}; + const uintptr_t base = (uintptr_t)buf; + for (size_t at = 0; len >= sizeof(PageFileRecord) && at <= len - sizeof(PageFileRecord);) { + PageFileRecord r; + std::memcpy(&r, buf + at, sizeof r); + char drive = 0; + if (r.name_bytes) { + const uintptr_t n = (uintptr_t)r.name; + if (n < base || n - base > len || r.name_bytes > len - (n - base)) break; + uint16_t chars[64] = {}; + const size_t count = std::min(r.name_bytes / 2, 64); + std::memcpy(chars, buf + (n - base), count * 2); + drive = drive_of(chars, count); + } + if (drive && r.max_pages > r.total_pages) by_drive[drive - 'A'] += (uint64_t)(r.max_pages - r.total_pages) * page; + if (!r.next) break; + if (r.next < sizeof r || r.next > len - at) break; + at += r.next; + } + uint64_t sum = 0; + for (int d = 0; d < 26; d++) + if (by_drive[d]) sum += std::min(by_drive[d], free_of((char)('A' + d))); + return sum; +} + +} // namespace commit_room diff --git a/gpu/vk/compute_pin.h b/gpu/vk/compute_pin.h new file mode 100644 index 0000000..0e2c7b5 --- /dev/null +++ b/gpu/vk/compute_pin.h @@ -0,0 +1,125 @@ +// Which compute programs are held to 32 lanes (requiredSubgroupSize), by what the program can tell (no game, no GPU: a rule). +// +// (Triforce, 11-22, the performance audit's PIPE-52, after Source A's and Source B's.) The console's programs are written +// for warps of 32 lanes, so every compute pipeline asked for subgroups of 32 (vk_draw.cpp pin_subgroup). Most programs +// cannot tell how many lanes run beside them: they never vote, never read a lane mask or their lane's index, never +// shuffle between lanes. Mesa's Intel driver compiles a pinned program for 32 lanes only (SIMD32), which on its large +// programs is minutes and gigabytes: Source A measured a 730 KB program at 0.8-1 GB over 23-25 s pinned and at most 300 MB +// unpinned (an Iris Xe, Mesa 26.2.4), with 271 of 1046 variants still pinned by this rule. +// +// A program is pinned when it can see its lanes: +// - a vote, a lane mask, the lane's index, a shuffle, FSWZADD or a matrix multiply (Shader::Info's flags); +// - a barrier of Subgroup scope in its module: this translator's own case, the barrier its lowering puts between a +// warp's shared-memory store and load, which takes the subgroup to be the warp. No flag records it, so the module is +// looked at (subgroup_barrier below), and a module nobody looked at counts as having one; +// - shared memory in a workgroup of 32 invocations or fewer, where the workgroup is one warp and its lanes run in step. +// Everything else gets the driver's own choice: no required size, and no full-subgroups flag either. +// +// RX_VK_COMPUTE_PIN, read once: +// all every compute program pinned, as it was (the default on every driver but Mesa's Intel one) +// lanes the rule above (the default on Mesa's Intel driver) +// shared the rule above, and any shared memory pins, whatever the workgroup: for a program whose warps keep +// shared memory in step in a workgroup over 32, which nothing else here sees +// A doubt pins: an unknown value of the switch is the driver's default, a program without a stage is pinned. +// The graphics stages and the mesh fallback's compute stages are not asked here: they stay with pin_subgroup. +#pragma once +#include +#include +#include + +namespace compute_pin { +enum class Mode { All, Lanes, Shared }; + +/// What one compute program is, as the rule needs it +struct Facts { + bool has_stage = false; // (false: nothing known, pinned) + bool vote = false; // Shader::Info::uses_subgroup_vote + bool mask = false; // uses_subgroup_mask + bool lane_index = false; // uses_subgroup_invocation_id + bool shuffles = false; // uses_subgroup_shuffles + bool fswzadd = false; // uses_fswzadd + bool matrices = false; // uses_cooperative_matrix + bool subgroup_barrier = true; // its module has an OpControlBarrier of Subgroup scope (true until a look says no) + uint32_t shared_memory = 0; // bytes of shared memory + uint64_t invocations = 0; // the workgroup's x * y * z +}; + +/// The program can tell how many lanes run in its subgroup, or which one it is +constexpr bool sees_lanes(const Facts& f) { + return f.vote || f.mask || f.lane_index || f.shuffles || f.fswzadd || f.matrices || f.subgroup_barrier; +} + +/// RX_VK_COMPUTE_PIN's value (nullptr: unset) on this driver +inline Mode mode_for(const char* env, bool mesa_intel) { + if (env && !std::strcmp(env, "all")) return Mode::All; + if (env && !std::strcmp(env, "lanes")) return Mode::Lanes; + if (env && !std::strcmp(env, "shared")) return Mode::Shared; + return mesa_intel ? Mode::Lanes : Mode::All; +} + +inline const char* name(Mode m) { + return m == Mode::All ? "all" : m == Mode::Lanes ? "lanes" : "shared"; +} + +/// The program's pipeline asks for subgroups of 32 +constexpr bool needed(Mode m, const Facts& f) { + if (m == Mode::All || !f.has_stage) return true; + if (sees_lanes(f)) return true; + if (f.shared_memory) return m == Mode::Shared || f.invocations <= 32; + return false; +} + +/// Whether a SPIR-V module's entry point is a compute shader's (OpEntryPoint with the GLCompute model): the only modules +/// the rule is asked about, so the only ones worth a note. One that does not read as SPIR-V: no +inline bool compute_entry(const uint32_t* code, size_t words) { + constexpr uint32_t kMagic = 0x07230203, kOpEntryPoint = 15, kOpFunction = 54, kGLCompute = 5; + if (!code || words < 5 || code[0] != kMagic) return false; + for (size_t at = 5; at < words;) { + const uint32_t count = code[at] >> 16, op = code[at] & 0xffffu; + if (!count || at + count > words || op == kOpFunction) return false; + if (op == kOpEntryPoint && count >= 2) return code[at + 1] == kGLCompute; + at += count; + } + return false; +} + +/// Whether a SPIR-V module has an OpControlBarrier whose execution scope is the constant Subgroup (3). A module that does +/// not read as SPIR-V counts as having one. words: the module's 32-bit words +inline bool subgroup_barrier(const uint32_t* code, size_t words) { + constexpr uint32_t kMagic = 0x07230203, kOpConstant = 43, kOpControlBarrier = 224, kScopeSubgroup = 3; + if (!code || words < 5 || code[0] != kMagic) return true; + // (the scopes are constants, and constants come before the functions: one pass for the barriers' scope ids, kept small, + // and their values looked up at the end) + constexpr size_t kMost = 64; + uint32_t scopes[kMost]; + size_t n_scopes = 0; + for (size_t at = 5; at < words;) { + const uint32_t count = code[at] >> 16, op = code[at] & 0xffffu; + if (!count || at + count > words) return true; + if (op == kOpControlBarrier && count >= 4) { + const uint32_t id = code[at + 1]; + bool seen = false; + for (size_t i = 0; i < n_scopes; i++) seen = seen || scopes[i] == id; + if (!seen) { + if (n_scopes == kMost) return true; // (more scope ids than a module has: a doubt) + scopes[n_scopes++] = id; + } + } + at += count; + } + if (!n_scopes) return false; + size_t resolved = 0; + for (size_t at = 5; at < words;) { + const uint32_t count = code[at] >> 16, op = code[at] & 0xffffu; + if (op == kOpConstant && count >= 4) { + for (size_t i = 0; i < n_scopes; i++) + if (scopes[i] == code[at + 2]) { + if (code[at + 3] == kScopeSubgroup) return true; + resolved++; + } + } + at += count; + } + return resolved != n_scopes; // (a scope that is no plain constant: a doubt) +} +} // namespace compute_pin diff --git a/gpu/vk/cpu_count.h b/gpu/vk/cpu_count.h new file mode 100644 index 0000000..b5f4d52 --- /dev/null +++ b/gpu/vk/cpu_count.h @@ -0,0 +1,41 @@ +// (Triforce, 11-22, the performance audit's OS-49, after Source B's) The logical processors the renderer sizes its pools and +// its threads' counts by: std::thread::hardware_concurrency, or, while the lab's HOS_SIM_CPU=/ holds the +// process to fewer of the PC's cores (host/hos_sim_cpu.h), its : an affinity mask alone does not change what +// hardware_concurrency answers, and the pools would be a 24-thread PC's on 6 cores. The host clears the variable when it did +// not apply it, so the two agree. Windows only, as the host's part; unset, hardware_concurrency as before. Every such count in +// gpu/ goes through cpu_count() but vk_shader.cpp's (a file the shader cache's stamp is made from): +// tests/test_sim_cores.py checks that, and the rule. +#pragma once +#include +#include + +namespace cpu_count_rule { +/// The threads HOS_SIM_CPU asks for ("6/12": 12; "6": 6), when fewer than the PC's hw (0: not known); else 0: not set, not +/// "[/]" with each from 1 to 256 and threads not under cores, or not fewer +inline unsigned sim_threads(const char* e, unsigned hw) { + if (!e || !*e) return 0; + char* end = nullptr; + const long c = std::strtol(e, &end, 10); + if (end == e) return 0; + long t = c; + if (*end == '/') { + const char* s = end + 1; + t = std::strtol(s, &end, 10); + if (end == s) return 0; + } + if (*end || c < 1 || c > 256 || t < c || t > 256) return 0; + return hw && (unsigned)t >= hw ? 0 : (unsigned)t; +} +} // namespace cpu_count_rule + +/// The logical processors to size by (read once) +inline unsigned cpu_count() { + static const unsigned n = [] { + const unsigned hw = std::thread::hardware_concurrency(); +#ifdef _WIN32 + if (const unsigned sim = cpu_count_rule::sim_threads(std::getenv("HOS_SIM_CPU"), hw)) return sim; +#endif + return hw; + }(); + return n; +} diff --git a/gpu/vk/dead_count.h b/gpu/vk/dead_count.h new file mode 100644 index 0000000..99fa6ee --- /dev/null +++ b/gpu/vk/dead_count.h @@ -0,0 +1,39 @@ +// (Triforce, 11-21, the performance audit's THREAD-21, its first step) What destroying the dead images costs a present. +// +// A texture let go waits in a list until the GPU is done with it, and texture_frame then destroys every one that is, in +// the middle of the present being recorded: a render target's image goes to the pool (cheap), every other image is +// destroyed with its views. The audit's port bounds that to a share a present, after the present's submit, as Source B +// does on its GPU thread's idle moments. Its own proof asks for the cost first: "if no present spends 3 ms in the loop, +// stop". This is that count, and nothing else changes: a line for a present whose loop took kSayMs or more, and every +// 600 presents the totals and the worst present. +// +// No Vulkan here: tests/test_dead_count.py compiles it. +#pragma once +#include + +namespace dead_count { + +inline constexpr double kSayMs = 2.0; // (a present's loop that gets a line of its own) +inline constexpr double kWorthMs = 3.0; // (the audit's line: under it in every present, the bounded step is not built) +inline constexpr uint64_t kReport = 600; // (presents between the totals' lines) + +struct Counts { + uint64_t presents = 0, destroyed = 0, pooled = 0, over_say = 0, over_worth = 0; + double ms = 0, worst_ms = 0; + uint64_t worst_destroyed = 0, worst_left = 0; + /// A present's loop: n_destroyed images destroyed and n_pooled put in the pool in loop_ms, left still waiting. True + /// when the present gets a line of its own + bool note(uint64_t n_destroyed, uint64_t n_pooled, double loop_ms, uint64_t left) { + presents++, destroyed += n_destroyed, pooled += n_pooled, ms += loop_ms; + if (loop_ms > worst_ms) worst_ms = loop_ms, worst_destroyed = n_destroyed, worst_left = left; + if (loop_ms >= kWorthMs) over_worth++; + if (loop_ms < kSayMs) return false; + over_say++; + return true; + } + /// Whether the totals' line is due, at present `present` + bool due(uint64_t present) const { return presents != 0 && present % kReport == 0; } + void reset() { *this = Counts{}; } +}; + +} // namespace dead_count diff --git a/gpu/vk/desc_counts.h b/gpu/vk/desc_counts.h new file mode 100644 index 0000000..41bb312 --- /dev/null +++ b/gpu/vk/desc_counts.h @@ -0,0 +1,67 @@ +// A program's descriptor counts kept with the program (no game, no GPU: a rule; vk_shader.h Program::desc_n, vk_draw.cpp +// build_descriptors). +// +// (Triforce, 11-22, the performance audit's DESC-22, after Source A's.) build_descriptors sums the program's buffers, +// images and texel views at every build, six lists a stage, only to size three reserve() calls that keep the pointers +// it hands the writes stable. The sums are fixed once the program is translated: its stages are filled before it is +// ready and never after. So the three are packed into one word kept on the program, stored at its first build once +// ready and read at every later one; a word that is not a count walks as before. +// +// The word is 32 bits because that is what the program has room for: the padding after Program::ready. The struct must +// not grow: sizeof(Program) is in the program cache's stamp, and a change of it throws every player's caches away. So +// three counts of 10 bits; a program with more than 1023 of a kind is not kept (kUnset is what its pack gives) and walks +// at every build, as before. +// +// A short count would let a vector move in the middle of a build and leave the writes with stale pointers. It cannot +// come from the walk that stores it, but the build checks all the same that none of the three moved, and a program +// whose did is marked kNever, walked from then on, and built again at once. +// +// RX_VK_DESC_COUNTS=0: the walk at every build, as before. The lab's knob "descwalk" does the same in its phases. +#pragma once +#include +#include +#include + +namespace mkwvk::desc_counts { + +inline constexpr uint32_t kUnset = ~0u; // not counted yet, or too many of a kind to pack +inline constexpr uint32_t kNever = 0xFFFFFFFEu; // its kept counts were short once: walked at every build from then on +inline constexpr uint32_t kMost = 1023; // the most of a kind a word holds (10 bits) + +/// RX_VK_DESC_COUNTS: on unless set to 0 +inline bool on(const char* env) { + return !env || std::atoi(env) != 0; +} + +/// The three counts as one word, or kUnset where one does not fit +constexpr uint32_t pack(size_t bufs, size_t imgs, size_t views) { + if (bufs > kMost || imgs > kMost || views > kMost) return kUnset; + return (uint32_t)bufs | (uint32_t)imgs << 10 | (uint32_t)views << 20; +} + +/// Whether the word holds counts (its two top bits are clear: kUnset and kNever have them set) +constexpr bool holds(uint32_t word) { + return (word >> 30) == 0; +} + +/// The counts of a word that holds them; false, and nothing written, for one that does not (walk) +constexpr bool unpack(uint32_t word, size_t* bufs, size_t* imgs, size_t* views) { + if (!holds(word)) return false; + *bufs = word & kMost, *imgs = (word >> 10) & kMost, *views = (word >> 20) & kMost; + return true; +} + +/// Whether a build that walked stores its counts: the rule on, the program translated (its stages do not change after), +/// and the word still unset (never over kNever) +constexpr bool stores(bool on, bool ready, uint32_t word) { + return on && ready && word == kUnset; +} + +/// The guard at a build's end: whether one of the three vectors' storage moved since the reserve (before/after: their +/// data() pointers), in a build that went by kept counts. A build that walked reserved exactly, as it always did +constexpr bool moved(bool walked, const void* bufs0, const void* bufs1, const void* imgs0, const void* imgs1, const void* views0, + const void* views1) { + return !walked && (bufs0 != bufs1 || imgs0 != imgs1 || views0 != views1); +} + +} // namespace mkwvk::desc_counts diff --git a/gpu/vk/desc_pool_grow.h b/gpu/vk/desc_pool_grow.h new file mode 100644 index 0000000..96e9688 --- /dev/null +++ b/gpu/vk/desc_pool_grow.h @@ -0,0 +1,97 @@ +// A recording slot's descriptor pool, made at half size and made whole once a recording runs out of it (no game, no GPU: +// a rule; vk_draw.cpp make_pool, slot_reset_locked, draw_begin_recording). +// +// (Triforce, 11-22, the performance audit's DESC-33, after Source A's.) Each of the recording slots (48 at most) keeps a +// descriptor pool of 8192 sets and up to 32768 descriptors a type, and the driver may keep host memory for all of that, +// 48 times. Few recordings come near it: Source A saw one take at most 289 sets on a desktop card, and no pool at half +// size run out in three scenes on a handheld. So a slot's pool starts at half (4096 sets, 16384 uniform buffers and +// combined image samplers, and so on down the types). +// +// A recording that runs out is given a whole extra pool, as it always was (bind_descriptors), and that is the mark: at +// the slot's next reset, where the extra pools are destroyed anyway, a slot that has any and whose own pool is not whole +// has it destroyed and made again whole. From then on the slot is as it was before this rule. A busy slot so pays one +// extra pool, once. +// +// RX_VK_DESC_POOL_GROW=0: whole pools from the start, as before. RX_VK_DESC_POOL_DIV= still divides both sizes. +// What the recordings took (sets and descriptors a type, the most one recording took of each) and how many pools were +// added and slots made whole go into the log's "recording streams" line, so a size can be judged from a run's log. +#pragma once +#include +#include +#include + +namespace mkwvk::desc_pool_grow { + +/// RX_VK_DESC_POOL_GROW: on unless set to 0 +inline bool on(const char* env) { + return !env || std::atoi(env) != 0; +} + +/// A whole pool's sets, and its descriptors by type in the order of Took::n +inline constexpr uint32_t kSets = 8192; +enum Kind { kUniform, kStorage, kSampled, kStorageImage, kUniformTexel, kStorageTexel, kKinds }; +inline constexpr uint32_t kWhole[kKinds] = {32768, 16384, 32768, 4096, 2048, 2048}; + +/// The shift a new slot's pool is made with: half with the rule on, whole without it +constexpr uint32_t first_shift(bool on) { + return on ? 1u : 0u; +} + +/// A count of a pool made at `shift` (0 whole, 1 half) and divided by RX_VK_DESC_POOL_DIV's n (1..32): never 0 +constexpr uint32_t count(uint32_t whole, uint32_t shift, uint32_t div) { + const uint32_t n = (whole >> (shift > 8 ? 8 : shift)) / (div ? div : 1); + return n ? n : 1; +} + +/// Whether a slot's own pool is made again whole at its reset: its last recording ran out (it has extra pools) and the +/// pool is not whole yet +constexpr bool remake(uint32_t shift, size_t extra_pools) { + return shift != 0 && extra_pools != 0; +} + +/// Which of the six a Vulkan descriptor type is (VkDescriptorType's numbers, so that this header needs no Vulkan): +/// kKinds for any other +constexpr int kind(uint32_t vk_type) { + switch (vk_type) { + case 6: return kUniform; // VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER + case 7: return kStorage; // VK_DESCRIPTOR_TYPE_STORAGE_BUFFER + case 1: return kSampled; // VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER + case 3: return kStorageImage; // VK_DESCRIPTOR_TYPE_STORAGE_IMAGE + case 4: return kUniformTexel; // VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER + case 5: return kStorageTexel; // VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER + default: return kKinds; + } +} + +/// What one recording took of its slot's pools: sets, and descriptors by type +struct Took { + uint32_t sets = 0; + uint32_t n[kKinds] = {}; +}; + +/// One more set allocated, with these writes (W: VkWriteDescriptorSet, or a test's own with the same two fields) +template +inline void took(Took& t, const W* writes, size_t count_of) { + t.sets++; + for (size_t i = 0; i < count_of; i++) { + const int k = kind((uint32_t)writes[i].descriptorType); + if (k < kKinds) t.n[k] += (uint32_t)writes[i].descriptorCount; + } +} + +/// The most any recording took of each, raised by one recording's +inline void peak(Took& most, const Took& t) { + if (t.sets > most.sets) most.sets = t.sets; + for (int k = 0; k < kKinds; k++) + if (t.n[k] > most.n[k]) most.n[k] = t.n[k]; +} + +/// Whether what a recording took fits a pool made at `shift` and divided by div (the tests', and the log's reader's) +constexpr bool fits(const Took& t, uint32_t shift, uint32_t div) { + if (t.sets > count(kSets, shift, div)) return false; + for (int k = 0; k < kKinds; k++) + if (t.n[k] > count(kWhole[k], shift, div)) return false; + return true; +} + +} // namespace mkwvk::desc_pool_grow diff --git a/gpu/vk/dlss_checks.h b/gpu/vk/dlss_checks.h index 019aee3..341672c 100644 --- a/gpu/vk/dlss_checks.h +++ b/gpu/vk/dlss_checks.h @@ -4,6 +4,7 @@ #pragma once #include +#include #include #include @@ -16,6 +17,50 @@ inline bool scene_size_fits(uint32_t w, uint32_t h, uint32_t ow, uint32_t oh) { return w && h && ow && oh && w <= ow && h <= oh && (uint64_t)w * 3 >= ow && (uint64_t)h * 3 >= oh; } +/// ((Triforce, 11-22, the performance audit's PRESENT-38, after Source A's) One quality mode's range of render sizes for +/// an output, as NGX answers it (ok: it answered one). +struct ModeRange { + uint32_t min_w = 0, min_h = 0, max_w = 0, max_h = 0; + bool ok = false; +}; +/// How far outside a range a render size may be and still be that mode's: a 64th of the range's largest size on that +/// axis, 2 pixels at least. The Render resolution row's shares are twelfths of the output rounded up, NGX's own sizes +/// are rounded down: 854x480 against NGX's 853x480 for Ultra Performance at 2560x1440. +constexpr uint32_t mode_slack(uint32_t largest) { return largest / 64 > 2 ? largest / 64 : 2; } +/// The mode (an index into ranges) that takes a render size of w x h for an output of ow x oh, or -1. The first range +/// that holds the size; else, with slack on, the first it is outside of by no more than mode_slack on each axis, and +/// only for a size DLSS scales at all (scene_size_fits: the output 1 to 3 times it). *widened: taken by the slack. +inline int mode_for(const ModeRange* ranges, size_t n, uint32_t w, uint32_t h, uint32_t ow, uint32_t oh, bool slack, bool* widened) { + *widened = false; + for (size_t i = 0; i < n; i++) { + const ModeRange& r = ranges[i]; + if (r.ok && w >= r.min_w && h >= r.min_h && w <= r.max_w && h <= r.max_h) return (int)i; + } + if (!slack || !scene_size_fits(w, h, ow, oh)) return -1; + for (size_t i = 0; i < n; i++) { + const ModeRange& r = ranges[i]; + if (!r.ok) continue; + const uint32_t sw = mode_slack(r.max_w), sh = mode_slack(r.max_h); + if (w + sw >= r.min_w && h + sh >= r.min_h && w <= r.max_w + sw && h <= r.max_h + sh) return *widened = true, (int)i; + } + return -1; +} + +/// ((Triforce, 11-22, the performance audit's PRESENT-32, after Source A's) Whether a feature is made in a one-size mode +/// (NGX's DLAA mode: vk_dlss.cpp prepare's `same` list): its render size is the output's. Before, whenever the two sizes +/// were equal. NVIDIA DLSS with the Render resolution row at Auto starts at the output's size, so it got a one-size +/// feature, and the game's dynamic resolution left that size some frames later: a second feature made in play (Source +/// A's save load-in: 119 ms, then 29.5 ms 17 frames on). With the rule (rule: RX_VK_DLSS_PREWARM=1), one-size only where +/// NVIDIA DLAA is the Upscaler's choice (dlaa); DLSS at the output's size takes the scaled mode whose range holds it +/// (mode_for: Quality's reaches the output's size), and Auto's smaller sizes then draw with that same feature. Off by +/// default: DLSS at full size then draws with the scaled mode's network, not DLAA's, a picture nobody has compared yet +inline bool dlaa_feature(bool rule, bool dlaa, uint32_t rw, uint32_t rh, uint32_t ow, uint32_t oh) { + const bool full = rw == ow && rh == oh; + return rule ? full && dlaa : full; +} +/// RX_VK_DLSS_PREWARM (value, null when unset): the rule above is on only when it is 1 +inline bool prewarm_rule(const char* text) { return text && *text && std::atoi(text) == 1; } + /// Whether s is all ASCII: NGX's Linux library takes only such paths (vk_dlss.cpp wide()) inline bool ascii(std::string_view s) { for (const char c : s) @@ -84,4 +129,28 @@ inline std::wstring utf8_to_wide(std::string_view s, bool* bad = nullptr) { return out; } +/// ((Triforce, 11-21, the performance audit's PRESENT-37, after Source A's) The preset DLSS draws with while the DLSS model +/// row is stepped through (vk_dlss.cpp prepare). The row goes K, M, L, NVIDIA's default and applies live, so K to L passes +/// M: a feature was made for each preset passed, tens of ms of CPU each, and NGX keeps a released feature's video memory for +/// some seconds: Source A's 16 GB card at 3440x1440 held 1419 MB of DLSS's for K, M and L at once against 813 MB for L +/// alone, and where memory was short NGX refused the next feature. A new preset is taken once the row has held it for +/// `frames`; until then the feature already made draws on with its own. With no feature (the first, or one let go) the +/// row's preset is taken at once +struct PresetSettle { + unsigned wanted = ~0u; // (the row's preset, as last seen) + uint64_t since = 0; // (the frame it was first seen in) + /// The preset for frame `now`: want, the row's; have, the feature's; has: there is a feature + unsigned pick(unsigned want, unsigned have, bool has, uint64_t now, uint64_t frames) { + if (want != wanted) wanted = want, since = now; + if (!has || want == have) return want; + return now - since >= frames ? want : have; + } +}; +/// RX_VK_DLSS_MODEL_SETTLE's frames: 30 when not set; 0 (and what is no number, or less): at once, as before +inline uint64_t settle_frames(const char* text) { + if (!text || !*text) return 30; + const long v = std::atol(text); + return v > 0 ? (uint64_t)v : 0; +} + } // namespace mkwvk::dlss_checks diff --git a/gpu/vk/dlss_feature.h b/gpu/vk/dlss_feature.h new file mode 100644 index 0000000..c0cd080 --- /dev/null +++ b/gpu/vk/dlss_feature.h @@ -0,0 +1,58 @@ +// (Triforce, 11-21, the upscaler audit's UPS-01) The render size NVIDIA DLSS's feature is made for. +// +// vk_dlss.cpp made every feature for the largest render size its quality mode allows, so that the game's dynamic +// resolution could change the render size under it without the feature being made again. NGX answers that largest size as +// the output's own (the owner's log of 11-21: "feature 1 made ...: 2944x1243 (2208x932 to 4416x1864) -> 4416x1864, mode +// 2, preset M"), so with the Render resolution row at a fixed share, where the render size never changes, the feature was +// still made for 2.25 times the pixels it was ever given at Quality, 4 times at Performance and 9 at Ultra Performance, +// and the chain's three input images with it. NVIDIA's guide (DLSS Programming Guide 310.6.0, 3.2 and 5.2.8) has the +// optimal render size passed to the creation "exactly as given", and says of the second generation transformer's presets +// L and M that their cost follows the size the feature is initialized for, which is why it is to be the dynamic range's +// largest only where there is a dynamic range. The same log has DLSS's own video memory at 1069 MB for that feature, where +// NVIDIA's table has 456 MB for preset M at 4K. +// +// So: a fixed share gets a feature of its own render size (exact), as AMD FSR 4 gets a context of its own +// (temporal_rules.h plan_context). Where the render size is the game's to change (the row's Auto), or once it has changed +// under one output in this run, the feature is made for the mode's whole range, as before. DLAA's is the output's either +// way. +// +// RX_VK_DLSS_FEATURE_MAX=1: every feature for its mode's largest render size, as before (whole_range()). +// +// No NGX, no Vulkan: tests/test_dlss_feature.py compiles this. +#pragma once +#include +#include + +namespace dlss_feature { + +/// RX_VK_DLSS_FEATURE_MAX (value, null when unset): not 0, every feature for its mode's largest render size, as before +inline bool whole_range(const char* value) { return value && *value && std::atoi(value) != 0; } + +/// What a feature is made for: w x h, and whether that is the one render size it takes (exact) or the largest of a range +struct Size { + uint32_t w, h; + bool exact; +}; + +/// The size to make a feature for, for a frame of rw x rh in a mode whose render sizes go up to max_w x max_h. aa: DLAA, +/// the render size the output's (its range is that one size). range: the render size may change under this output (the +/// row's Auto, the size seen changing, or the switch) +inline Size size(bool aa, bool range, uint32_t rw, uint32_t rh, uint32_t max_w, uint32_t max_h) { + if (aa || range || rw > max_w || rh > max_h) return {max_w, max_h, false}; + return {rw, rh, true}; +} + +/// Whether a feature made for fw x fh takes a frame of rw x rh: its one size (exact), or any from min_w x min_h up to its own +inline bool takes(bool exact, uint32_t fw, uint32_t fh, uint32_t min_w, uint32_t min_h, uint32_t rw, uint32_t rh) { + if (exact) return rw == fw && rh == fh; + return rw <= fw && rh <= fh && rw >= min_w && rh >= min_h; +} + +/// Whether this frame shows that the render size changes under one output: a feature of one size (exact) for the same +/// output, preset and colour (same), and a frame of another size. The caller keeps it for the run: the next feature, and +/// every one after it, is made for the mode's whole range +inline bool found_range(bool same, bool exact, uint32_t fw, uint32_t fh, uint32_t rw, uint32_t rh) { + return same && exact && (rw != fw || rh != fh); +} + +} // namespace dlss_feature diff --git a/gpu/vk/dlss_math.h b/gpu/vk/dlss_math.h index 24ea320..d975114 100644 --- a/gpu/vk/dlss_math.h +++ b/gpu/vk/dlss_math.h @@ -93,6 +93,36 @@ inline void unjitter(float m[16], const float jitter[2]) { } } +/// ((Triforce, 11-20) the owner's "DLSS flicker": the whole picture and the shadows, while the camera moves, with NVIDIA +/// DLSS frame generation on, and not without it.) Streamline's camera matrices are in the clip space of the depth buffer it +/// is given: its depth value is that clip space's z over w, 1 at the near plane with depthInverted. The game's projection +/// is NVN's, clip z from -1 at the near plane to 1 at the far one, and the depth frame generation gets is FSR's copy +/// (vk_fsr3_inputs.cpp): d = (1 - ndc z) / 2 where the game's buffer has 0 at the near plane (flipped: Camera::flip_depth), +/// (ndc z + 1) / 2 where it is reversed already. Given the game's own matrices with that depth, a depth read as clip z put +/// what is far a fifth of a metre from the camera and what is near at the far plane, so whatever Streamline works out from +/// the depth and the camera's move was wrong as soon as the camera moved, and right while it stood. Only DLSS frame +/// generation takes matrices (FSR takes the near and far planes and the field of view), so nothing else showed it. +/// The depth's clip space is the game's with z' = (w - z) / 2 (flipped) or (w + z) / 2: Z * clip, Z the identity but for +/// its z row, (0 0 -1/2 1/2) or (0 0 1/2 1/2). +/// A projection (or a view-projection) into the depth's clip space: Z * m, row major (clip = M * position) +inline void depth_clip_projection(const float m[16], bool flipped, float out[16]) { + const float s = flipped ? -0.5f : 0.5f; + for (int i = 0; i < 16; i++) out[i] = m[i]; + for (int c = 0; c < 4; c++) out[2 * 4 + c] = s * m[2 * 4 + c] + 0.5f * m[3 * 4 + c]; +} +/// A clip-to-clip matrix of the game's (this frame's clip space to the frame before's: reprojection) between the depth's +/// clip spaces: Z * m * inverse(Z), inverse(Z)'s z row (0 0 -2 1) or (0 0 2 -1). The identity stays the identity +inline void depth_clip_reprojection(const float m[16], bool flipped, float out[16]) { + const float k = flipped ? -2.0f : 2.0f, t = flipped ? 1.0f : -1.0f; + float r[16]; + for (int row = 0; row < 4; row++) { // (m * inverse(Z): the z column scaled, and added to the w column) + r[row * 4 + 0] = m[row * 4 + 0], r[row * 4 + 1] = m[row * 4 + 1]; + r[row * 4 + 2] = k * m[row * 4 + 2]; + r[row * 4 + 3] = m[row * 4 + 3] + t * m[row * 4 + 2]; + } + depth_clip_projection(r, flipped, out); // (Z * that) +} + /// A jitter in clip space as DLSS takes it: in render pixels, x right and y down (clip y is up) inline void jitter_to_pixels(const float clip[2], unsigned width, unsigned height, float out[2]) { out[0] = clip[0] * 0.5f * (float)width; diff --git a/gpu/vk/dmabuf_gate.h b/gpu/vk/dmabuf_gate.h new file mode 100644 index 0000000..ae364ee --- /dev/null +++ b/gpu/vk/dmabuf_gate.h @@ -0,0 +1,38 @@ +// (Triforce, 11-22, the performance audit's OS-08, after Source A's) Whether guest memory is offered to the driver as +// udmabuf dma-bufs at all, and what the start's line says (vk_memory.cpp import_dma_buf asks at its first call; plain code: +// tests/test_dmabuf_driver_gate.py runs it anywhere). +// +// On Linux the guest window is a memfd and each pool is first offered to the driver as a dma-buf made by udmabuf. NVIDIA's +// proprietary driver refuses every one (VK_ERROR_OUT_OF_DEVICE_MEMORY) and the pool ends as a host-pointer import anyway: +// 49-50 refused imports a run and a kernel message at each start on Source A's RTX 4090, the first refusal 2.5-3.7 ms, one +// of a 64 MB piece 198 ms on the presentation thread during a save load. So with that driver /dev/udmabuf is not opened +// and the pools are host-pointer imports from the start. NVK and RADV have other driver ids and keep their dma-bufs. +// RX_VK_DMABUF_NVIDIA=1: tried on NVIDIA's driver too, as before (for a later driver that takes them). +#pragma once + +namespace dmabuf_gate { + +/// What the start found. fn: the host gave its memfd function (false also with RX_VK_NO_DMABUF). memfd: the window is a +/// memfd. ext: the device imports dma-bufs (VK_EXT_external_memory_dma_buf and its fd properties call). nvidia: NVIDIA's +/// proprietary driver, and RX_VK_DMABUF_NVIDIA not set +struct Facts { + bool fn = false, memfd = false, ext = false, nvidia = false; +}; + +/// Whether /dev/udmabuf is opened: only then can a pool be offered as a dma-buf +inline bool try_udmabuf(const Facts& f) { return f.fn && f.memfd && f.ext && !f.nvidia; } + +/// As it was: NVIDIA's driver was offered them too +inline bool try_udmabuf_before(const Facts& f) { return f.fn && f.memfd && f.ext; } + +/// The start's "vk: guest memory as dma-bufs: ..." answer. opened: /dev/udmabuf opened (asked only after try_udmabuf) +inline const char* answer(const Facts& f, bool opened) { + if (try_udmabuf(f) && opened) return "yes"; + if (!f.fn) return "no (no memfd window, or RX_VK_NO_DMABUF)"; + if (!f.memfd) return "no (the window is anonymous memory)"; + if (!f.ext) return "no (VK_EXT_external_memory_dma_buf)"; + if (f.nvidia) return "no (NVIDIA's driver refuses them: host pointers)"; + return "no (/dev/udmabuf cannot be opened)"; +} + +} // namespace dmabuf_gate diff --git a/gpu/vk/drawn_page_map.h b/gpu/vk/drawn_page_map.h new file mode 100644 index 0000000..41a15f7 --- /dev/null +++ b/gpu/vk/drawn_page_map.h @@ -0,0 +1,41 @@ +// (Triforce, 11-21, the performance audit's TEX-21, after Source A's) A texture the GPU drew gets its page map at its first +// draw. +// +// A texture in a virtual pool (its storage mapped 64 KB a page) is checked through its mapping: texture_dirty keeps each +// page's guest address (page_map) and says which pages were mapped anew or written since. The map is recorded at the +// texture's first check, which for a texture uploaded from memory is its first upload. One the GPU drew first is marked +// uploaded by the draw and never checked, so it has no map; once the game finalizes it, its first check finds none, takes +// that as "everything was mapped since", and the whole storage is uploaded from memory over what the GPU drew: a HUD +// label or a render target gone blank after an area change, and an upload of up to 15 MB each. Source A's sessions from +// Hyrule Field to Castle Town: such whole uploads of HUD labels 6, other HUD 4 to 6, render targets 4 to 9, all to 0. +// Here: the map is recorded at the first draw that says who drew it, by one walk of the mapping (the storage's size / 64 KB +// lookups, once a texture). The later check then gives only the pages mapped anew or written by the CPU. +// Not for a write that gives no reason ("?": a clear, the movie upscaler): those keep the path that brings a font cache's +// letters back from memory. RX_VK_DRAWN_PAGE_MAP=0: no map at the draw, as before. +// +// The rule alone: tests/test_drawn_page_map.py compiles it, and texture_dirty cut from the source. +#pragma once +#include +#include +#include +#include + +namespace drawn_page_map { + +inline constexpr uint64_t kVirtual = 1ull << 40; // (a virtual pool's addresses start here) +inline constexpr uint64_t kPage = 1ull << 16; // (and its pages are 64 KB) + +/// RX_VK_DRAWN_PAGE_MAP: off only with 0 +inline bool on(const char* asked) { return !asked || !*asked || atoi(asked) != 0; } + +/// The pages of a storage of guest_size bytes +constexpr size_t pages(uint64_t guest_size) { return (size_t)((guest_size + kPage - 1) / kPage); } + +/// Whether a write by `why` into a texture at gpu, of guest_size bytes of storage, whose map holds map_pages pages now, +/// records its map first +inline bool seeds(bool switch_on, const char* why, uint64_t gpu, uint64_t guest_size, size_t map_pages) { + if (!switch_on || !why || !std::strcmp(why, "?")) return false; + return gpu >= kVirtual && guest_size != 0 && map_pages != pages(guest_size); +} + +} // namespace drawn_page_map diff --git a/gpu/vk/driver_faults.h b/gpu/vk/driver_faults.h new file mode 100644 index 0000000..56f1dbb --- /dev/null +++ b/gpu/vk/driver_faults.h @@ -0,0 +1,140 @@ +// Programs the driver faulted on, kept out of it, and one build of a program at a time (no game, no GPU: the rules and the +// gate; vk_draw.cpp fault_quarantine, build_graphics, guarded_graphics). Apart from Vulkan so that +// tests/test_driver_faults.py runs it alone, on many made-up PCs. +// +// (Triforce, 11-22, the performance audit's PIPE-57, after Source A's.) The guards catch a fault inside the driver's +// compiler (vk_draw.cpp guarded_graphics), and a fault during the start's own shader work starts the game again +// (driver_fault_restart). Nothing remembered the program: the next start asked the driver for it again, faulted again and +// restarted again, and in play a second thread could ask for the same program while the driver was still inside the +// first. Source A saw that on AMD's Windows driver (an RX 6800 XT): in a freeze, three threads sat in the driver on two +// programs' builds while the presenting thread waited for one of them; a preparation stuck at "6625 of 6627, nothing +// finished for 30 s" took 32.5 s, and 4.2 s with three faulted programs kept out on the first run, 2.3 s after. Triforce +// has the same class of report: "two of 4926 on an RX 5700 XT, stuck at 99%". +// +// What. On AMD's own Windows driver (or everywhere with RX_VK_DRIVER_FAULTS=2): +// - a program whose pipeline build faulted in the driver is not asked of the driver again this run, and its key goes to +// triforce_driver_faults.txt beside the executable, one line under the vendor and the driver's version, written by the +// fault's handler before the restart can end the process (so a start that faults does not fault again); later starts +// with that driver skip those programs, and say so once (delete the file to try again; another driver version ignores +// the lines); +// - one build of a program at a time: a second waits at the gate, outside the driver, and a probe that asks the caches +// only is told a compile is required. +// Elsewhere (NVIDIA's, Intel's, Mesa's drivers, AMD's on Linux) nothing of it runs, as before: the NVIDIA fault of 11-13 +// was a race in the driver, not a bad program. RX_VK_DRIVER_FAULTS=0: nowhere. +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace driver_faults { + +/// RX_VK_DRIVER_FAULTS: 0 off, 1 (unset) AMD's Windows driver, 2 every driver +inline int mode(const char* env) { + if (!env || !*env) return 1; + const int v = std::atoi(env); + return v <= 0 ? 0 : v >= 2 ? 2 : 1; +} +/// Whether the quarantine, its file and the gate are on: mode 2 anywhere, mode 1 on Windows with AMD's own driver +/// (VK_DRIVER_ID_AMD_PROPRIETARY) +inline bool on(int mode, bool windows, bool amd_proprietary) { + return mode >= 2 || (mode == 1 && windows && amd_proprietary); +} + +inline constexpr int kLine = 35; // ("vvvvvvvv dddddddd kkkkkkkkkkkkkkkk\n") + +/// One line of the file into out (kLine bytes and a 0): made without the C library, as the fault's handler makes it +inline int line(char* out, uint32_t vendor, uint32_t driver, uint64_t key) { + static const char hex[] = "0123456789abcdef"; + int n = 0; + for (int i = 7; i >= 0; i--) out[n++] = hex[(vendor >> (4 * i)) & 0xF]; + out[n++] = ' '; + for (int i = 7; i >= 0; i--) out[n++] = hex[(driver >> (4 * i)) & 0xF]; + out[n++] = ' '; + for (int i = 15; i >= 0; i--) out[n++] = hex[(key >> (4 * i)) & 0xF]; + out[n++] = '\n'; + out[n] = 0; + return n; +} +/// The keys of the file's lines for this vendor and driver version; another's, and a broken line, are passed over +inline std::vector parse(const std::string& text, uint32_t vendor, uint32_t driver) { + std::vector keys; + const auto hexval = [](const std::string& s, size_t at, int digits, uint64_t* v) { + if (at + (size_t)digits > s.size()) return false; + uint64_t x = 0; + for (int i = 0; i < digits; i++) { + const char c = s[at + i]; + const int d = c >= '0' && c <= '9' ? c - '0' : c >= 'a' && c <= 'f' ? c - 'a' + 10 : c >= 'A' && c <= 'F' ? c - 'A' + 10 : -1; + if (d < 0) return false; + x = x << 4 | (uint64_t)d; + } + *v = x; + return true; + }; + size_t at = 0; + while (at < text.size()) { + size_t end = text.find('\n', at); + if (end == std::string::npos) end = text.size(); + std::string l = text.substr(at, end - at); + if (!l.empty() && l.back() == '\r') l.pop_back(); + uint64_t v = 0, d = 0, k = 0; + if (l.size() == (size_t)kLine - 1 && l[8] == ' ' && l[17] == ' ' && hexval(l, 0, 8, &v) && hexval(l, 9, 8, &d) && + hexval(l, 18, 16, &k) && v == vendor && d == driver) + keys.push_back(k); + at = end + 1; + } + return keys; +} + +/// The run's quarantine and the gate: begin before a program's build, end after it (with whether it faulted) +class Quarantine { +public: + enum class Begin { Go, Busy, Faulted }; + void load(const std::vector& keys) { + std::lock_guard lk(mu_); + bad_.insert(keys.begin(), keys.end()); + } + /// Faulted: not asked of the driver. Busy: another build of it is inside the driver and the caller may not wait (a + /// probe of the caches). Go: this build may go in; the caller ends it with end(). may_wait: wait at the gate instead + Begin begin(uint64_t key, bool may_wait) { + std::unique_lock lk(mu_); + for (;;) { + if (bad_.count(key)) return Begin::Faulted; + if (!busy_.count(key)) break; + if (!may_wait) return Begin::Busy; + cv_.wait(lk); + } + busy_.insert(key); + return Begin::Go; + } + /// True when it faulted now for the first time + bool end(uint64_t key, bool faulted) { + bool fresh = false; + { + std::lock_guard lk(mu_); + busy_.erase(key); + if (faulted) fresh = bad_.insert(key).second; + } + cv_.notify_all(); + return fresh; + } + bool faulted(uint64_t key) const { + std::lock_guard lk(mu_); + return bad_.count(key) != 0; + } + size_t count() const { + std::lock_guard lk(mu_); + return bad_.size(); + } + +private: + mutable std::mutex mu_; + std::condition_variable cv_; + std::unordered_set bad_, busy_; +}; + +} // namespace driver_faults diff --git a/gpu/vk/evict_reason.h b/gpu/vk/evict_reason.h new file mode 100644 index 0000000..f0c4846 --- /dev/null +++ b/gpu/vk/evict_reason.h @@ -0,0 +1,65 @@ +// Whether the write-back of a texture the eviction chose is stale: another image was drawn over its storage since its own +// last drawing (no game, no GPU: a rule). tests/test_evict_reason.py +// +// (Triforce, 11-22, the performance audit's TEX-R404, after Source A's.) Over the budget, the eviction writes each chosen +// texture the GPU drew back into the game's memory before it goes, so that one made again over that storage is uploaded +// with what the GPU drew. The game draws its HUD labels (110 x 110, 46 x 46, 140 x 140) inside the storage of a +// 1920 x 1080 target it no longer uses. The eviction's own tests do not see that: drawn_over looks only at images of the +// same start, and target_overlap.h leaves out an image inside the target. So when the target went, its old texels landed +// over the labels, the pages counted as written, and the labels, finalized, were uploaded from them as noise (Source A: +// in 3 of 3 runs of two minutes in Hyrule Field at a 6 GB allowance, none with the rule). +// +// The rule: a write-back is left out when another image the GPU drew overlaps the texture's storage and was drawn in a +// later frame than the texture's own last drawing. The texture still goes. One drawn in the same frame keeps the +// write-back (a frame's targets drawn side by side), and so does one that only touches it. What is given up: the parts +// of the texture the newer image does not cover are not written back either; made again there and sampled undrawn, it +// starts from what the memory holds, as before the write-back existed. RX_VK_EVICT_STALE_FLUSH=1: written back, as before. +#pragma once +#include +#include +#include +#include +#include + +namespace evict_reason { + +/// RX_VK_EVICT_STALE_FLUSH: unset, empty or 0 leaves a stale write-back out; anything else writes it back, as before +inline bool flush_stale(const char* env) { return env && *env && std::atoi(env) != 0; } + +/// Whether the storage [lo, hi) and [olo, ohi) share a byte (ranges that touch do not) +constexpr bool overlaps(uint64_t lo, uint64_t hi, uint64_t olo, uint64_t ohi) { + return lo < hi && olo < ohi && lo < ohi && olo < hi; +} + +/// Whether a texture last drawn in frame `last` holds texels older than what is in its storage: `other` is the frame of +/// the newest drawing of another image over that storage (0: none) +constexpr bool write_back_stale(uint64_t last, uint64_t other) { return other > last; } + +/// A texture whose write-back the eviction is about to make +struct Candidate { + uint64_t lo = 0, hi = 0; // its storage + size_t index = 0; // the caller's own: its place in the eviction's list + const void* self = nullptr; // itself, so that it is not taken for another image over its own storage + uint64_t newest = 0; // the frame of the newest drawing of another image over its storage (0: none) +}; + +/// Puts the candidates in the order of their starts, for note(); returns the longest one's size +inline uint64_t by_start(std::vector& c) { + std::sort(c.begin(), c.end(), [](const Candidate& a, const Candidate& b) { return a.lo < b.lo; }); + uint64_t longest = 0; + for (const Candidate& k : c) longest = std::max(longest, k.hi > k.lo ? k.hi - k.lo : 0); + return longest; +} + +/// One image the GPU drew, [olo, ohi), last drawn in frame `o_written`: every candidate it overlaps (but itself) takes +/// that frame when it is newer than what the candidate has. `c` is in by_start's order, `longest` what by_start gave +inline void note(std::vector& c, uint64_t longest, uint64_t olo, uint64_t ohi, uint64_t o_written, const void* other) { + if (ohi <= olo) return; + // (a candidate over [olo, ohi) starts before ohi and after olo less its size: the search begins at olo less the longest) + auto it = std::lower_bound(c.begin(), c.end(), olo > longest ? olo - longest : 0, + [](const Candidate& k, uint64_t g) { return k.lo < g; }); + for (; it != c.end() && it->lo < ohi; ++it) + if (it->self != other && overlaps(it->lo, it->hi, olo, ohi) && o_written > it->newest) it->newest = o_written; +} + +} // namespace evict_reason diff --git a/gpu/vk/fallback_size_policy.h b/gpu/vk/fallback_size_policy.h new file mode 100644 index 0000000..cc643f6 --- /dev/null +++ b/gpu/vk/fallback_size_policy.h @@ -0,0 +1,73 @@ +// Mesh-fallback compute stages too big for two Mesa drivers are not handed to them (no game, no GPU: a rule). +// +// (Triforce, 11-22, the performance audit's PIPE-56, after Source A's.) A mesh program on a GPU without mesh shaders runs +// as compute stages (vk_draw.cpp fallback_compute_pipelines). Every stage went to the driver, and a refusal was only +// remembered after the build. Two drivers do not get that far on the largest stages: +// - Mesa's RADV on GCN 1 to 4 (GFX6-8: vk_vendor.h radv_gcn). Source A's numbers under drm-shim with a 6 GB cap: the +// 79,947-phi stage not built after 45-90 s and killed (25 GB at 150 s), where GFX9 and Navi built it in 13-14 s at +// 2.9 GB. Limit: 50,000 OpPhi in one stage (2 of about 800 modules are over it); +// - Mesa's Intel driver. An Iris Xe: a 251-call stage hit a 14 GB cap unfinished after 27 minutes. Limits: 20,000 OpPhi +// or 200 OpFunctionCall in one stage (10 of 376 stages left out in Source A's full preparation). +// A program with a stage over its driver's limit goes into the failed set before anything is built, as a program the +// driver refused does: its draws are left out. Which objects those are is not known. No other driver has a limit, and the +// switches set none there: NVIDIA, Intel's Windows driver, AMD's own drivers and RADV from Vega on build every stage. +// +// RX_VK_FALLBACK_PHI_LIMIT=, RX_VK_FALLBACK_CALL_LIMIT=: the two limits on those two drivers; 0: no limit. +#pragma once +#include +#include +#include + +namespace fallback_size { +constexpr uint32_t kOpFunctionCall = 57, kOpPhi = 245; +constexpr uint32_t kGcnPhis = 50000, kIntelPhis = 20000, kIntelCalls = 200; + +struct Size { + uint32_t phis = 0, calls = 0; +}; +struct Limits { + uint32_t phis = 0, calls = 0; // (0: none) +}; +enum class Driver { Other, RadvGcn, MesaIntel }; + +/// A module's OpPhi and OpFunctionCall instructions. One that does not read as SPIR-V counts 0 of both +inline Size count(const uint32_t* code, size_t words) { + Size s; + if (!code || words < 5 || code[0] != 0x07230203u) return s; + for (size_t at = 5; at < words;) { + const uint32_t n = code[at] >> 16, op = code[at] & 0xffffu; + if (!n || at + n > words) return Size{}; + s.phis += op == kOpPhi, s.calls += op == kOpFunctionCall; + at += n; + } + return s; +} +inline uint32_t spirv_phis(const uint32_t* code, size_t words) { return count(code, words).phis; } +inline uint32_t spirv_calls(const uint32_t* code, size_t words) { return count(code, words).calls; } + +/// A limit's switch: unset, empty or not a number, the driver's own; 0, none; else the number +inline uint32_t parse_limit(const char* env, uint32_t own) { + if (!env || !*env) return own; + char* end = nullptr; + const unsigned long long v = std::strtoull(env, &end, 10); + if (end == env || *end) return own; + return v > 0xffffffffull ? 0xffffffffu : (uint32_t)v; +} + +/// The driver: RADV on a GCN 1 to 4 GPU first (it is never Mesa's Intel driver), then Mesa's Intel driver +constexpr Driver driver_of(bool radv_gcn, bool mesa_intel) { + return radv_gcn ? Driver::RadvGcn : mesa_intel ? Driver::MesaIntel : Driver::Other; +} + +/// The limits on that driver, with the two switches' values (nullptr: unset). None on any other driver, whatever is set +inline Limits limits(Driver d, const char* phi_env, const char* call_env) { + if (d == Driver::Other) return Limits{}; + const Limits own = d == Driver::RadvGcn ? Limits{kGcnPhis, 0} : Limits{kIntelPhis, kIntelCalls}; + return Limits{parse_limit(phi_env, own.phis), parse_limit(call_env, own.calls)}; +} + +/// The stage is over a limit: its program is not built +constexpr bool refused(const Limits& l, const Size& s) { + return (l.phis && s.phis > l.phis) || (l.calls && s.calls > l.calls); +} +} // namespace fallback_size diff --git a/gpu/vk/fg_motion.h b/gpu/vk/fg_motion.h new file mode 100644 index 0000000..7bdb6a7 --- /dev/null +++ b/gpu/vk/fg_motion.h @@ -0,0 +1,38 @@ +// (Triforce, 11-22, the owner's DLSS frame generation flicker, again) Whose motion vectors frame generation is given. +// +// Since FG-4 (vk_fsr3_inputs.cpp chain_motion), while NVIDIA DLSS or AMD FSR 4 upscales, frame generation's motion is the +// upscaler chain's: the camera's from the depth, with every draw of the scene that stored its positions drawn again over +// it (vk_temporal.cpp capture_motion). With NVIDIA DLSS frame generation that showed as the owner's flicker of 11-20 again +// (the whole picture while the camera moved), on a build with the clip-space fix of that day (dlss_math.h +// depth_clip_projection) in it. The owner's runs of 11-22 (DLSS at 50%, 2x frame generation, HDR, before the bloom): +// - RX_VK_FG_OWN_MOTION=0, the camera's motion alone: no flicker; +// - the chain's motion with the upscaler audit's commits all set back (RX_VK_CAPTURE_TABLES=0, RX_VK_REDRAW_STATE=0, +// RX_VK_DLSS_FEATURE_MAX=1): the flicker, so it is not theirs; +// - RX_VK_DLSS_PROBE=1 in that run: the redraw's motion of the scene's largest draw within 0.03 px of the camera's. +// What in the chain's motion Streamline takes for a flicker is not known: its made frames are not in any log or shot. DLSS +// frame generation finds what moves by itself with its own optical flow, and FSR's inputs give it the depth and the +// camera, so it is given the camera's motion alone: what the first run showed. AMD's frame generation and FSR 3.1's +// interpolation keep the chain's, as since FG-4 (no flicker of theirs is reported). +// +// RX_VK_DLSSG_OWN_MOTION=1: DLSS frame generation is given the chain's motion again, as before. RX_VK_FG_OWN_MOTION=0 still +// gives every generator the camera's alone. +// +// No Vulkan: tests/test_fg_motion.py compiles this. +#pragma once +#include + +namespace fg_motion { + +/// A switch read as the renderer reads its others: set and not 0 (value null when unset) +inline bool set_on(const char* value) { return value && *value && std::atoi(value) != 0; } +/// RX_VK_FG_OWN_MOTION=0 (value null when unset): the camera's motion alone, for every generator +inline bool camera_alone(const char* value) { return value && *value && std::atoi(value) == 0; } + +/// Whether frame generation is given the upscaler chain's motion this frame. dlssg: NVIDIA DLSS frame generation runs in +/// this session (vk_dlssg.h active). fg_own: RX_VK_FG_OWN_MOTION; dlssg_own: RX_VK_DLSSG_OWN_MOTION (null when unset) +inline bool chain(bool dlssg, const char* fg_own, const char* dlssg_own) { + if (camera_alone(fg_own)) return false; + return !dlssg || set_on(dlssg_own); +} + +} // namespace fg_motion diff --git a/gpu/vk/finalized_obsolete.h b/gpu/vk/finalized_obsolete.h new file mode 100644 index 0000000..8fb5b4c --- /dev/null +++ b/gpu/vk/finalized_obsolete.h @@ -0,0 +1,42 @@ +// (Triforce, 11-21, the performance audit's TEX-43, after Source A's) A finalized texture the GPU drew whose storage the +// game has since written over. +// +// A texture the GPU drew in one frame is kept after the game finalizes it (texture_sweep keep_finalized, up to +// RX_VK_KEEP_FINALIZED_MB): the game may make a texture over that storage again and sample it without drawing it, as the +// terrain does with its 260 x 260 tiles. Within the budget only a draw over one let it go, and the game's own CPU writes +// to that memory never did: in players' logs about 200 render target images an hour stayed for good, most of them such +// tiles, with the blocks they pin. Source A's rule: once the CPU has written three quarters or more of the storage since +// the release, the game has put other data there, the image holds nothing that memory holds on the console, and the +// sweep lets it go whatever the budget says. Only one of 32 KB or more (a small one shares its pages with what the game +// keeps beside it, and a write there says nothing of it) and not one in a virtual pool. Its laps of Hyrule Field at a +// 6 GB allowance: render target images 129, 166, 221 against 128, 131, 129; the kept ones 131 against 41. +// The answer is kept with the texture: once rewritten it stays so until it is released again, and one not yet rewritten +// is looked at again at most every kRecheck presents (the look walks its pages' write generations). +// RX_VK_KEEP_OBSOLETE=1: kept, as before. +// +// The rule alone: tests/test_finalized_obsolete.py compiles it. +#pragma once +#include +#include + +namespace finalized_obsolete { + +inline constexpr uint64_t kMinBytes = 32u << 10; +inline constexpr uint64_t kVirtual = 1ull << 40; +inline constexpr uint64_t kRecheck = 60; + +/// RX_VK_KEEP_OBSOLETE: not 0 keeps them, as before +inline bool keep(const char* asked) { return asked && atoi(asked) != 0; } + +/// Whether a texture can be one: finalized after the GPU wrote it, in no virtual pool, of 32 KB of storage or more +constexpr bool candidate(bool released, bool gpu_written, uint64_t gpu, uint64_t guest_size) { + return released && gpu_written && gpu < kVirtual && guest_size >= kMinBytes; +} + +/// Whether `written` bytes of a storage of guest_size, written by the CPU since the release, make it rewritten +constexpr bool rewritten(uint64_t written, uint64_t guest_size) { return guest_size != 0 && written * 4 >= guest_size * 3; } + +/// Whether one not rewritten when last looked at, in present `checked` (0: never), is looked at again in present `frame` +constexpr bool due(uint64_t frame, uint64_t checked) { return !checked || frame - checked >= kRecheck; } + +} // namespace finalized_obsolete diff --git a/gpu/vk/first_start.h b/gpu/vk/first_start.h new file mode 100644 index 0000000..55d36f4 --- /dev/null +++ b/gpu/vk/first_start.h @@ -0,0 +1,74 @@ +// (Triforce, 11-22, the performance audit's PRESENT-46, after Source A's) A first start sized to the card: the Render +// resolution row set once on a weak GPU. +// +// With no options file every card started the same: the Render resolution at Auto, the game's dynamic resolution. Source A +// reads the card and puts it in a tier; its tier 1 is an integrated GPU (a handheld's is one), a card without mesh shaders, +// or a card with 4.5 GB of video memory or less, and it draws the 3D at 50% there, or at 33% on an integrated GPU beside +// 4 processor threads or fewer (its note: an i3-1115G4's own GPU took 60 to 220 ms a frame at 50%). +// +// This is that test, for a first start only (vk_menu.cpp menu_first_start): no options file, RX_VK_LOW_SPEC not set, +// nothing saved by it, so a start after the player has saved anything is the player's own. Left out, as the audit's +// entry says: Source A's preset row, its fingerprint that chooses again on new hardware (it would overwrite the player's +// rows), its tuner in play, any change of the upscaler or of the output's size, and its test of the SteamDeck variable, +// which took desktop PCs started from that launcher for handhelds: a handheld is found by its integrated GPU alone. +// +// RX_VK_FIRST_START_AUTO: not set or 0, off (as before; the default until the owner has decided what a first start on a +// weak card gets). 1: on. A larger number: on, the card taken as having that many MB of video memory, for the lab. +// +// No Vulkan, no menu: tests/test_first_start.py compiles this. +#pragma once +#include +#include + +namespace first_start { + +/// What is read of the card +struct Card { + bool integrated = false; // (VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU) + bool mesh_shaders = true; // (VK_EXT_mesh_shader, as the device was made) + uint64_t vram_mb = 0; // (its video memory; 0: not known) + unsigned threads = 0; // (the processor's threads; 0: not known) +}; + +constexpr uint64_t kSmallVramMb = 4608; // 4.5 GB: tier 1 up to here +constexpr unsigned kFewThreads = 4; // an integrated GPU beside this many threads or fewer: 33% +constexpr int kRowHalf = 4, kRowThird = 6; // opts.resolution's 50% and 33% (upscaler_rows.h) + +enum Why { kKeep = 0, kIntegrated, kNoMeshShaders, kSmallVram }; +/// The Render resolution row for the card's first start: row 0, the row left as it is (not tier 1) +struct Choice { + int row = 0; + Why why = kKeep; +}; + +inline Choice choose(const Card& c) { + if (c.integrated) return {c.threads && c.threads <= kFewThreads ? kRowThird : kRowHalf, kIntegrated}; + if (!c.mesh_shaders) return {kRowHalf, kNoMeshShaders}; + if (c.vram_mb && c.vram_mb <= kSmallVramMb) return {kRowHalf, kSmallVram}; + return {}; +} + +inline const char* why_text(Why w) { + switch (w) { + case kIntegrated: return "an integrated GPU"; + case kNoMeshShaders: return "no mesh shaders"; + case kSmallVram: return "4.5 GB of video memory or less"; + default: return "not a weak card"; + } +} + +/// RX_VK_FIRST_START_AUTO (value, null when unset): -1 off; 0 on, the card's own video memory; otherwise on, the card +/// taken as having that many MB +inline int64_t switch_mb(const char* text) { + if (!text || !*text) return -1; + const long long v = std::atoll(text); + return v <= 0 ? -1 : v == 1 ? 0 : (int64_t)v; +} + +/// Whether the first start's choice is made at all: the switch on, no options file, RX_VK_LOW_SPEC not asked (it sets the +/// row itself), and no switch that sets the Render resolution for this run (HOS_DYNRES) +inline bool applies(bool has_options, bool low_spec, bool resolution_env, int64_t switch_value) { + return switch_value >= 0 && !has_options && !low_spec && !resolution_env; +} + +} // namespace first_start diff --git a/gpu/vk/frame_trace.h b/gpu/vk/frame_trace.h index 8cc0d0d..e553a15 100644 --- a/gpu/vk/frame_trace.h +++ b/gpu/vk/frame_trace.h @@ -28,6 +28,7 @@ struct Split { double wb_wait = 0; // (the waits for write-backs still on the GPU, wherever they were: memory sync's capture, a read) double wb_held = 0; // (of them, the part after the writer's recording was done: the timeline held, not the GPU) double qs_bg = 0; // (the queue submits made while a background import ran) + double gpu_waits = 0; // ((Triforce, 11-22, SYNC-27) every thread's time in calls that block on the GPU: gpu_wait_kinds.h) }; /// Outside the replay and the present: the game's own update, its waits for the replay and the GPU among it. As the stall line's @@ -35,12 +36,13 @@ inline double outside(const Split& s) { return s.interval - s.replay - s.present /// The columns, after the trace's first ones (their names; the line's values in this order: line) constexpr const char* kColumns = - "replay_ms,present_ms,dispatch_ms,qsubmit_ms,par_wait_ms,outside_ms,game_wait_ms,captures_ms,wb_wait_ms,wb_held_ms,qs_bg_ms"; + "replay_ms,present_ms,dispatch_ms,qsubmit_ms,par_wait_ms,outside_ms,game_wait_ms,captures_ms,wb_wait_ms,wb_held_ms,qs_bg_ms," + "gpu_waits_ms"; /// The values for kColumns, comma-separated, no line end: snprintf's count inline int line(const Split& s, char* out, size_t cap) { - return std::snprintf(out, cap, "%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f", s.replay, s.present, s.dispatch, - s.qsubmit, s.par_wait, outside(s), s.game_wait, s.captures, s.wb_wait, s.wb_held, s.qs_bg); + return std::snprintf(out, cap, "%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f", s.replay, s.present, s.dispatch, + s.qsubmit, s.par_wait, outside(s), s.game_wait, s.captures, s.wb_wait, s.wb_held, s.qs_bg, s.gpu_waits); } } // namespace mkwvk::frame_trace diff --git a/gpu/vk/framegen_idle.h b/gpu/vk/framegen_idle.h new file mode 100644 index 0000000..ef5a542 --- /dev/null +++ b/gpu/vk/framegen_idle.h @@ -0,0 +1,49 @@ +// (Triforce, 11-21, the performance audit's PRESENT-18, after Source A's) When frame generation's images are let go. +// +// The built-in interpolation's frames, lumas, flows and made-frame sets (and, for AMD FSR 4's frame generation, the four +// images of the picture without the HUD) are made at the first frame that needs them and were kept until a resize: turned +// off, or left for NVIDIA's own frame generation, they stayed in VRAM for the run. Source A measured 150 to 160 MiB given +// back at 2560 x 1440 with 2x after its switch away. They are let go after kPresents presents in a row with frame +// generation off, about two seconds at 60: a row stepped through on the way to another choice keeps them. A present that +// recorded made frames counts as on whatever the row says by then: its set is busy until the display thread has shown +// it, and a drain then would wait for a job that is only queued afterwards. So the count is taken after the present's +// queue (vk_replay.cpp, after framegen_queue). They are made again at the next frame that needs them, as after a resize. +// RX_VK_FG_KEEP=1: kept, as before. +// +// The rule alone, no Vulkan: tests/test_framegen_idle.py compiles it. +#pragma once +#include + +namespace framegen_idle { + +inline constexpr uint32_t kPresents = 120; + +/// The presents in a row off, after this one: on now, or used at this present, starts again +inline uint32_t next_off(uint32_t off, bool on_now, bool used) { return on_now || used ? 0 : off < UINT32_MAX ? off + 1 : off; } + +/// Whether this present lets them go: there are some, and they have been off long enough +inline bool due(uint32_t off, bool have) { return have && off >= kPresents; } + +/// One present, as framegen_idle sees it after the present's queue +struct Present { + bool on; // frame generation on by the row, read again (framegen_on) + bool recorded; // this present recorded made frames (framegen_record's true) + bool hudless_wanted; // AMD FSR 4's frame generation is the row's (fsr4fg::frames_wanted) + bool have_images; // the shared images are there (fg.w) + bool have_hudless; // the images of the picture without the HUD are there +}; +struct Counts { + uint32_t off = 0, hudless_off = 0; +}; +struct Release { + bool shared, hudless; +}; +/// The two counts after this present, and what it lets go. keep: RX_VK_FG_KEEP, nothing ever +inline Release step(Counts& c, const Present& p, bool keep) { + c.off = next_off(c.off, p.on, p.recorded); + c.hudless_off = next_off(c.hudless_off, p.on && p.hudless_wanted, false); + if (keep) return {false, false}; + return {due(c.off, p.have_images), due(c.hudless_off, p.have_hudless)}; +} + +} // namespace framegen_idle diff --git a/gpu/vk/framegen_refresh.h b/gpu/vk/framegen_refresh.h new file mode 100644 index 0000000..94522ad --- /dev/null +++ b/gpu/vk/framegen_refresh.h @@ -0,0 +1,40 @@ +// (Triforce, 11-21, the upscaler audit's UPS-05) A fixed multiplier's frames, no more than the display can show. +// +// The Frame generation row's 2x to 8x make that many frames for each of the game's whatever the display's refresh rate +// is; only the row's Auto goes by it. The owner's log of 11-20 has 5x on a 165 Hz display with the game near 66 frames a +// second: 1,655 frames shown in every 5 s (331 a second, two for each refresh), each present call 2.2 to 3.0 ms where it +// takes 0.03 ms without frame generation, and the display thread's line "0 frames shown on time, 1655 late". Half of the +// made frames were never on the screen (the swapchain is mailbox: the newest at each refresh), and each was a pass over +// the whole picture and a present the display thread could not keep up with. +// +// With the switch, a fixed multiplier makes no more frames than refreshes fall in one frame of the game: the game's +// recent frame time over the display's period, rounded up, a tenth of a refresh forgiven so a game at a whole fraction +// of the refresh rate is not given one more for its frames' jitter. Never more than the row asks, never fewer than the +// game's own frame. The row's Auto and AMD FSR's own are not touched. +// +// RX_VK_FRAMEGEN_REFRESH=1: on. Off unless set (the row's number is the frames made, as before): it changes what a row +// the player set shows, so it is the owner's to turn on. +// +// No Vulkan: tests/test_framegen_refresh.py compiles this. +#pragma once +#include +#include + +namespace framegen_refresh { + +/// RX_VK_FRAMEGEN_REFRESH (value, null when unset): on when it is not 0; unset, off +inline bool on(const char* value) { return value && *value && std::atoi(value) != 0; } + +/// The frames shown for each of the game's (the game's own among them) with a fixed multiplier m, the game's frames game_ns +/// apart, the display refreshing every period_ns. An unknown frame time or period: m +inline int frames(int m, int64_t game_ns, int64_t period_ns) { + if (m <= 1 || game_ns <= 0 || period_ns <= 0) return m; + // (the refreshes in a frame of the game, rounded up, a tenth of one forgiven: ceil(game / period - 0.1)) + const int64_t most = (game_ns * 10 + period_ns * 9 - 1) / (period_ns * 10); + return most < 1 ? 1 : most < m ? (int)most : m; +} + +/// The display's period for a refresh rate in mHz (vk_main.cpp display_refresh_mhz); 0 when it is not known +inline int64_t period_ns(int refresh_mhz) { return refresh_mhz > 0 ? (int64_t)(1000000000000LL / refresh_mhz) : 0; } + +} // namespace framegen_refresh diff --git a/gpu/vk/framegen_rows.h b/gpu/vk/framegen_rows.h index e0bb82f..a600875 100644 --- a/gpu/vk/framegen_rows.h +++ b/gpu/vk/framegen_rows.h @@ -232,4 +232,24 @@ template void teardown(Ops& ops, bool exiting) { ops.destroy_images(); } +// ---- (Triforce, 11-20) What a present writes into a set of frames: never past the frames the set was made with ---- +// The owner's "upscaling crash": 0xc0000005 reading 0x590 in NVIDIA's driver, inside a vkCmdCopyImage the recorder thread +// ran, in three runs of 11-20, each with frame interpolation on and the Frame generation row stepped up in the options +// (Auto in the first, 3x at the next start, 5x at the one after). The sets' frames are made for the row's multiplier when +// a present begins (vk_framegen.cpp ensure_size); the present then waits for the display thread to let a set go, and +// while frames are made the display thread is the one that runs the menu (present_image, menu_tick), so a step of the row +// lands in that wait. The multiplier read again after it was more than the frames made, and the game's frame was copied +// into an image that was never made (VK_NULL_HANDLE). A step down only left frames unused. The row's new multiplier takes +// effect at the next present, which makes the sets anew. +/// The multiplier a present records with: the row's, and no more than the frames its sets were made with +constexpr int set_multiplier(int row_multiplier, int made_with) { + const int most = made_with < 1 ? 1 : made_with; + return row_multiplier < 1 ? 1 : row_multiplier > most ? most : row_multiplier; +} +/// The game's own frame's place in a set, after `made` made ones: the set's last frame at most +constexpr int game_slot(int made, int made_with) { + const int last = (made_with < 1 ? 1 : made_with) - 1; + return made < 0 ? 0 : made > last ? last : made; +} + } // namespace mkwvk::framegen_rows diff --git a/gpu/vk/gpu_choice.h b/gpu/vk/gpu_choice.h index 3331bbb..007635e 100644 --- a/gpu/vk/gpu_choice.h +++ b/gpu/vk/gpu_choice.h @@ -46,4 +46,24 @@ inline int pick(const std::string& want, const std::vector& names, return menu_device(menu, (int)names.size()); } +/// ((Triforce) 11-22, tickets "igpu targeted for shader cache", "Launcher \"GPU device\" setting is non-functional" (RX 6600 +/// with a Ryzen 5600G's graphics) and on 0.6.0 "still picking IGPU for building shaders") The choice that pick takes by its +/// number alone (no saved name, or a name no card has here) when that number is the processor's graphics while a card of its +/// own is listed: not taken, Auto instead. Such a number comes from an options file written before 11-20, when the launcher +/// numbered the cards in Windows' order (the display class's driver keys: the processor graphics' driver first on a 5600G) +/// and saved no name with it; the update to 0.6.0 still prepared its shaders there and the game started there, as no name +/// came with the number until the row was chosen again. A choice saved with its name (the launcher and the menu since 11-20) +/// is taken as it is, the processor graphics too. types: the usable devices' VkPhysicalDeviceType (1 integrated, 2 discrete), +/// in the game's order, as names +inline bool unnamed_integrated(const std::string& want, const std::vector& names, const std::vector& types, + int menu) { + const int k = pick(want, names, menu); + if (k < 0 || (size_t)k >= types.size() || types[(size_t)k] != 1) return false; + const std::string key = name_key(want); + if (!key.empty() && name_key(names[(size_t)k]) == key) return false; // (chosen by its name) + for (const int t : types) + if (t == 2) return true; + return false; +} + } // namespace mkwvk::gpu_choice diff --git a/gpu/vk/gpu_clock.h b/gpu/vk/gpu_clock.h new file mode 100644 index 0000000..4f9bb20 --- /dev/null +++ b/gpu/vk/gpu_clock.h @@ -0,0 +1,57 @@ +// (Triforce) Whether the renderer holds an AMD GPU at amdgpu's stable power state while the game runs, and which one (Linux: +// vk_gpu_clock.cpp; RX_VK_GPU_CLOCK). Plain code with no system calls: tests/test_gpu_clock.py runs it on made-up devices. +// +// The Steam Deck's automatic GPU clock drops while the frame waits on the CPU or a readback (200 MHz for 27% of a Kokiri run), so +// the GPU work after the wait runs slow. amdgpu lets any process that opens the GPU's render node ask for a stable power state +// through a context of its own (DRM_IOCTL_AMDGPU_CTX, AMDGPU_CTX_OP_SET_STABLE_PSTATE, Linux 5.18): no root and no sysfs write. +// The kernel gives the level back when that context goes, at the process's end too, a crash or kill -9 included. Measured on a +// Deck OLED (SteamOS 3.9.5, plugged in): Kokiri handheld 11.1 -> 16.2 fps, 1% low 6.8 -> 12.1, GPU time 58 -> 32 ms at the same +// 15 W; Hyrule Field +19% for 4 W more. It costs battery where the APU was under its power cap. +// +// - Unset: the peak state, on the Deck's APU only (its Vulkan name, or SteamDeck=1 from Steam on an integrated AMD GPU), and +// only while the level is amdgpu's own (power_dpm_force_performance_level "auto"): a level the player set is left alone. +// - RX_VK_GPU_CLOCK=0: nothing opened. =peak (or 1), =standard: that state on any AMD GPU, whatever the level. +#pragma once + +#include +#include + +namespace mkwvk::gpu_clock { + +enum Want { kNone, kPeak, kStandard }; + +// (uapi/drm/amdgpu_drm.h, written out: no libdrm and no kernel headers needed to build) +constexpr uint32_t kCtxOpAlloc = 1, kCtxOpSetStablePstate = 6; +constexpr uint32_t kPstateStandard = 1, kPstatePeak = 4; +constexpr uint32_t kCtxArgBytes = 16; // (union drm_amdgpu_ctx) +/// DRM_IOCTL_AMDGPU_CTX: DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_CTX, union drm_amdgpu_ctx) = _IOWR('d', 0x40 + 0x02, 16 bytes) +constexpr unsigned long kCtxIoctl = (3ul << 30) | ((unsigned long)kCtxArgBytes << 16) | ((unsigned long)'d' << 8) | 0x42ul; + +/// The Deck's APU: "AMD Custom GPU 0405" (LCD) or "0932" (OLED) in its Vulkan name; or SteamDeck=1 (Steam sets it there) on an +/// integrated GPU +inline bool deck_apu(const char* device_name, bool integrated, const char* steam_deck_env) { + if (device_name && (std::strstr(device_name, "Custom GPU 0405") || std::strstr(device_name, "Custom GPU 0932"))) return true; + return integrated && steam_deck_env && !std::strcmp(steam_deck_env, "1"); +} + +struct Choice { + Want want; + const char* why; +}; + +/// switch_value: RX_VK_GPU_CLOCK (nullptr unset); level: power_dpm_force_performance_level's text, first word (nullptr: unread) +inline Choice choose(const char* switch_value, uint32_t vendor_id, bool integrated, const char* device_name, + const char* steam_deck_env, const char* level) { + if (vendor_id != 0x1002) return {kNone, "not an AMD GPU"}; + if (switch_value && *switch_value) { + if (!std::strcmp(switch_value, "0")) return {kNone, "RX_VK_GPU_CLOCK=0"}; + if (!std::strcmp(switch_value, "peak") || !std::strcmp(switch_value, "1")) return {kPeak, "RX_VK_GPU_CLOCK=peak"}; + if (!std::strcmp(switch_value, "standard")) return {kStandard, "RX_VK_GPU_CLOCK=standard"}; + } + if (!deck_apu(device_name, integrated, steam_deck_env)) return {kNone, "not a Steam Deck (RX_VK_GPU_CLOCK=peak: any AMD GPU)"}; + if (!level) return {kNone, "the GPU's performance level could not be read"}; + if (std::strcmp(level, "auto")) return {kNone, "the GPU's performance level is set by hand"}; + return {kPeak, "the Steam Deck's GPU, its level amdgpu's own"}; +} + +} // namespace mkwvk::gpu_clock diff --git a/gpu/vk/gpu_wait_kinds.h b/gpu/vk/gpu_wait_kinds.h new file mode 100644 index 0000000..8567c8d --- /dev/null +++ b/gpu/vk/gpu_wait_kinds.h @@ -0,0 +1,86 @@ +// (Triforce, 11-22, the performance audit's SYNC-27, after Source A's) The host's time in calls that block on the GPU, +// counted by kind where the Vulkan calls are wrapped (vk_work.h). +// +// The log's "GPU-bound wait" counts only the waits for a recording slot: it reads about 0 while threads wait on the GPU +// elsewhere (a fence of the game's, a write-back, the present's frame fence, a device idle, a query read that waits). +// Those were timed apart or not at all. Here every blocking wait is timed in one place and summed by kind: +// fences vkWaitForFences with a timeout above 0 +// timelines vkWaitSemaphores with a timeout above 0 +// idle vkQueueWaitIdle, vkDeviceWaitIdle +// queries vkGetQueryPoolResults with VK_QUERY_RESULT_WAIT_BIT +// A poll (timeout 0, no wait bit) counts nothing. A wait made inside another counts once, as the outer one (a depth of +// the thread's). A wait inside an Exclude scope counts nothing: pacing (the driver's low-latency sleep), which is not a +// wait for the GPU's work. The totals only grow; each reader keeps its own last values (Reader), so the log's line and +// the frame trace's column each see every nanosecond. +// +// The cost is two clock reads a blocking wait. RX_VK_GPU_WAITS=0, or the lab knob nogpuwaits in its phases: the calls +// straight through, nothing counted. No Vulkan here: tests/test_gpu_waits.py compiles it. +#pragma once +#include +#include +#include +#include + +#include "lab_ab.h" + +namespace gpu_wait_kinds { + +enum Kind : int { FENCE = 0, TIMELINE = 1, IDLE = 2, QUERY = 3, KINDS = 4 }; + +struct Totals { + std::atomic ns[KINDS]{}, n[KINDS]{}; +}; +inline Totals totals; // (every thread's, since the start: they only grow) +inline thread_local int depth = 0; // (this thread is inside a counted wait) +inline thread_local int excluded = 0; // (this thread is inside an Exclude scope) + +/// Counted at all (RX_VK_GPU_WAITS, read once; on unless 0), and not in the lab knob's phases +inline bool on() { + static const bool env = [] { + const char* e = getenv("RX_VK_GPU_WAITS"); + return !(e && *e && atoi(e) == 0); + }(); + return env && !mkwvk::lab_ab::on("nogpuwaits"); +} + +/// The waits made on this thread while one of these lives count nothing +struct Exclude { + Exclude() { excluded++; } + ~Exclude() { excluded--; } + Exclude(const Exclude&) = delete; + Exclude& operator=(const Exclude&) = delete; +}; + +/// call(), and its time under `kind` when it blocks (blocks: a timeout above 0, the wait bit), unless this thread is +/// inside a counted wait or an Exclude scope, or the count is off. What call() returns +template +inline auto timed(Kind kind, bool blocks, Call&& call) -> decltype(call()) { + if (!blocks || depth || excluded || !on()) return call(); + depth++; + const auto t0 = std::chrono::steady_clock::now(); + const auto result = call(); + const int64_t ns = std::chrono::duration_cast(std::chrono::steady_clock::now() - t0).count(); + depth--; + totals.ns[kind].fetch_add(ns, std::memory_order_relaxed); + totals.n[kind].fetch_add(1, std::memory_order_relaxed); + return result; +} + +/// One reader's last values: take() gives what was counted since its last take (ns and waits by kind, either may be +/// null) and returns the nanoseconds of all kinds together +struct Reader { + int64_t last_ns[KINDS]{}, last_n[KINDS]{}; + int64_t take(int64_t* ns_by_kind, int64_t* n_by_kind) { + int64_t all = 0; + for (int k = 0; k < KINDS; k++) { + const int64_t ns = totals.ns[k].load(std::memory_order_relaxed), n = totals.n[k].load(std::memory_order_relaxed); + if (ns_by_kind) ns_by_kind[k] = ns - last_ns[k]; + if (n_by_kind) n_by_kind[k] = n - last_n[k]; + all += ns - last_ns[k]; + last_ns[k] = ns, last_n[k] = n; + } + return all; + } +}; + +} // namespace gpu_wait_kinds diff --git a/gpu/vk/guest_holes.h b/gpu/vk/guest_holes.h new file mode 100644 index 0000000..f88e1be --- /dev/null +++ b/gpu/vk/guest_holes.h @@ -0,0 +1,112 @@ +// (Triforce) The all-zero upload check's reads (vk_texture.cpp upload_all_zero): which bytes of a storage it reads, split between its +// threads. tests/test_guest_holes.py. +// +// On Linux, where /dev/udmabuf is usable, the guest window is a memfd's (host/hos_main.c). A page the game never wrote is a hole +// there: it reads as zero, but a read through the shared mapping makes the kernel allocate and zero it. The check read every byte +// of a 32 MB+ storage, so the never-written 288 MB shadow map (12288 x 12288 D16) became RAM at each Hyrule Field arrival, and +// took 200-300 ms of the replay thread on a Steam Deck. fd_data_extents asks the memfd where its data is (lseek SEEK_DATA and +// SEEK_HOLE): a hole is zero by definition, and SEEK_DATA may call a hole data, never the other way round (a page in swap is +// data), so reading only the data says the same. Anywhere else, or on any failure, one extent of the whole storage. +#pragma once + +#include +#include +#include +#include +#include +#include +#if defined(__linux__) || defined(GUEST_HOLES_TEST) // (the test: its own lseek) +#include +#include +#endif + +namespace mkwvk::guest_holes { + +struct Extent { + uint64_t lo, hi; // (bytes from the storage's start) +}; + +inline uint64_t bytes(const std::vector& ex) { + uint64_t n = 0; + for (const Extent& e : ex) n += e.hi - e.lo; + return n; +} + +/// Thread t's part of n: the extents' bytes, not their range, split into n runs of whole 4 KB (the last takes the rest); every +/// byte of the extents in exactly one part +inline std::vector share(const std::vector& ex, uint32_t t, uint32_t n) { + std::vector out; + const uint64_t total = bytes(ex); + if (!n || t >= n || !total) return out; + const uint64_t per = ((total / n) + 4095) & ~4095ull; + const uint64_t a = std::min(total, (uint64_t)t * per), b = t + 1 == n ? total : std::min(total, a + per); + uint64_t at = 0; // (bytes of the extents before e) + for (const Extent& e : ex) { + const uint64_t len = e.hi - e.lo, lo = std::max(a, at), hi = std::min(b, at + len); + if (lo < hi) out.push_back({e.lo + (lo - at), e.lo + (hi - at)}); + at += len; + if (at >= b) break; + } + return out; +} + +/// Whether any byte of p's extents is not 0, read on nthreads threads (this one among them), each stopping once one found it +inline bool any_set(const uint8_t* p, const std::vector& ex, uint32_t nthreads) { + std::atomic found{false}; + if (ex.empty()) return false; + nthreads = std::max(nthreads, 1u); + auto part = [&](uint32_t t) { + for (const Extent& e : share(ex, t, nthreads)) + for (uint64_t at = e.lo; at < e.hi && !found.load(std::memory_order_relaxed); at += 65536) { + const uint64_t end = std::min(e.hi, at + 65536); + uint64_t acc = 0; + uint64_t k = at; + for (; k + 8 <= end; k += 8) { + uint64_t w; + std::memcpy(&w, p + k, 8); + acc |= w; + } + for (; k < end; k++) acc |= p[k]; + if (acc) found.store(true, std::memory_order_relaxed); + } + }; + if (nthreads > 1) { + std::vector th; + for (uint32_t t = 1; t < nthreads; t++) th.emplace_back(part, t); + part(0); + for (auto& x : th) x.join(); + } else { + part(0); + } + return found.load(); +} + +#if defined(__linux__) || defined(GUEST_HOLES_TEST) // (the test: its own lseek) +/// The data of [off, off + len) in fd, as extents from off (none: all holes). At most max extents: past them, the rest as one. +/// False when the file cannot say (a seek failed): the caller reads it all +inline bool fd_data_extents(int fd, uint64_t off, uint64_t len, std::vector* out, size_t max = 1024) { + out->clear(); + const uint64_t end = off + len; + uint64_t pos = off; + while (pos < end) { + if (out->size() + 1 >= max) { // (the last one: the rest, whole) + out->push_back({pos - off, len}); + return true; + } + const off_t d = lseek(fd, (off_t)pos, SEEK_DATA); + if (d < 0) { + if (errno == ENXIO) return true; // (no data from pos to the file's end) + return out->clear(), false; + } + if ((uint64_t)d >= end) return true; + const off_t h = lseek(fd, d, SEEK_HOLE); + if (h < 0 || (uint64_t)h <= (uint64_t)d) return out->clear(), false; + const uint64_t hi = std::min(end, (uint64_t)h); + out->push_back({(uint64_t)d - off, hi - off}); + pos = hi; + } + return true; +} +#endif + +} // namespace mkwvk::guest_holes diff --git a/gpu/vk/helper_spin.h b/gpu/vk/helper_spin.h index b88075c..5588e67 100644 --- a/gpu/vk/helper_spin.h +++ b/gpu/vk/helper_spin.h @@ -7,7 +7,7 @@ // what it waits for comes within it, it never decides anything. The audit's trace: the helpers began a segment 0.262 ms after // they were given it, and the replay thread waited 1.74 ms a frame for them (r22-trace). tests/test_helper_spin.py // -// RX_VK_HELPER_SPIN_US=: the spin's bound (100, NX2navi's; at most 1000000); 0: no spin, straight to the wait as before +// RX_VK_HELPER_SPIN_US=: the spin's bound (20 since 11-21; 100 before, NX2navi's; at most 1000000); 0: no spin, straight to the wait as before #pragma once #include #include @@ -20,7 +20,8 @@ namespace helper_spin { /// The bound in nanoseconds that RX_VK_HELPER_SPIN_US's text gives (null: unset) inline int64_t bound_ns(const char* text) { - if (!text || !*text) return 100000; + if (!text || !*text) return 20000; // ((Triforce, 11-21) 20 us, was 100 (NX2navi's): with the game's threads the frame's limit, a + // helper spinning 100 us took its SMT sibling from them. Lab A/B 79.3 -> 80.0, 1% low +2.0) const long long us = std::atoll(text); return us <= 0 ? 0 : us >= 1000000 ? 1000000000LL : us * 1000; } diff --git a/gpu/vk/import_budget.h b/gpu/vk/import_budget.h new file mode 100644 index 0000000..9fbca38 --- /dev/null +++ b/gpu/vk/import_budget.h @@ -0,0 +1,53 @@ +// What the idle imports' valve measures the system heap against (no game, no GPU: a rule). +// +// (Triforce, 11-22, the performance audit's MEM-22, after Source A's.) An import is a pool of the game's memory the driver +// pins and counts in the system heap. vk_memory.cpp imports_give_back gives idle ones back past RX_VK_IMPORT_HIGH percent +// (85) of that heap's budget, down to RX_VK_IMPORT_LOW percent (75), but by default only where the budget is under 12 GB: +// on a PC with more, live imports pile up for the whole session (2.0 GB at a fresh Market, 6.5 GB after ten areas), and +// on Linux's amdgpu every submission pays for each one. Source A's numbers (Arch, RX 9070 XT, RADV, 32 GB, a 15,434 MB +// budget): uncapped after about two hours 221 live imports (7,392 MB), queue submit 1.11-1.14 ms, read-back waits +// 6.29-7.03 ms, 1,511 MB of swap, 44-48 fps; capped, 50-64 live (2,256-2,720 MB), 0.32-0.73 ms, 0.64-1.14 ms, no swap, +// 63-68 fps (with other settings changed in the same runs: the cap's own share is not known). +// +// So with Mesa's RADV and a budget of 12 GB or more the valve is on and measures the heap against 4,500 MB, not the whole +// budget, and a pass of it gives back 1 GB at most. Everything else is as it was: +// - a budget under 12 GB: the whole budget, on (any driver); +// - 12 GB or more on any other driver: off unless RX_VK_IMPORT_HIGH is set, then the whole budget; +// - RX_VK_IMPORT_HIGH or RX_VK_IMPORT_LOW set by the player: the whole budget, as before, RADV too. +// RX_VK_IMPORT_CAP_MB=: the cap on any driver and any budget (it turns the valve on); =0: no cap, which is RADV's +// behaviour before. A given-back import costs 5-45 ms when its pool is bound again, which is why only pools idle for 600 +// submissions go. +#pragma once +#include + +namespace import_budget { +constexpr uint64_t kMB = 1ull << 20; +constexpr uint64_t kBigBudget = 12ull << 30; // from here the valve was off by default +constexpr uint64_t kCapMb = 4500; // RADV's cap at a big budget +constexpr uint64_t kPassMost = 1ull << 30; // a capped pass gives back this much at most + +struct Rule { + bool on = false; // the heap's trigger runs + uint64_t cap = 0; // bytes the heap is measured against at most (0: the whole budget) +}; + +/// budget: the system heap's budget when the rule is first asked. high_set, low_set: the player set RX_VK_IMPORT_HIGH, +/// RX_VK_IMPORT_LOW. cap_mb: RX_VK_IMPORT_CAP_MB (-1: unset). radv: the driver is Mesa's RADV +constexpr Rule rule(uint64_t budget, bool high_set, bool low_set, int64_t cap_mb, bool radv) { + const bool before = high_set || budget < kBigBudget; // (as it was: on where the budget is small, or asked for) + if (cap_mb > 0) return Rule{true, (uint64_t)cap_mb * kMB}; + if (cap_mb == 0 || high_set || low_set || !radv || budget < kBigBudget) return Rule{before, 0}; + return Rule{true, kCapMb * kMB}; +} + +/// What the heap's use is measured against now (the budget moves; the cap does not) +constexpr uint64_t measured(const Rule& r, uint64_t budget) { + return r.cap && r.cap < budget ? r.cap : budget; +} + +/// How much a pass by the heap's trigger wants back: down to the target, and 1 GB at most where the cap is what measures +constexpr uint64_t pass_want(const Rule& r, uint64_t budget, uint64_t used, uint64_t target) { + const uint64_t want = used > target ? used - target : 0; + return r.cap && r.cap < budget && want > kPassMost ? kPassMost : want; +} +} // namespace import_budget diff --git a/gpu/vk/index_rebase.h b/gpu/vk/index_rebase.h new file mode 100644 index 0000000..7ade3d6 --- /dev/null +++ b/gpu/vk/index_rebase.h @@ -0,0 +1,30 @@ +// (Triforce, 11-21, the performance audit's DESC-12, after Source A's) An indexed draw's index buffer bound at offset 0, +// with the draw's offset given as its first index. +// +// The game's draws share one buffer at different offsets, so bind_index's skip (the same buffer, offset and type as the +// one on the command buffer) rarely held: nearly every indexed draw called vkCmdBindIndexBuffer, about 2,900 calls a frame +// in the Market. Bound at 0 with the offset as the draw's firstIndex, the skip holds until the buffer or the index type +// changes. Only where the offset is a whole number of indices that fits a first index; otherwise the draw binds at its +// offset as before. Indirect indexed draws are not touched: their first index is in the GPU's records. +// RX_VK_INDEX_REBASE=0: every draw binds at its own offset, as before. +// +// The rule alone, no Vulkan: tests/test_index_rebase.py compiles it. +#pragma once +#include +#include + +namespace index_rebase { + +/// The switch's value: on unless it is set to 0 +inline bool on(const char* asked) { return !(asked && *asked && atoi(asked) == 0); } + +/// Whether a draw whose indices start offset bytes into its buffer, unit bytes an index, can be drawn from the buffer bound +/// at 0: *first is then its first index. Not when the offset falls inside an index, nor past what a first index the draw's +/// own range is added to can hold +inline bool rebase(uint64_t offset, uint32_t unit, uint32_t* first) { + if (!unit || offset % unit || offset / unit > 0x7fffffffull) return false; + *first = (uint32_t)(offset / unit); + return true; +} + +} // namespace index_rebase diff --git a/gpu/vk/lab_ab.h b/gpu/vk/lab_ab.h index d59a55f..1b5ed73 100644 --- a/gpu/vk/lab_ab.h +++ b/gpu/vk/lab_ab.h @@ -10,12 +10,21 @@ A knob named d_ is a delay: a busy wait of microseconds at that site, at most once a frame (delay(), below): what the frame's time answers to a part of it taking longer, which is whether that part is on the frame's critical path (1 ms in for 1 ms of delay) or has slack (nothing). Any other name is a switch the code reads with on(""). + (Triforce, 11-22, the performance audit's REPLAY-51, after Source B's) A knob named e_ is a delay at every + occurrence of an event, not once a frame: a busy wait of NANOseconds at each draw (e_draw) or dispatch + (e_dispatch), first in vk_draw.cpp's draw() and dispatch() so that the profiles leave it out, on whichever thread + records, counted (each(), below). The frame's change against the time spun says how much of that path is on the frame's + critical path: 200 ns at ~3,050 draws a frame is 0.6 ms of work. One line a phase gives each knob's spins and ms a second + (each_report). d_present_after is d_present's after: the same wait just after the present's drain(0), where d_present is + just before it. Pin RX_VK_PARALLEL for e_ knobs: Automatic moves draws between threads (the replay warns). Not for players: nothing reads these without the variable. */ #pragma once +#include #include #include #include +#include #include #include @@ -32,6 +41,17 @@ inline std::atomic frame_no{0}; // (presents counted here) inline std::atomic fired[16]; // (the frame each delay last waited in) inline std::atomic pres_thread{0}; // (the thread that presents: the game's Presentation thread, in the plain mode) inline int64_t phase_frames = 300, skip_frames = 30; +// ((Triforce, 11-22, REPLAY-51) the e_ knobs: each event's wait in this phase (ns, 0: none), and what was spun) +enum Each { E_DRAW = 0, E_DISPATCH = 1, E_COUNT = 2 }; +inline const char* const each_names[E_COUNT] = {"e_draw", "e_dispatch"}; +inline std::atomic each_ns[E_COUNT]; +inline std::atomic each_spins[E_COUNT], each_spun_ns[E_COUNT]; +struct EachPhase { // (the presenting thread's: the phase being counted) + int knob = 0; + std::chrono::steady_clock::time_point since{}; +}; +inline EachPhase each_phase; +inline char each_line[256]; // (the line of a phase just over, for the replay to log; empty: none) inline void init() { if (n_knobs >= 0) return; @@ -53,6 +73,53 @@ inline void init() { if (const char* s = getenv("RX_VK_LAB_AB_SKIP")) skip_frames = atoll(s); n_knobs = n; } +/// ((Triforce, 11-22, REPLAY-51) Whether any knob is an e_ one (the replay warns once unless RX_VK_PARALLEL is pinned) +inline bool any_each() { + for (int i = 0; i < n_knobs; i++) + for (const char* name : each_names) + if (!strcmp(knobs[i].name, name)) return true; + return false; +} +/// (phase_tick, the presenting thread) The phase is knob k (0: none): at a change, the line of the phase that ended (each_line, +/// when it had an e_ knob: its spins and ms a second), and the events' waits for this one +inline void each_tick(int k) { + if (k == each_phase.knob && each_phase.since.time_since_epoch().count()) return; + const auto now = std::chrono::steady_clock::now(); + each_line[0] = 0; + const int was = each_phase.knob; + if (was > 0 && each_phase.since.time_since_epoch().count()) { + const double s = std::chrono::duration(now - each_phase.since).count(); + for (int e = 0; e < E_COUNT; e++) { + if (strcmp(knobs[was - 1].name, each_names[e])) continue; + const uint64_t spins = each_spins[e].exchange(0, std::memory_order_relaxed); + const uint64_t ns = each_spun_ns[e].exchange(0, std::memory_order_relaxed); + snprintf(each_line, sizeof each_line, "vk: lab A/B: phase %s:%lld over: %.0f waits a second, %.2f ms a second spun " + "(%llu waits in %.1f s)", each_names[e], (long long)knobs[was - 1].value, s > 0 ? spins / s : 0.0, + s > 0 ? ns / 1e6 / s : 0.0, (unsigned long long)spins, s); + } + } + for (int e = 0; e < E_COUNT; e++) + each_ns[e].store(k > 0 && !strcmp(knobs[k - 1].name, each_names[e]) ? std::max(0, knobs[k - 1].value) : 0, + std::memory_order_relaxed); + each_phase.knob = k, each_phase.since = now; +} +/// At every occurrence of an event (first in draw() and dispatch()): one relaxed load without its knob; with it, a busy wait +/// of the knob's nanoseconds, counted +inline void each(int event) { + const int64_t ns = each_ns[event].load(std::memory_order_relaxed); + if (!ns) return; + const auto t0 = std::chrono::steady_clock::now(), until = t0 + std::chrono::nanoseconds(ns); + auto now = t0; + while (now < until) { +#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) + __builtin_ia32_pause(); +#endif + now = std::chrono::steady_clock::now(); + } + each_spins[event].fetch_add(1, std::memory_order_relaxed); + each_spun_ns[event].fetch_add((uint64_t)std::chrono::duration_cast(now - t0).count(), + std::memory_order_relaxed); +} /// Once a present, on the presenting thread: the phase this frame belongs to (the trace's column) inline int phase_tick() { init(); @@ -61,6 +128,7 @@ inline int phase_tick() { const uint64_t phase = f / (uint64_t)phase_frames; // (0 none, 1 knob 1, 2 none, 3 knob 2, ...) const int k = (phase & 1) ? (int)((phase / 2) % (uint64_t)n_knobs) + 1 : 0; active.store(k, std::memory_order_relaxed); + each_tick(k); // ((Triforce, 11-22, REPLAY-51) the e_ knobs' waits for this phase, and the line of the one before) const bool settling = (int64_t)(f % (uint64_t)phase_frames) < skip_frames; return settling ? -(k + 1) : k; // (-1: none, settling; -(k + 1): knob k, settling) } diff --git a/gpu/vk/late_inline.h b/gpu/vk/late_inline.h new file mode 100644 index 0000000..82ac7b0 --- /dev/null +++ b/gpu/vk/late_inline.h @@ -0,0 +1,49 @@ +// (Triforce, 11-22, the performance audit's MEM-80, after Source B's) Which of take_capture's no-wait rules for late pages run. +// +// A late page is one the CPU wrote while a submission still on the GPU writes it back (vk_memory.cpp capture_make). By default +// every late page is waited for (RX_VK_LATE_QUEUE: at the queue's next submit) and taken again. RX_VK_LATE_NEW=1 (the lab +// knob latenew) turns four rules on that take the wait away from some of them: pages of pools with no mirror skipped, pages +// with the bytes they were last synced with left as they are, pages written back in part synced in their other parts (the +// partial rule, late_partial), pages written back whole put off. Each brought brightness steps back at the Castle Town save. +// +// RX_VK_LATE_INLINE=1 (the lab knob lateinline), OFF unless set: the partial rule ALONE, Source B's analogue (the CPU's +// bytes go in, the GPU's ranges stay the GPU's, nothing waits); the other three stay off, and the pages they would take +// (no mirror, texture storage, whole write-backs, unknown ranges) are waited for as before. +// +// And the partial rule asks which ranges are still on the GPU against the GPU's progress capture_make read before its +// copies, not a second read at the take: a write-back that landed between the copy and the take is then still counted in +// flight, so its range is not synced from the older copy (the bug Source B fixed with the same change). This is so under +// RX_VK_LATE_NEW too. +// +// The rule alone: no Vulkan, no game. tests/test_late_inline.py. +#pragma once +#include + +namespace late_inline { + +/// RX_VK_LATE_INLINE: a number other than 0 on, 0 off; unset: the lab knob lateinline (on only in its phases) +inline bool on(const char* env, bool knob) { + if (env) return std::atoi(env) != 0; + return knob; +} + +struct Rules { + bool any = false; // (any rule: the late pages looked at one by one; else every one waited for, as by default) + bool skip = false; // (a page of a pool with no mirror: not waited for) + bool same = false; // (a page with the bytes it was last synced with: left as it is) + bool part = false; // (a page written back in part: its other parts synced, no wait) + bool defer = false; // (a page written back whole: put off) +}; +/// late_new: RX_VK_LATE_NEW or its knob; inline_on: on() above; same_on, part_on, defer_on: RX_VK_LATE_SAME, RX_VK_LATE_PART, +/// RX_VK_LATE_DEFER as read today (on, on, off unless set) +inline Rules rules(bool late_new, bool inline_on, bool same_on, bool part_on, bool defer_on) { + Rules r; + r.any = late_new || inline_on; + r.skip = late_new; + r.same = late_new && same_on; + r.part = r.any && part_on; + r.defer = late_new && defer_on; + return r; +} + +} // namespace late_inline diff --git a/gpu/vk/linux_pressure.h b/gpu/vk/linux_pressure.h new file mode 100644 index 0000000..99a9939 --- /dev/null +++ b/gpu/vk/linux_pressure.h @@ -0,0 +1,153 @@ +// (Triforce, 11-22, the performance audit's MEM-141, after Source A's) A memory pressure hold for Linux and the Steam Deck that +// counts the swap too (vk_memory.cpp steps it from memory_ram_watch, off Windows; this is the rule, plain code: +// tests/test_linux_pressure.py runs it anywhere). +// +// The RAM rules looked at MemAvailable alone. A Linux PC with swap keeps MemAvailable up by pushing the game's pages out: +// on a 16 GB Steam Deck Source A saw the swap fill (4,036 MB at the peak, 0 MB free at the lowest) while MemAvailable said +// little, and the stalls were the pages coming back. So the swap left counts: the PC is short when MemAvailable is under +// 1536 MB (the imports' own line), or when MemAvailable and SwapFree together are under the reserve (RX_VK_SWAP_RESERVE_MB). +// While it is short a hold is on and the renderer gives memory back sooner: +// - a capture buffer is kept as a spare only up to 32 MB of spares (not 768); +// - the staging arenas the GPU is done with are freed when the hold begins, then after 5 s unused (not 30); +// - malloc_trim is asked every 600 presents from 64 MB free (not every 1800 from 192); +// - idle imports are given back, in all of one episode at most as many bytes as the PC was short at its worst (not up to +// 1 GB a pass: the thrash of imports given back and made again was seen here before), and still only after 3,600 idle +// submissions; +// - on an integrated GPU, whose "VRAM" is the same RAM, the texture budget is less by what the PC is short (never under +// 512 MB), and comes back 256 MB a sample. +// The hold ends once MemAvailable and the sum are both 256 MB clear of their lines. A PC without swap, or one whose swap +// cannot be read, is never short by the sum: not known is not short. The program modules are let go by the program pack +// already (vk_shader.cpp), so the hold has no valve for them. +// +// RX_VK_SWAP_RESERVE_MB=: the reserve; 0 or not set: no hold at all, as before. Source A's is 2560; it is off here until +// a Steam Deck run has shown it. RX_VK_TEST_SWAP_MB= plays that much swap free, as RX_VK_TEST_AVAIL_MB plays MemAvailable. +#pragma once + +#include +#include +#include +#include + +namespace linux_pressure { + +constexpr uint64_t kMB = 1ull << 20; +constexpr uint64_t kAvailLine = 1536 * kMB; // (short under this much MemAvailable, whatever the swap) +constexpr uint64_t kClear = 256 * kMB; // (the hold ends once both are this far clear of their lines) +constexpr uint64_t kSourceReserve = 2560 * kMB; // (Source A's reserve: MemAvailable + SwapFree) +constexpr uint64_t kSampleEvery = 30; // (presents between two looks) +constexpr uint64_t kSpareCap = 32 * kMB; // (capture spares while held) +constexpr int64_t kArenaIdleS = 5; // (a staging arena unused this long goes, while held) +constexpr uint64_t kTrimEvery = 600; // (presents between two malloc_trim looks, while held) +constexpr uint64_t kTrimMB = 64; // (free MB in the arenas from which malloc_trim is asked, while held) +constexpr uint64_t kBudgetFloor = 512 * kMB; // (an integrated GPU's textures are never cut under this) +constexpr uint64_t kBudgetBack = 256 * kMB; // (the cut comes back this much a sample) + +/// RX_VK_SWAP_RESERVE_MB's text as bytes; 0 (not set, empty, 0, not a number): no hold +inline uint64_t reserve_of(const char* e) { + const long long mb = e && *e ? atoll(e) : 0; + return mb > 0 ? (uint64_t)mb * kMB : 0; +} + +/// What /proc/meminfo said. A field that is not there is not known (has_*), never 0 +struct Meminfo { + uint64_t total = 0, avail = 0, swap_total = 0, swap_free = 0; + bool has_total = false, has_avail = false, has_swap_total = false, has_swap_free = false; +}; + +/// /proc/meminfo's text ("Name: kB" a line) +inline Meminfo parse(const char* text) { + Meminfo m; + auto field = [text](const char* name, uint64_t* out) { + const size_t len = strlen(name); + for (const char* p = text; p && *p; p = strchr(p, '\n'), p = p ? p + 1 : p) { + if (strncmp(p, name, len) || p[len] != ':') continue; + const char* v = p + len + 1; + while (*v == ' ' || *v == '\t') v++; + if (*v < '0' || *v > '9') return false; + *out = strtoull(v, nullptr, 10) << 10; // (kB) + return true; + } + return false; + }; + m.has_total = field("MemTotal", &m.total); + m.has_avail = field("MemAvailable", &m.avail); + m.has_swap_total = field("SwapTotal", &m.swap_total); + m.has_swap_free = field("SwapFree", &m.swap_free); + return m; +} + +/// One look at the PC's memory. swap_known: there is swap and what is free of it was read +struct Sample { + bool avail_known = false, swap_known = false; + uint64_t avail = 0, swap_free = 0; +}; + +/// The sample of a meminfo, with the lab's played values over it (negative: not played). Swapless (SwapTotal 0), or SwapFree +/// or SwapTotal unreadable: the swap is not known. A played swap counts as known +inline Sample sample_of(const Meminfo& m, long long test_avail_mb = -1, long long test_swap_mb = -1) { + Sample s; + s.avail_known = m.has_avail, s.avail = m.avail; + s.swap_known = m.has_swap_total && m.has_swap_free && m.swap_total > 0, s.swap_free = s.swap_known ? m.swap_free : 0; + if (test_avail_mb >= 0) s.avail_known = true, s.avail = (uint64_t)test_avail_mb * kMB; + if (test_swap_mb >= 0) s.swap_known = true, s.swap_free = (uint64_t)test_swap_mb * kMB; + return s; +} + +/// How many bytes the PC is short: the larger of what MemAvailable lacks to its line and what the sum lacks to the reserve. +/// 0: not short, or not known +inline uint64_t shortfall(const Sample& s, uint64_t reserve) { + if (!reserve || !s.avail_known) return 0; + const uint64_t by_avail = s.avail < kAvailLine ? kAvailLine - s.avail : 0; + const uint64_t sum = s.avail + s.swap_free; + const uint64_t by_sum = s.swap_known && sum < reserve ? reserve - sum : 0; + return std::max(by_avail, by_sum); +} + +/// Both lines cleared by kClear (the hold may end). Not known: clear +inline bool clear(const Sample& s, uint64_t reserve) { + if (!reserve || !s.avail_known) return true; + if (s.avail < kAvailLine + kClear) return false; + return !s.swap_known || s.avail + s.swap_free >= reserve + kClear; +} + +/// The hold: on from a short sample until a clear one. One episode is one stretch of it +struct Hold { + bool held = false; + uint64_t deepest = 0; // (the episode's largest shortfall) + uint64_t given = 0; // (the bytes of imports given back in this episode) + uint64_t now = 0; // (the last sample's shortfall) + uint64_t cut = 0; // (an integrated GPU's texture budget is less by this) + uint64_t episodes = 0; + + enum Step { Same, Entered, Left }; + /// One sample. reserve 0: never held + Step step(const Sample& s, uint64_t reserve) { + const uint64_t lacking = shortfall(s, reserve); + Step r = Same; + if (!held && lacking) held = true, deepest = 0, given = 0, episodes++, r = Entered; + else if (held && clear(s, reserve)) held = false, r = Left; + now = held ? lacking : 0; + if (held) deepest = std::max(deepest, lacking); + // (the budget's cut: down at once to what is lacking, back up kBudgetBack a sample) + cut = now > cut ? now : std::max(now, cut > kBudgetBack ? cut - kBudgetBack : 0); + return r; + } + /// The bytes of idle imports this episode may still give back: its largest shortfall, less what it gave + uint64_t episode_want() const { return held && deepest > given ? deepest - given : 0; } + void gave(uint64_t bytes) { given += bytes; } +}; + +/// The valves' sizes, held or not +inline uint64_t spare_cap(bool held, uint64_t usual) { return held ? std::min(usual, kSpareCap) : usual; } +/// (idle_s 0: the player keeps the arenas: kept while held too) +inline int64_t arena_idle_s(bool held, int64_t usual) { return held && usual > kArenaIdleS ? kArenaIdleS : usual; } +inline uint64_t trim_every(bool held, uint64_t usual) { return held ? std::min(usual, kTrimEvery) : usual; } +/// (trim_mb 0: never, held or not) +inline uint64_t trim_mb(bool held, uint64_t usual) { return held && usual ? std::min(usual, kTrimMB) : usual; } +/// An integrated GPU's texture budget less the cut: never under the floor, never more than it was +inline uint64_t cut_budget(uint64_t budget, uint64_t cut) { + if (!cut || budget <= kBudgetFloor) return budget; + return std::max(kBudgetFloor, budget > cut ? budget - cut : 0); +} + +} // namespace linux_pressure diff --git a/gpu/vk/lock_census.h b/gpu/vk/lock_census.h new file mode 100644 index 0000000..2c79c6f --- /dev/null +++ b/gpu/vk/lock_census.h @@ -0,0 +1,213 @@ +// (Triforce, 11-22, the performance audit's THREAD-68, after Source B's) The lock census: who holds the lock a thread waits for. +// +// The log has totals of the waits for the renderer's locks (the memory sync's lock wait, the virtual cores' line, the order lock's +// slow takes), none of which names the holder. With RX_VK_LOCK_CENSUS=1 a lock that is busy when a thread comes for it notes the +// waiter, the thread that held it, and how long the wait was; every 10 s a thread of the census's own logs the 12 pairs that +// waited longest: +// vk: lock census: Presentation Thread <- MainThread (replay_mu) 17.2 ms/s (2900/s, 6 us each) +// Counted: replay_mu (DrainingMutex), sync_order_mu (OrderMutex), held_mu, and in vk_memory.cpp pools_mu, sync_mu and cap_mu. +// Only the counting is ported: Source B's also samples the holders' stacks, which disturbs a millisecond or two a second. +// +// Off (unset, or 0): every lock is taken exactly as before, behind one branch on a value read once. On: a free lock is one +// try_lock and a store of its holder; only a busy lock reads the clock, twice. The holder is read without the lock, so a holder +// that lets go at that moment can be misnamed, never more. The census's own mutex is taken last, with the counted lock held, and +// nothing that holds it ever takes a counted lock. With RX_VK_LAB_AB naming a knob lockcensus it counts only in that knob's phases. +// +// No windows.h here (vk_recorder.h, which includes this, has none): a thread's name is read through g_thread_name, which +// vk_replay.cpp sets. tests/test_lock_census.py cuts DrainingMutex and OrderMutex out and compiles them with this. +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace mkwvk { +void logf(const char* fmt, ...); // (vk_main.cpp) + +namespace lock_census { + +/// RX_VK_LOCK_CENSUS, read once: 1 (any number but 0) counts +inline bool enabled() { + static const bool v = [] { + const char* e = getenv("RX_VK_LOCK_CENSUS"); + return e && atoi(e) != 0; + }(); + return v; +} +inline std::atomic phase_on{true}; // (lab: false outside the knob's phases) +inline bool counting() { return enabled() && phase_on.load(std::memory_order_relaxed); } + +/// The clock a busy lock reads (a test counts the reads through it) +inline int64_t steady_ns() { + return std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); +} +inline int64_t (*now_ns)() = steady_ns; +inline std::atomic period_ms{10000}; // (between two reports) +constexpr int kTop = 12; + +struct Pair { + int64_t ns = 0, n = 0; +}; +/// The census's own state. Names are kept as pointers to strings that are never freed, one per distinct name +struct Book { + std::mutex mu; + std::map names; + std::map, Pair> pairs; // (waiter, holder, lock) + bool reporter = false; + int64_t since_ns = 0; +}; +inline Book& book() { + static Book* b = new Book; // (never destroyed: a thread may count while the process ends) + return *b; +} +/// the one pointer for a name (book().mu held) +inline const char* intern(Book& b, const char* name) { + auto it = b.names.find(name); + if (it == b.names.end()) { + char* kept = new char[strlen(name) + 1]; + strcpy(kept, name); + it = b.names.emplace(name, kept).first; + } + return it->second; +} + +/// A thread as the census names it: its description (the game's threads and the renderer's own are named), else its number. The +/// slot is never freed, so a lock's note of its holder can be read after that thread has ended +struct Slot { + std::atomic name{"?"}; + int64_t named_ns = 0; // (its own thread's: when the name was last read) +}; +/// (set by vk_replay.cpp, which has windows.h: the calling thread's description into out, left empty when it has none; and what +/// the reporting thread does first, which names it) +inline void (*g_thread_name)(char* out, size_t cap) = nullptr; +inline void (*g_report_start)() = nullptr; +inline void name_slot(Slot* s) { + char text[96] = ""; + if (g_thread_name) g_thread_name(text, sizeof text); + if (!text[0]) snprintf(text, sizeof text, "thread %zu", std::hash()(std::this_thread::get_id()) % 100000); + Book& b = book(); + std::lock_guard lk(b.mu); + s->name.store(intern(b, text), std::memory_order_relaxed); +} +/// the calling thread's slot; named at its first use, and again by `again_at` (a waiter's clock read: a thread named after its +/// first lock is not "thread 1234" for the whole session) when a second has passed +inline const Slot* me(int64_t again_at = 0) { + thread_local Slot* mine = nullptr; + if (!mine) { + mine = new Slot; + mine->named_ns = again_at; // (0 on a free lock, which reads no clock: named again at this thread's first wait) + name_slot(mine); + } else if (again_at && again_at - mine->named_ns > 1000000000) { + mine->named_ns = again_at; + name_slot(mine); + } + return mine; +} + +/// The pairs counted since the last call, the longest waited first, kTop at most, as the log's lines; all counts start again +inline std::vector report(int64_t now) { + Book& b = book(); + std::vector, Pair>> rows; + double seconds = 0; + { + std::lock_guard lk(b.mu); + rows.assign(b.pairs.begin(), b.pairs.end()); + b.pairs.clear(); + seconds = (double)(now - b.since_ns) / 1e9; + b.since_ns = now; + } + if (seconds <= 0) seconds = 1e-9; + std::sort(rows.begin(), rows.end(), [](const auto& x, const auto& y) { return x.second.ns > y.second.ns; }); + std::vector out; + for (size_t i = 0; i < rows.size() && i < (size_t)kTop; i++) { + const auto& [key, p] = rows[i]; + char line[320]; + snprintf(line, sizeof line, "vk: lock census: %s <- %s (%s) %.1f ms/s (%.0f/s, %.0f us each)", std::get<0>(key), std::get<1>(key), + std::get<2>(key), (double)p.ns / 1e6 / seconds, (double)p.n / seconds, p.n ? (double)p.ns / 1e3 / (double)p.n : 0.0); + out.emplace_back(line); + } + return out; +} +inline void report_main() { + if (g_report_start) g_report_start(); + for (;;) { + // (slept in steps, the period read at each: a test shortens it while this thread sleeps) + for (int slept = 0; slept < period_ms.load(std::memory_order_relaxed); slept += 50) + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + for (const std::string& line : report(now_ns())) logf("%s", line.c_str()); + } +} +/// A wait is over: self waited ns for the lock `lock`, which `holder` held when it came (null: not known). The first one starts +/// the reporting thread +inline void waited(const Slot* self, const Slot* holder, const char* lock, int64_t ns, int64_t now) { + Book& b = book(); + bool start = false; + { + std::lock_guard lk(b.mu); + if (!b.since_ns) b.since_ns = now - ns; + Pair& p = b.pairs[{self->name.load(std::memory_order_relaxed), holder ? holder->name.load(std::memory_order_relaxed) : intern(b, "?"), + intern(b, lock)}]; + p.ns += ns, p.n++; + if (!b.reporter) b.reporter = start = true; + } + if (start) std::thread(report_main).detach(); +} + +/// m.lock() for a counted lock. owner: where that lock keeps its holder (set here, cleared by unlocking()) +template +inline void lock(M& m, std::atomic& owner, const char* name) { + if (!counting()) { // (off: the lock as it was) + m.lock(); + return; + } + if (m.try_lock()) { // (free: no clock read) + owner.store(me(), std::memory_order_relaxed); + return; + } + const Slot* holder = owner.load(std::memory_order_relaxed); + const int64_t t0 = now_ns(); + m.lock(); + const int64_t t1 = now_ns(); + const Slot* self = me(t1); + owner.store(self, std::memory_order_relaxed); + waited(self, holder, name, t1 - t0, t1); +} +/// m.try_lock() for a counted lock +template +inline bool try_lock(M& m, std::atomic& owner) { + if (!m.try_lock()) return false; + if (counting()) owner.store(me(), std::memory_order_relaxed); + return true; +} +/// just before a counted lock's unlock: it has no holder +inline void unlocking(std::atomic& owner) { + if (owner.load(std::memory_order_relaxed)) owner.store(nullptr, std::memory_order_relaxed); +} + +/// A std::mutex the census counts, for std::lock_guard and std::unique_lock +struct CensusMutex { + const char* name; + std::mutex m; + std::atomic owner{nullptr}; + explicit CensusMutex(const char* n) : name(n) {} + void lock() { lock_census::lock(m, owner, name); } + bool try_lock() { return lock_census::try_lock(m, owner); } + void unlock() { + unlocking(owner); + m.unlock(); + } +}; + +} // namespace lock_census +} // namespace mkwvk diff --git a/gpu/vk/mem_parts.h b/gpu/vk/mem_parts.h index ed95c86..4883a53 100644 --- a/gpu/vk/mem_parts.h +++ b/gpu/vk/mem_parts.h @@ -18,6 +18,7 @@ #include #include +#include #include #include @@ -210,5 +211,30 @@ inline uint64_t own_for_keep(uint64_t ws, uint64_t cache_read, bool amd) { if (amd) return ws; return ws > cache_read ? ws - cache_read : 0; } +/// (11-22, after Casual Gamer's fix) Whether the game is told its mapped pages are used (host/hos_mem.c query_used), from the +/// host's switches as it reads them: HOS_MEM_QUERY_USED not "0", and the census on (hos_mem_census_on: HOS_MEM_CENSUS not 0 or +/// less, or HOS_MEM_GIVE_BACK above 0) +inline bool query_used_on(const char* query_used, const char* census, const char* give_back) { + const bool census_on = !(census && atoi(census) <= 0) || (give_back && atoi(give_back) > 0); + return !(query_used && query_used[0] == '0' && !query_used[1]) && census_on; +} +/// (11-22) Whether the precompile keeps every pipeline for a PC of under 20 GB (vk_draw.cpp precompile_keeps_all): on a discrete +/// GPU with the game told its mapped pages are used (query_on), which lets old areas go; total 0: not known +inline bool keep_all_small_pc(bool discrete, uint64_t total, bool query_on) { + return discrete && query_on && total && total < (20ull << 30); +} + +/// (11-22, after Casual Gamer's fix) Whether the retired imports kept for good (vk_memory.cpp retired_imports_give_back: Windows, +/// AMD's and Intel's drivers) are given back all the same: the system heap past high percent of its budget (used of budget; +/// a budget of 0: not known), or an import refused for want of room within the last kRefusedHoldMs (refused_ms: when, as now_ms +/// counts; 0: never) +constexpr int64_t kRefusedHoldMs = 60000; +inline bool retired_pressed(uint64_t used, uint64_t budget, int high, int64_t now_ms, int64_t refused_ms) { + return (budget && high > 0 && used > budget / 100 * (uint64_t)high) || (refused_ms && now_ms - refused_ms < kRefusedHoldMs); +} +/// (11-22, after Casual Gamer's fix) Whether the capture's spare buffers are held to 32 MB (vk_memory.cpp cap_spare_max): on a +/// card the driver allows 6.5 GB or less (small_card), or on a PC of under 20 GB of RAM (ram_total; 0: not known) +constexpr uint64_t kSpareRamBytes = 20ull << 30; +inline bool spare_capped(bool small_card, uint64_t ram_total) { return small_card || (ram_total && ram_total < kSpareRamBytes); } } // namespace mem_parts diff --git a/gpu/vk/memory_priority.h b/gpu/vk/memory_priority.h new file mode 100644 index 0000000..ca55636 --- /dev/null +++ b/gpu/vk/memory_priority.h @@ -0,0 +1,73 @@ +// VK_EXT_memory_priority: which device memory the system should keep in video memory when the card is short (no game, no +// GPU: the switch's reading and the tiers). tests/test_memory_priority.py +// +// (Triforce, 11-22, the performance audit's TEX-47, after Source A's.) When a card is short of video memory, Windows moves +// some of a program's device memory to system memory, which the GPU then reads over the bus, and it picks what goes. With +// this extension a program ranks its memory from 0 to 1: the lower goes first. Source A gives the render targets 1.0, +// the sampled textures 0.25 and the rest 0.5, and measured it on an RTX 4090 with the video memory taken by other +// programs and the textures held to an 8 GB card's blocks: with 1.3 GB moved out, the GPU's median frame was 109 to +// 179 ms unset and 24 to 29 ms on. The textures' low rank is the lever: with them at 0.5 (`1,0.5`) it ended at 68 ms. +// +// Off unless RX_VK_MEMORY_PRIORITY is set: no small card has run it, drivers may ignore it, and in Source A's runs the +// wrong tiers were worse than none. Unset, the extension is not asked for and nothing here is used. +// RX_VK_MEMORY_PRIORITY unset, empty, 0, off, no: off +// 1: render targets 1.0, textures 0.25, the rest 0.5 +// ,: those two, each from 0 to 1; the rest 0.5 +// anything else (one number that is not 0 or 1, three numbers, a number outside 0 to 1): off +// With the extension on, the allocator (VMA) gives every pool and every allocation of its own the priority its create +// info names, and one that names none gets 0, below the textures: so every one in vk_vma.cpp names its tier. +#pragma once +#include +#include +#include +#include + +namespace memory_priority { + +inline constexpr float kDefault = 0.5f; // (what a driver gives memory that names no priority) + +struct Tiers { + bool on = false; + float targets = kDefault, textures = kDefault, other = kDefault; +}; + +/// RX_VK_MEMORY_PRIORITY as written (see the top of the file); what cannot be read is off +inline Tiers parse(const char* env) { + Tiers t; + if (!env) return t; + while (*env == ' ' || *env == '\t') env++; + char word[32] = {}; + size_t n = std::strlen(env); + while (n && (env[n - 1] == ' ' || env[n - 1] == '\t')) n--; + if (!n || n >= sizeof word) return t; + for (size_t i = 0; i < n; i++) word[i] = (char)std::tolower((unsigned char)env[i]); + if (!std::strcmp(word, "0") || !std::strcmp(word, "off") || !std::strcmp(word, "no")) return t; + if (!std::strcmp(word, "1")) { + t.on = true, t.targets = 1.0f, t.textures = 0.25f; + return t; + } + // (two numbers and a comma between them, nothing else) + char* end = nullptr; + const double a = std::strtod(word, &end); + if (end == word || *end != ',') return t; + const char* second = end + 1; + const double b = std::strtod(second, &end); + if (end == second || *end) return t; + if (!std::isfinite(a) || !std::isfinite(b) || a < 0 || a > 1 || b < 0 || b > 1) return t; + t.on = true, t.targets = (float)a, t.textures = (float)b; + return t; +} + +/// The tiers a run uses: the switch's where the device has the extension and its feature, else none (all the default) +inline Tiers active(const char* env, bool device_has_it) { return device_has_it ? parse(env) : Tiers{}; } + +/// What a piece of device memory is for. Held: the lab's ballast (RX_VK_TEST_CARD_MB), which must stay on the card +enum class Kind { Target, Texture, Other, Held }; + +/// Its priority under the tiers; the default for everything while they are off +inline float of(const Tiers& t, Kind k) { + if (!t.on) return kDefault; + return k == Kind::Target ? t.targets : k == Kind::Texture ? t.textures : k == Kind::Held ? 1.0f : t.other; +} + +} // namespace memory_priority diff --git a/gpu/vk/mesa_spilling.h b/gpu/vk/mesa_spilling.h new file mode 100644 index 0000000..c4a534e --- /dev/null +++ b/gpu/vk/mesa_spilling.h @@ -0,0 +1,38 @@ +// Mesa's Intel shader compiler told to spill registers at rate 2, before the driver is loaded (no game, no GPU: a rule). +// +// (Triforce, 11-22, the performance audit's PIPE-55, after Source A's.) Mesa's drivers read their options (driconf) from +// the environment by name. shader_spilling_rate is read by the Intel compiler alone: after a register allocation that +// failed, it spills that many times more registers at once, so a program that does not fit is built in fewer, larger +// steps. A program that does not spill builds the same. Source A's numbers on an Iris Xe (i9-13900H, 32 GB, Mesa 26.2.4): +// one compute program not built after 20 minutes and 13 GB at the driver's own rate, 32 s and 1.7 GB at rate 2; another +// 531 s and 7.6 GB against 87 s and 4 GB; rate 3 took 165 s there and rate 4 did not finish in 8 minutes. The driver's +// own estimate for a spilling program's speed at rate 2 is from 12% better to 53% worse. +// +// The Linux build sets it in create_instance (vk_main.cpp) without overwriting, so a value the player set wins. +// RX_VK_NO_SPILLING_RATE=1 leaves the driver as it is. The Windows build sets nothing: under Proton the driver reads the +// Linux environment, which the Windows build cannot write, and those players put shader_spilling_rate=2 in the launch +// options. +// Nothing here is read by NVIDIA's or AMD's drivers, RADV included. +#pragma once +#include + +namespace mesa_spilling { +constexpr const char* kName = "shader_spilling_rate"; +constexpr const char* kRate = "2"; + +/// The build sets the rate: off Windows, unless RX_VK_NO_SPILLING_RATE is set to something other than 0. off: its value +inline bool wanted(bool windows_build, const char* off) { + return !windows_build && !(off && std::atoi(off) != 0); +} + +/// What the driver will read once the build has done its part: the player's own value where there is one (set without +/// overwriting), else the rate where it is wanted, else nullptr (the driver's own rate) +inline const char* effective(const char* players, bool is_wanted) { + return players ? players : is_wanted ? kRate : nullptr; +} + +/// The log names the rate on Mesa's Intel driver alone +constexpr bool told(bool mesa_intel) { + return mesa_intel; +} +} // namespace mesa_spilling diff --git a/gpu/vk/mirror_budget.h b/gpu/vk/mirror_budget.h new file mode 100644 index 0000000..354ce3f --- /dev/null +++ b/gpu/vk/mirror_budget.h @@ -0,0 +1,27 @@ +// (Triforce) The VRAM mirrors' budget by the card (vk_memory.cpp mirror_budget; RX_VK_MIRROR_MB= goes over it). Plain code with +// no Vulkan calls: tests/test_mirror_budget.py runs it on made-up cards. +// +// - A discrete card over 4.5 GB: a quarter of its video memory, at most 3 GB. On a 6 GB card (RTX 3060 Laptop, Hyrule Field, +// whole mirrors) 512 MB of mirrors gave 17.6 fps, 1024 MB 30.7 and a quarter (1498 MB) 39.2: the pools left in system memory +// were read over PCIe each frame. The eighth those cards had (768 MB) sat between two sizes measured as losses. Textures on +// cards of 8.5 GB or less are 45% of the allowance and give memory back at 90% and 97% of it (vk_texture.cpp), the room a +// quarter needs; the eighth was set for a 6 GB laptop before that rule, when a quarter and the textures went past the card. +// - 4.5 GB or less, and an integrated GPU up to 6.5 GB: an eighth (no smaller size was measured on a real small card, and an +// integrated GPU's "video memory" is the system's). +// - Video memory not known: 3 GB. +#pragma once + +#include +#include + +namespace mkwvk::mirror_size { + +constexpr uint64_t kMB = 1ull << 20; + +constexpr uint64_t bytes(uint64_t vram, bool discrete) { + if (!vram) return 3072 * kMB; + if (vram <= 4608 * kMB || (!discrete && vram <= 6656 * kMB)) return vram / 8; + return std::min(vram / 4, 3072 * kMB); +} + +} // namespace mkwvk::mirror_size diff --git a/gpu/vk/mirror_spare.h b/gpu/vk/mirror_spare.h new file mode 100644 index 0000000..6fa4ce4 --- /dev/null +++ b/gpu/vk/mirror_spare.h @@ -0,0 +1,42 @@ +// (Triforce, 11-21, the performance audit's MEM-27, after Source A's: its first step) What the mirrors' sparse buffers cost. +// +// A mirror in pieces is a sparse buffer: address space, not memory, yet the driver charges by its size. Source A measured +// about 0.06 ms a MB to create one and 0.07 to destroy it (NVIDIA 617.42), 52 to 64 ms batches of destructions at an area +// change, and the game frees its pools there and makes them again at the same sizes. Its answer is to keep a freed +// mirror's buffer, unbound, for the next mirror of exactly that size. Before that is built here, this counts: the creates +// and destroys, their time, the longest, and how many creates a buffer freed earlier, of exactly their size, could have +// served. The line "vk: mirrors' sparse buffers" every 1200 submissions has them (vk_memory.cpp). +// +// The count alone, no Vulkan: tests/test_mirror_spare.py compiles it. +#pragma once +#include +#include + +namespace mirror_spare { + +struct Counts { + uint64_t creates = 0, destroys = 0, servable = 0; // (servable: creates a freed buffer of their size was there for) + double create_ms = 0, destroy_ms = 0, longest_create_ms = 0, longest_destroy_ms = 0; + uint64_t create_bytes = 0, destroy_bytes = 0, over = 0; // (over: calls of more than kLongMs) + std::map freed; // (sizes freed and not yet matched by a create) +}; +inline constexpr double kLongMs = 4.0; + +/// A sparse buffer of size bytes was destroyed in ms +inline void destroyed(Counts& c, uint64_t size, double ms) { + c.destroys++, c.destroy_ms += ms, c.destroy_bytes += size, c.over += ms > kLongMs; + if (ms > c.longest_destroy_ms) c.longest_destroy_ms = ms; + c.freed[size]++; +} +/// A sparse buffer of size bytes was created in ms; true when one freed before, of exactly that size, could have served +inline bool created(Counts& c, uint64_t size, double ms) { + c.creates++, c.create_ms += ms, c.create_bytes += size, c.over += ms > kLongMs; + if (ms > c.longest_create_ms) c.longest_create_ms = ms; + auto it = c.freed.find(size); + if (it == c.freed.end()) return false; + if (!--it->second) c.freed.erase(it); + c.servable++; + return true; +} + +} // namespace mirror_spare diff --git a/gpu/vk/mods_entry_anim.h b/gpu/vk/mods_entry_anim.h index 56759ba..44764a3 100644 --- a/gpu/vk/mods_entry_anim.h +++ b/gpu/vk/mods_entry_anim.h @@ -16,6 +16,7 @@ // No ImGui and no Vulkan: tests/test_mods_entry_anim.py compiles it alone. #pragma once #include +#include #include namespace mods_entry { @@ -69,6 +70,16 @@ inline int64_t in_at(float amount) { return (int64_t)((double)hi * (double)kFrameNs); } +/// ((Triforce, 11-20) the game's own alpha of its buttons as the amount the entry is drawn with. The game blends its +/// buttons in linear light; the overlay blends in the window's code values unless the window is HDR. A white word at alpha +/// a over a dark picture is a^(1/2.2) of the way to white in code values when blended in linear light: drawn at a itself +/// in code values, the entry was dimmer than the game's buttons all through a fade (a quarter of the way where they were +/// half way), though it began and ended with them. linear: the overlay blends in linear light too (HDR): a itself) +inline float list_amount(float game_alpha, bool linear) { + const float a = std::clamp(game_alpha, 0.0f, 1.0f); + return linear ? a : std::pow(a, 1.0f / 2.2f); +} + /// The entry's state from one drawing to the next: how chosen it looks (focus) and how much of it shows with the list struct Anim { bool sel = false; // (it had the cursor at the last step) @@ -78,8 +89,12 @@ struct Anim { int64_t list_ns = 0; // (when the list came) float gone_from = 0; // (shown when the list went) float focus = 0, shown = 0; - /// One drawing at now: whether the entry has the cursor, how long since the host last told of the list - void step(int64_t now, bool selected, int64_t list_age_ns) { + /// One drawing at now: whether the entry has the cursor, how long since the host last told of the list. + /// ((Triforce, 11-20) list_alpha: how much of its own buttons the game shows now (0 to 1, the host's word each frame: + /// HOS_OPT_MODS_ALPHA); the entry shows as much while the list is there, so it comes and goes with them, frame for + /// frame, whatever animation the game plays (the title's opening, a second long, after a second and a half of + /// nothing; the 15 frames as the list comes back). Under 0: not told, the list's In by this clock, as before) + void step(int64_t now, bool selected, int64_t list_age_ns, float list_alpha = -1.0f) { if (!since_ns || selected != sel) from = since_ns ? focus : (selected ? 1.0f : 0.0f), sel = selected, since_ns = now; const float f = frames(now - since_ns); focus = sel ? std::max(from, curve(kSelect, f)) : from * curve(kUnSelect, f); @@ -87,7 +102,8 @@ struct Anim { if (seen && !there) list_ns = now - in_at(shown); // ((Triforce, 11-19) from where it is: no blink to nothing) if (!seen && there) gone_from = shown; there = seen; - shown = seen ? curve(kListIn, frames(now - list_ns)) : gone_from * curve(kUnSelect, frames(list_age_ns - kListGoneNs)); + shown = !seen ? gone_from * curve(kUnSelect, frames(list_age_ns - kListGoneNs)) + : list_alpha >= 0 ? list_alpha : curve(kListIn, frames(now - list_ns)); shown = std::clamp(shown, 0.0f, 1.0f), focus = std::clamp(focus, 0.0f, 1.0f); } /// LoopRandom's value now (it runs from the cursor's coming) diff --git a/gpu/vk/mods_page_anim.h b/gpu/vk/mods_page_anim.h new file mode 100644 index 0000000..f3cd50d --- /dev/null +++ b/gpu/vk/mods_page_anim.h @@ -0,0 +1,48 @@ +// (Triforce, 11-20) The Mods screen as it opens. The owner: "Mod menu fade in". The entry that opens it is animated as the +// game's own entries are (mods_entry_anim.h on the title, focus_anim.h under the pause menu's System list), and Quit's +// dialog opens as the game's SystemWindow_00; the screen itself came at once, whole, on the frame after the press. It +// fades in now as the game's lists and windows do: the alpha of Title_00's list and of SystemWindow_00's N_In_00 (the +// same keys in both layouts: 0 to 1 over 15 frames, slow to start, slopes 0.02 and 0.14), with no rise, as a whole screen +// has none. It takes input once that has played, as the game's own screens and Quit's dialog do. Back is a cut, as the +// game's windows go in 2 frames. +// No ImGui and no Vulkan: tests/test_mods_page_anim.py compiles it alone. +#pragma once +#include +#include +#include "mods_entry_anim.h" // (the layouts' curves: Key, curve, frames) + +namespace mods_page { + +using mods_entry::curve; +using mods_entry::frames; +using mods_entry::Key; +using mods_entry::kFrameNs; + +constexpr Key kInAlpha[] = {{0, 0, 0.02f}, {15, 1, 0.14f}}; +constexpr int64_t kInNs = 15 * kFrameNs; + +/// How much of the screen shows t ns after it opened (0 to 1) +inline float in_alpha(int64_t t) { + if (t <= 0) return 0; + return std::clamp(curve(kInAlpha, frames(t)), 0.0f, 1.0f); +} + +/// The screen from one tick to the next: whether it is shown, and since when +struct Page { + bool shown = false; // (it was the page shown at the last step) + int64_t since_ns = 0; // (when it came; 0 while it is not shown) + /// One tick at now: whether the Mods screen is the page shown. A screen that comes starts its In; one that goes has none + void step(int64_t now, bool on) { + if (on && !shown) since_ns = now; + if (!on) since_ns = 0; + shown = on; + } + /// How much of it shows at now. settle: as it stands once its In is done (a still, a picture that compares). A screen + /// asked for and not stepped yet (the page was set this tick, the step comes at the tick's end) shows nothing: drawn in + /// between at full, it flashed whole for a frame before its In began + float alpha(int64_t now, bool settle = false) const { return settle ? 1.0f : !shown ? 0.0f : in_alpha(now - since_ns); } + /// (it takes input once its In has played, as the game's) + bool ready(int64_t now) const { return shown && now - since_ns >= kInNs; } +}; + +} // namespace mods_page diff --git a/gpu/vk/msync_pass.h b/gpu/vk/msync_pass.h new file mode 100644 index 0000000..4f25759 --- /dev/null +++ b/gpu/vk/msync_pass.h @@ -0,0 +1,95 @@ +// (Triforce, 11-21, the performance audit's MEM-48, after Source A's) A mirror sync's pages from the captures by one compute +// dispatch: the rules. +// +// A pool the GPU reads has a mirror in VRAM, and a sync brings the pages the CPU wrote into it. Those that come from a +// capture's buffer (host memory) went as copy regions, a page or a few each, 700 to 1,300 a frame: the GPU reads each +// region over PCIe after the one before. The write-back diff's pass (shaders/wb_diff.comp) reads a source, compares it +// with a shadow and writes a destination where they differ; with the shadow and the destination both the mirror it is a +// copy, a workgroup a chunk, every chunk of a sync in one dispatch. Source A's RTX 5080 in Hyrule Field: the syncs' GPU +// time 1.41 to 0.69 ms a frame, 79.7 to 83.1 fps where the GPU bounds the frame. +// +// What keeps it right: +// - a run goes as chunks only in whole words at word-aligned addresses (words); any other run of a sync is copied; +// - the chunk tables are host memory in a ring of blocks, each table from an atom of its own (the flush covers whole +// atoms), a block taken again only once the GPU is done with the recording that last read it (Ring); a sync whose +// table finds no room is copied, as before; +// - copies into one buffer are in order among themselves, a dispatch is not: where a mirror is written twice in one +// upload command buffer and one of the two is the pass, a full barrier goes between (barrier_first). The pass's own +// chunks and a sync's other copies never share a page. +// RX_VK_MSYNC_PASS: on for NVIDIA when not set (measured there; no gain on an RX 5600 XT), 1: on every GPU, 0: copies. +// +// No Vulkan here: tests/test_msync_pass.py compiles it. +#pragma once +#include +#include + +namespace msync_pass { + +inline constexpr int kBlocks = 16; +inline constexpr uint64_t kBlockBytes = 256u << 10; // (8192 chunks of 32 bytes: a frame's syncs took about 1,300) +inline constexpr uint64_t kChunkBytes = 64u << 10; // (the pass's chunk at most) + +/// Whether the pass is used: asked, RX_VK_MSYNC_PASS's value; not set: on NVIDIA's GPUs +inline bool on(const char* asked, bool nvidia) { + if (!asked || !*asked) return nvidia; + return atoi(asked) != 0; +} + +/// Whether a run from src to dst (addresses) of size bytes can go as chunks +constexpr bool words(uint64_t src, uint64_t dst, uint64_t size) { return size != 0 && !((src | dst | size) & 3); } + +/// A run's chunks: emit(source, destination, count, vectors), the count in 16-byte vectors or in words. split(at, size, +/// e(at, bytes, vectors)) is the write-back diff's (wbd_split: the words before the first 16-byte line, the lines 64 KB +/// at a time, the words after). Addresses not alike within 16 bytes go in words alone +template +void run_chunks(uint64_t src, uint64_t dst, uint64_t size, Split&& split, Emit&& emit) { + if ((src ^ dst) & 15) { + for (uint64_t at = 0; at < size; at += kChunkBytes) { + const uint64_t len = size - at < kChunkBytes ? size - at : kChunkBytes; + emit(src + at, dst + at, (uint32_t)(len / 4), false); + } + return; + } + split(dst, size, [&](uint64_t a, uint64_t len, bool vecs) { emit(src + (a - dst), a, (uint32_t)(len / (vecs ? 16 : 4)), vecs); }); +} + +/// What wrote a mirror so far in an upload command buffer +enum : uint8_t { kCopies = 1, kPass = 2 }; +/// Whether a full barrier goes before a write of kind `now` into a mirror that `before` wrote earlier in the same +/// upload command buffer: the pass after anything, and anything after the pass +constexpr bool barrier_first(uint8_t before, uint8_t now) { return now == kPass ? before != 0 : (before & kPass) != 0; } + +/// The tables' blocks: which are made, the newest recording that reads each, and how far each is filled +struct Ring { + struct Block { + bool made = false; + uint64_t value = 0, fill = 0; + }; + Block b[kBlocks]; + int cur = -1; + /// Room for a table of `bytes` read by the recording rec_value, the GPU done up to `completed`: the block's index + /// and *at, the table's offset in it, a multiple of align; -1: none (too big, or the next block is still read). + /// A block returned with made false is the caller's to make before it writes + int get(uint64_t bytes, uint64_t align, uint64_t rec_value, uint64_t completed, uint64_t* at) { + if (!bytes || bytes > kBlockBytes || !align) return -1; + if (cur >= 0) { + Block& c = b[cur]; + const uint64_t start = (c.fill + align - 1) / align * align; + if (start + bytes <= kBlockBytes) { + c.fill = start + bytes; + if (rec_value > c.value) c.value = rec_value; + *at = start; + return cur; + } + } + const int next = (cur + 1) % kBlocks; + Block& n = b[next]; + if (n.made && n.value > completed) return -1; + cur = next; + n.fill = bytes, n.value = rec_value; + *at = 0; + return next; + } +}; + +} // namespace msync_pass diff --git a/gpu/vk/new_wait.h b/gpu/vk/new_wait.h new file mode 100644 index 0000000..925c149 --- /dev/null +++ b/gpu/vk/new_wait.h @@ -0,0 +1,49 @@ +// (Triforce, 11-22, the performance audit's OS-04, after Source A's and Source B's) A `new` of the renderer's that fails at +// Windows' commit limit waits before it throws. +// +// A refused allocation in the renderer was an uncaught std::bad_alloc: the game ended. At the commit limit (RAM and the page +// file full) a refusal often ends in moments, as the page file grows or commit is given back, so the renderer's new handler +// (vk_main.cpp new_refused, set in mkwvk_init) sleeps 50 ms and lets the allocation be tried again, for up to 5 s a thread +// (HOS_COMMIT_WAIT_MS, the host's switch for the same wait: 0, none), and throws after that. Only while the system has less +// than 4 GiB of commit left: a `new` refused with more than that free was not refused for the limit (a size no memory could +// hold), and throws at once, as before. Source A's is the wait; the 4 GiB rule is Source B's, written anew here. +// +// A `new (std::nothrow)` whose caller has its own answer to a refusal (a staging block, a copy that is only a saving, the +// recorder's ring) is not held up: NoWait around it makes the handler throw at once, and the nothrow new answers null. +// +// The rule alone, no system calls: tests/test_commit_limit.py compiles it. +#pragma once + +#include +#include + +namespace mkwvk::new_wait { + +constexpr uint64_t kNear = 4ull << 30; // (the commit left under which a refusal is taken for the limit's) +constexpr int kStepMs = 50, kWaitMs = 5000, kMostMs = 60000; + +/// HOS_COMMIT_WAIT_MS in ms: not set (or empty) 5000; a number that many, 0 and less none, 60000 at most +inline int wait_ms(const char* env) { + if (!env || !*env) return kWaitMs; + const long v = std::atol(env); + return v <= 0 ? 0 : v > kMostMs ? kMostMs : (int)v; +} + +inline thread_local int t_waited_ms = 0; // (what this thread has waited in all: its 5 s are not given twice) +inline thread_local int t_no_wait = 0; // (NoWait scopes open on this thread) + +/// Around a new (std::nothrow) that must answer at once +struct NoWait { + NoWait() { t_no_wait++; } + ~NoWait() { t_no_wait--; } + NoWait(const NoWait&) = delete; + NoWait& operator=(const NoWait&) = delete; +}; + +/// A `new` was refused. true: sleep a step and let it be tried again; false: throw. known and avail: the system's commit left, +/// when it could be read; waited_ms: this thread's waits so far; max_ms: wait_ms's; no_wait: NoWait scopes open +inline bool waits(bool known, uint64_t avail, int waited_ms, int max_ms, int no_wait) { + return !no_wait && known && avail < kNear && waited_ms < max_ms; +} + +} // namespace mkwvk::new_wait diff --git a/gpu/vk/occ_answer.h b/gpu/vk/occ_answer.h index 88edb15..6d2ee2f 100644 --- a/gpu/vk/occ_answer.h +++ b/gpu/vk/occ_answer.h @@ -15,6 +15,8 @@ // The game also predicates draws on these words (render conditions, NOP_RENDER_COND): as NX2navi decides them // (runtime/vk_exec_xfer.c cond_enabled), a word whose newest report is a samples-passed one with its count still to come is // unknown, and the condition renders; a word whose count was written is read from memory, where a real 0 can skip the draw. +// (11-22) A count over a draw that such a skip left out is marked (Counter::gate, Writers::kGatedSrc, the line counts them); +// RX_VK_OCC_GATE=1: a condition on it renders (cond_decide), so a skip's 0 cannot hide the same object again and again. // // Counter one counting, from a reset to its report, over the queries its draws began: the replay's (one per NVN // queue's state, under replay_mu) or a parallel replay's segment's (the helper thread's: a segment starts with @@ -95,6 +97,7 @@ struct Rep { uint32_t first = 0, n = 0; uint64_t pos = 0; int64_t present = 0; + bool gated = false; // (a draw of its count left out by a condition that read a written count: Counter::gate) }; /// One samples-passed counting. The replay calls reset() at a SAMPLES_PASSED reset, draw_enter() as a draw goes into draw(), @@ -113,8 +116,10 @@ struct Counter { int open = -1; // (the query begun and not ended, its index in the block) bool lost = false; // (a draw in another recording, or the block full) uint32_t ops = 0, issued = 0; // (draws that went into draw() while counting, and those it recorded) - enum Why { kNone, kSkipped, kOtherCmd, kFull }; + enum Why { kNone, kSkipped, kOtherCmd, kFull, kColour }; Why why = kNone; + bool gated = false; // (since the reset a draw was left out by a render condition that read a written count) + bool colour = false; // ((Triforce, 11-22, GPUWORK-01) a draw that writes colour came in since the reset) /// A parallel replay's segment starts (or ends), or the replay's counting is given up: nothing is counting void segment() { *this = Counter{}; } @@ -126,7 +131,35 @@ struct Counter { state = kCounting, slot = none ? -1 : slot_, serial = none ? 0 : serial_, first = next, open = -1, lost = none; ops = issued = 0; why = none ? kOtherCmd : kNone; + gated = false; + colour = false; } + /// (11-22) A draw the replay passes with rendering off because a condition read a written count (a 0 of an earlier + /// report): while counting, this count is gated, NVN counts nothing for that draw and neither do the queries. Such a count + /// is a skip's result, not the object's visibility (Writers::kGatedSrc) + void gate(bool keep) { + if (counting(keep)) gated = true; + } + /// ((Triforce, 11-22, the performance audit's GPUWORK-01, after Source A's) A draw that writes colour comes in, its pass + /// open, before begin(). The game's occlusion tests draw depth only (a bounding box against the depth buffer); a query + /// that stays open over colour draws makes the GPU wait at its end for the work it encloses. What the caller does with + /// the answer: first, this is the window's first such draw, and open, a query is open over it (both counted, whether or + /// not the rule is on); lose, the rule is on (depth_only) and the count is given up: the caller ends the open query and + /// calls lose_colour(). Nothing for a count not counting or lost already + struct Colour { + bool first = false, open = false, lose = false; + }; + Colour colour_draw(bool keep, bool depth_only) { + Colour r; + if (!counting(keep) || lost) return r; + r.first = !colour, colour = true; + r.open = open >= 0; + r.lose = depth_only; + return r; + } + /// The count given up for a colour draw (its open query ended by the caller): no query begins until the next reset, + /// its report is incomplete, so the record keeps the placeholder and its render condition renders + void lose_colour() { lost = true, why = kColour; } void draw_enter(bool keep) { if (counting(keep)) ops++; } @@ -162,7 +195,7 @@ struct Counter { if (!complete && !lost) why = kSkipped; state = kReported; if (!complete) return kIncomplete; - rep->first = first, rep->n = next - first; + rep->first = first, rep->n = next - first, rep->gated = gated; return kMeasured; } }; @@ -170,57 +203,55 @@ struct Counter { /// A map from 64-bit keys, open addressing with linear probes and backward-shift deletion (no tombstones): no allocation per /// entry. The replay placed one entry for every samples-passed report (~2,600 a frame at the Market) in a std::unordered_map, /// a node allocated and freed each time: 0.76 ms a frame of the Presentation Thread (all-thread sample, 2026-11-20) +// ((Triforce, 11-21) The keys, the used marks and the values in three arrays: a probe reads 9 bytes a slot, not a 48-byte slot. +// The game's report addresses rotate through ~49,000 a 600-present period, ~27,000 live at once (kKeepAge): the table of slots was +// ~3 MB, past the core's L2, and its probes and the prune's walk over it were 5% of the Presentation Thread (exact profile of a +// line-table build, 2026-11-21). Same entries, same order of probes and shifts as before) template class FlatU64 { public: - struct Slot { - uint64_t key = 0; - V val{}; - bool used = false; - }; V* find(uint64_t k) { if (!n_) return nullptr; for (size_t i = idx(k);; i = (i + 1) & mask_) { - Slot& s = s_[i]; - if (!s.used) return nullptr; - if (s.key == k) return &s.val; + if (!used_[i]) return nullptr; + if (keys_[i] == k) return &vals_[i]; } } const V* find(uint64_t k) const { return const_cast(this)->find(k); } V& operator[](uint64_t k) { if (V* v = find(k)) return *v; - if ((n_ + 1) * 10 > s_.size() * 7) grow(); + if ((n_ + 1) * 10 > keys_.size() * 7) grow(); size_t i = idx(k); - while (s_[i].used) i = (i + 1) & mask_; - s_[i].key = k, s_[i].val = V{}, s_[i].used = true; + while (used_[i]) i = (i + 1) & mask_; + keys_[i] = k, vals_[i] = V{}, used_[i] = 1; n_++; - return s_[i].val; + return vals_[i]; } size_t count(uint64_t k) const { return find(k) ? 1 : 0; } size_t erase(uint64_t k) { if (!n_) return 0; size_t i = idx(k); for (;; i = (i + 1) & mask_) { - if (!s_[i].used) return 0; - if (s_[i].key == k) break; + if (!used_[i]) return 0; + if (keys_[i] == k) break; } // backward shift: the entries after the hole that would be found from a slot at or before it move into it for (size_t j = (i + 1) & mask_;; j = (j + 1) & mask_) { - if (!s_[j].used) break; - const size_t home = idx(s_[j].key); + if (!used_[j]) break; + const size_t home = idx(keys_[j]); const bool moves = i <= j ? (home <= i || home > j) : (home <= i && home > j); - if (moves) s_[i] = s_[j], i = j; + if (moves) keys_[i] = keys_[j], vals_[i] = vals_[j], i = j; } - s_[i] = Slot{}; + keys_[i] = 0, vals_[i] = V{}, used_[i] = 0; n_--; return 1; } template void erase_if(F pred) { // (the keys to go first, then each erased: a shift during the walk would skip or revisit entries) - std::vector gone; - for (const Slot& s : s_) - if (s.used && pred(s.key, s.val)) gone.push_back(s.key); - for (uint64_t k : gone) erase(k); + gone_.clear(); + for (size_t i = 0; i < keys_.size(); i++) + if (used_[i] && pred(keys_[i], vals_[i])) gone_.push_back(keys_[i]); + for (uint64_t k : gone_) erase(k); } bool empty() const { return n_ == 0; } size_t size() const { return n_; } @@ -228,18 +259,23 @@ public: private: size_t idx(uint64_t k) const { return (size_t)((k * 0x9E3779B97F4A7C15ull) >> shift_) & mask_; } void grow() { - std::vector old; - old.swap(s_); - const size_t cap = old.empty() ? 1024 : old.size() * 2; - s_.assign(cap, Slot{}); + std::vector ok; + std::vector ou; + std::vector ov; + ok.swap(keys_), ou.swap(used_), ov.swap(vals_); + const size_t cap = ok.empty() ? 1024 : ok.size() * 2; + keys_.assign(cap, 0), used_.assign(cap, 0), vals_.assign(cap, V{}); mask_ = cap - 1; shift_ = 64; for (size_t c = cap; c > 1; c >>= 1) shift_--; n_ = 0; - for (const Slot& s : old) - if (s.used) (*this)[s.key] = s.val; + for (size_t i = 0; i < ok.size(); i++) + if (ou[i]) (*this)[ok[i]] = ov[i]; } - std::vector s_; + std::vector keys_; + std::vector used_; + std::vector vals_; + std::vector gone_; // (erase_if's list, kept: no allocation a prune) size_t n_ = 0, mask_ = 0; int shift_ = 64; }; @@ -259,12 +295,15 @@ public: kWritten, // (its count is in the record: completion wrote it) kFinal // (no count comes: skip list, knob off, a clear flag, not counted whole, lost, late, the CPU wrote it) }; - enum Src { kNone, kPendingSrc, kWrittenSrc, kFinalSrc }; // (what a render condition reading the word finds) + // (what a render condition reading the word finds. kGatedSrc: written, a count over a draw a written count's condition + // left out, see Counter::gate) + enum Src { kNone, kPendingSrc, kWrittenSrc, kFinalSrc, kGatedSrc }; struct Placed { uint64_t pos = 0; int64_t present = 0; uint64_t value = 0; // (what the replay, or a count written since, put in the word: a count is written only over it) State state = kFinal; + bool gated = false; // (its count, written, was over a draw a written count's condition left out) }; /// A report's placeholder `value` written at the replay, at pos in the stream (the newest there from now on). write: /// its count is to be written once read back (not on RX_VK_OCC_QUERY_SKIP, the lab knob on, not a clear flag) @@ -287,10 +326,11 @@ public: const auto* it = m_.find(addr); return !it ? 0 : it->value; } - /// The count of the report at (addr, pos) is in its record now - void wrote(uint64_t addr, uint64_t pos, uint64_t count) { + /// The count of the report at (addr, pos) is in its record now (gated: counted over a draw a written count's condition + /// left out) + void wrote(uint64_t addr, uint64_t pos, uint64_t count, bool gated = false) { auto* it = m_.find(addr); - if (it && it->pos == pos) it->state = kWritten, it->value = count; + if (it && it->pos == pos) it->state = kWritten, it->value = count, it->gated = gated; } /// No count comes for the report at (addr, pos) (not counted whole, lost, late, never sent, the CPU wrote the record): /// a condition reads its word from memory from now on, as before @@ -317,7 +357,7 @@ public: if (!it) return kNone; switch (it->state) { case kPending: return now - it->present > kLateAge ? kFinalSrc : kPendingSrc; - case kWritten: return kWrittenSrc; + case kWritten: return it->gated ? kGatedSrc : kWrittenSrc; case kFinal: break; } return kFinalSrc; @@ -339,23 +379,33 @@ private: /// they differ) over the words' sources (Writers::word), after NX2navi's cond_enabled: a samples-passed count not done yet /// renders; a written one, as any other word, is read from memory (base_enabled: the replay's answer from memory). active: /// RX_VK_OCC_QUERY=1 or the lab knob's phase (else the answer is the base one, whatever the table says); cond_on: false -/// with RX_VK_OCC_COND=0, every condition on a samples-passed word renders +/// with RX_VK_OCC_COND=0, every condition on a samples-passed word renders. +/// (11-22) gate_on (RX_VK_OCC_GATE=1): a word whose count was written over a draw that a written count's condition left +/// out (Writers::kGatedSrc) is unknown, and the condition renders. The count is a skip's result, not the object's: on the +/// console the game's CPU reads a 0 before it records the next frame and draws the object's proxy instead; here the count +/// often lands after the CPU has read the placeholder (4096: visible), the game goes on drawing the object under the +/// condition, the condition reads the skip's 0, the next count is 0 again, and the object stays hidden for good. Off: as a +/// written count (read from memory) struct Cond { - enum Kind { kNotSamples, kBase, kUnknown, kForced, kWrittenOn, kWrittenOff, kMemory }; + enum Kind { kNotSamples, kBase, kUnknown, kForced, kWrittenOn, kWrittenOff, kMemory, kGated }; bool enabled; Kind kind; }; -inline Cond cond_decide(bool base_enabled, Writers::Src sx, Writers::Src sy, bool active, bool cond_on) { +inline Cond cond_decide(bool base_enabled, Writers::Src sx, Writers::Src sy, bool active, bool cond_on, bool gate_on = false) { if (sx == Writers::kNone && sy == Writers::kNone) return {base_enabled, Cond::kNotSamples}; if (!active) return {base_enabled, Cond::kBase}; if (!cond_on) return {true, Cond::kForced}; if (sx == Writers::kPendingSrc || sy == Writers::kPendingSrc) return {true, Cond::kUnknown}; - if (sx == Writers::kWrittenSrc || sy == Writers::kWrittenSrc) + const bool gated = sx == Writers::kGatedSrc || sy == Writers::kGatedSrc; + if (gated && gate_on) return {true, Cond::kGated}; + if (sx == Writers::kWrittenSrc || sy == Writers::kWrittenSrc || gated) return {base_enabled, base_enabled ? Cond::kWrittenOn : Cond::kWrittenOff}; return {base_enabled, Cond::kMemory}; } inline const char* src_name(Writers::Src s) { - return s == Writers::kPendingSrc ? "pending" : s == Writers::kWrittenSrc ? "written" : s == Writers::kFinalSrc ? "memory, no count to come" : "memory"; + return s == Writers::kPendingSrc ? "pending" : s == Writers::kWrittenSrc ? "written" + : s == Writers::kGatedSrc ? "written, over a draw a written count skipped" + : s == Writers::kFinalSrc ? "memory, no count to come" : "memory"; } /// What happened since the last line (under the caller's lock) @@ -370,9 +420,16 @@ struct Stats { int64_t rewritten = 0; // records rewritten in the stream after their report (a small clear, another counter type) int64_t measured = 0, no_reset = 0, incomplete = 0, doubles = 0; // counted whole by queries, and those not int64_t inc_skipped = 0, inc_other_cmd = 0, inc_full = 0; // why not whole + // ((Triforce, 11-22, GPUWORK-01) and of the incomplete ones, those given up for a colour draw in their window + // (RX_VK_OCC_DEPTH_ONLY); the windows that met a colour draw, the colour draws under an open query, the counts given up + // (colour_line, a line of its own: line() is at the log's length already) + int64_t inc_colour = 0, colour_windows = 0, colour_open = 0, colour_lost = 0; // render conditions that read a samples-passed report's word (Cond::Kind), and the work they skipped int64_t conds = 0, cond_base = 0, cond_unknown = 0, cond_forced = 0, cond_written_on = 0, cond_written_off = 0, cond_memory = 0; int64_t draws_skipped = 0, clears_skipped = 0; + // (11-22) counts over a draw a written count's condition left out (Counter::gate): counted whole, written (of them 0), + // the conditions that read one, and those RX_VK_OCC_GATE=1 rendered + int64_t gated = 0, gated_written = 0, gated_zeros = 0, cond_gated_read = 0, cond_gated = 0; // at completion int64_t written = 0, zeros = 0; // counts written into their records (of them 0: the object hidden) int64_t lost = 0; // a query not available once its recording was done: 4096 stays @@ -389,7 +446,17 @@ struct Stats { if (k == Cond::kNotSamples) return; conds++; (k == Cond::kBase ? cond_base : k == Cond::kUnknown ? cond_unknown : k == Cond::kForced ? cond_forced - : k == Cond::kWrittenOn ? cond_written_on : k == Cond::kWrittenOff ? cond_written_off : cond_memory)++; + : k == Cond::kWrittenOn ? cond_written_on : k == Cond::kWrittenOff ? cond_written_off : k == Cond::kGated ? cond_gated + : cond_memory)++; + } + /// ((Triforce, 11-22, the performance audit's GPUWORK-01) The colour draws' line, into out. on: the rule is on + int colour_line(char* out, size_t size, int64_t presents, bool on) const { + return snprintf(out, size, + "vk: occlusion queries (RX_VK_OCC_QUERY), %lld presents (3): %lld counting windows met a draw that " + "writes colour, %lld such draws were recorded under an open query; depth-only counting " + "(RX_VK_OCC_DEPTH_ONLY) is %s: %lld counts given up at a colour draw, %lld reports kept 4096 for it", + (long long)presents, (long long)colour_windows, (long long)colour_open, on ? "on" : "off", + (long long)colour_lost, (long long)inc_colour); } /// The line, into out int line(char* out, size_t size, int64_t presents, size_t tracked) const { @@ -407,7 +474,8 @@ struct Stats { "%lld (%lld as before: knob occq off; %lld rendered: a count pending (unknown); %lld rendered: " "RX_VK_OCC_COND=0; %lld rendered from a written count; %lld skipped from a written count; %lld from " "memory, no count to come); skipped by them: %lld draws and dispatches, %lld clears; %lld addresses " - "this period, %zu tracked", + "this period, %zu tracked; counts over a draw such a skip left out: %lld counted whole, %lld written " + "(%lld of them 0), %lld conditions read one (%lld rendered: RX_VK_OCC_GATE=1)", (long long)presents, (long long)reports, (long long)in_segments, (long long)skipped, (long long)fixed, (long long)flagged, (long long)rewritten, (long long)measured, (long long)no_reset, (long long)incomplete, (long long)inc_skipped, (long long)inc_other_cmd, (long long)inc_full, @@ -417,7 +485,8 @@ struct Stats { (long long)at[kAtWait], (long long)at[kAtBegin], (long long)at[kAtPresent], (long long)at[kAtReuse], (long long)conds, (long long)cond_base, (long long)cond_unknown, (long long)cond_forced, (long long)cond_written_on, (long long)cond_written_off, (long long)cond_memory, - (long long)draws_skipped, (long long)clears_skipped, (long long)addrs, tracked); + (long long)draws_skipped, (long long)clears_skipped, (long long)addrs, tracked, (long long)gated, + (long long)gated_written, (long long)gated_zeros, (long long)cond_gated_read, (long long)cond_gated); } }; @@ -458,9 +527,10 @@ void complete(const std::vector& reps, const uint64_t* pairs, uint32_t used w.settle(p.addr, p.pos); continue; } - w.wrote(p.addr, p.pos, sum); + w.wrote(p.addr, p.pos, sum, p.gated); const int64_t lat = std::max(0, now - p.present); st.written++, st.zeros += sum ? 0 : 1, st.lat_sum += lat, st.lat_ge1 += lat >= 1, st.lat_ge2 += lat >= 2; + if (p.gated) st.gated_written++, st.gated_zeros += sum ? 0 : 1; } } /// A recording's reports that never went to the GPU: no count comes for them diff --git a/gpu/vk/occ_lean.h b/gpu/vk/occ_lean.h new file mode 100644 index 0000000..c08f2d0 --- /dev/null +++ b/gpu/vk/occ_lean.h @@ -0,0 +1,82 @@ +// (Triforce, 11-22, the performance audit's DESC-82, after Source B's: its first step, the census) The game's occlusion tests +// draw proxies that write no colour, and each such draw runs its whole fragment shader and fills its fragment descriptors. +// Source B makes a sibling pipeline without the fragment stage for a counted draw that writes no colour and whose fragment +// shader has no effect but colour (no depth or sample-mask output, no demote, no stores, image writes, atomics or global +// memory) and whose state has neither alpha to coverage nor an alpha test. It was never on by default nor measured; its +// census at Kakariko counted about 2,570 colour-off draws of about 3,420 counted. The entry's own order is the census first, +// and the lean pipelines only if the census finds more than one fragment descriptor a proxy to save. +// +// This is the census alone. RX_VK_OCC_LEAN=3: each draw under an open occlusion query that writes no colour is classed +// (classify: lean, or the first refusal), and its fragment stage's descriptors added to its class; every 600 presents one +// line gives each class's draws and descriptors a frame (vk_replay.cpp status_tick). No pipeline, key or descriptor changes. +// 1 (the lean pipelines) and 2 (their check) are not built: they read as off. Unset: nothing of it runs. +// +// The rules alone: tests/test_occ_lean.py compiles them. +#pragma once +#include +#include +#include +#include +#include + +namespace occ_lean { + +/// RX_VK_OCC_LEAN: 3 for the census; anything else is off (1 and 2: the steps after the census, not built) +inline int mode_of(const char* e) { return e && *e && std::atoi(e) == 3 ? 3 : 0; } + +enum Class { kLean, kNoFragment, kDepth, kSampleMask, kDemote, kStores, kImageWrites, kAtomics, kGlobal, kA2c, kAlphaTest, kClasses }; +inline const char* const kNames[kClasses] = {"lean", "no fragment stage", "depth out", "sample mask", "demote", "stores", + "image writes", "atomics", "global memory", "alpha to coverage", "alpha test"}; + +/// What the class reads of a draw: its fragment stage's (Shader::Info) and its state's +struct Facts { + bool has_fragment = true; + bool depth_out = false, sample_mask = false, demote = false, stores = false, image_writes = false, atomics = false, + global = false; + bool a2c = false, alpha_test = false; +}; + +/// The draw's class: no fragment stage (nothing to leave out), the first refusal of the shader's then of the state's, or lean +inline Class classify(const Facts& f) { + if (!f.has_fragment) return kNoFragment; + if (f.depth_out) return kDepth; + if (f.sample_mask) return kSampleMask; + if (f.demote) return kDemote; + if (f.stores) return kStores; + if (f.image_writes) return kImageWrites; + if (f.atomics) return kAtomics; + if (f.global) return kGlobal; + if (f.a2c) return kA2c; + if (f.alpha_test) return kAlphaTest; + return kLean; +} + +/// The counts, any recording thread (relaxed) +struct Census { + std::atomic draws[kClasses]{}; + std::atomic descriptors[kClasses]{}; + void note(Class c, uint32_t frag_descriptors) { + draws[c].fetch_add(1, std::memory_order_relaxed); + descriptors[c].fetch_add(frag_descriptors, std::memory_order_relaxed); + } + /// The line for `presents` presents (draws and fragment descriptors a frame, the lean ones' descriptors a draw, then each + /// refusal that met any), and the counts set back. Its length + int line(char* out, size_t n, uint64_t presents) { + uint64_t d[kClasses], s[kClasses], all = 0; + for (int i = 0; i < kClasses; i++) + d[i] = draws[i].exchange(0, std::memory_order_relaxed), s[i] = descriptors[i].exchange(0, std::memory_order_relaxed), + all += d[i]; + const double p = presents ? (double)presents : 1.0; + int at = std::snprintf(out, n, "vk: occlusion proxies (RX_VK_OCC_LEAN=3, %llu presents): %.0f colour-off counted draws a frame; " + "lean %.0f a frame with %.2f fragment descriptors each", (unsigned long long)presents, all / p, + d[kLean] / p, d[kLean] ? (double)s[kLean] / (double)d[kLean] : 0.0); + for (int i = 1; i < kClasses && at > 0 && (size_t)at < n; i++) + if (d[i]) + at += std::snprintf(out + at, n - (size_t)at, "; %s %.0f (%.2f descriptors each)", kNames[i], d[i] / p, + (double)s[i] / (double)d[i]); + return at; + } +}; +inline Census census; + +} // namespace occ_lean diff --git a/gpu/vk/page_marks.h b/gpu/vk/page_marks.h new file mode 100644 index 0000000..0d30cc3 --- /dev/null +++ b/gpu/vk/page_marks.h @@ -0,0 +1,100 @@ +// (Triforce, 11-22, the performance audit's THREAD-25, after Source A's) Pages that something ever happened to, asked without a lock; +// and the question an invalidate asks of them first. +// +// The game invalidates each shape's 64-byte occlusion slot, per view: tens of thousands of nvnBufferInvalidateMappedRange calls in a +// few seconds. The host gives the calling thread's virtual core up before every one and queues for it again after, as the call can +// wait for the replay or for a write-back still on the GPU; nearly always it has nothing to wait for. What it would wait for is +// known from three things: a write-back ever made into the range's pages, a texture the GPU wrote ever listed there, and a live +// texture's storage there. The first two are tables under their own locks; PageMarks keeps, beside each, one bit a 4 KB page that is +// only ever set, set before the table changes and under the table's lock, and read with no lock. A range with no mark in either and +// no texture storage is one invalidate() returns from without waiting, so the host may keep the core and not make the call +// (invalidate_quick in vk_replay.cpp, asked through mkwvk_invalidate_quick by host/rx_backend.c). +// +// A mark is never taken back: a page written back once declines for good, and waits as before. Addresses at or past 512 GB are +// not kept and answer as marked. The header is written from the audit's description of Source A's, whose file is not here. +// +// RX_VK_INVALIDATE_QUICK=1 turns the question on; off unless set (see docs/development.md for why). tests/test_invalidate_quick.py +#pragma once + +#include +#include +#include + +namespace mkwvk { + +struct PageMarks { + static constexpr uint64_t kSpan = 1ull << 39; // (512 GB: the guest window, and the GPU's addresses the pools lie in) + static constexpr uint64_t kPageShift = 12, kPages = kSpan >> kPageShift; + static constexpr uint64_t kChunkPages = 1ull << 18; // (a chunk: 1 GB of addresses, 32 KB of bits, made at its first mark) + static constexpr uint64_t kChunks = kPages / kChunkPages, kChunkWords = kChunkPages / 64; + std::atomic*> chunks[kChunks] = {}; + + /// every page with a byte in [lo, hi) marked (what lies at or past 512 GB is not kept: any() answers true there) + void mark(uint64_t lo, uint64_t hi) { + if (hi > kSpan) hi = kSpan; + if (hi <= lo) return; + const uint64_t last = (hi - 1) >> kPageShift; + for (uint64_t p = lo >> kPageShift; p <= last;) { + std::atomic* c = chunks[p / kChunkPages].load(std::memory_order_acquire); + if (!c) { + std::atomic* made = new std::atomic[kChunkWords]; + for (uint64_t i = 0; i < kChunkWords; i++) made[i].store(0, std::memory_order_relaxed); + if (chunks[p / kChunkPages].compare_exchange_strong(c, made, std::memory_order_acq_rel)) c = made; + else delete[] made; // (another thread made it first: c is its chunk) + } + // (the pages of this word from p to the word's end, or to `last`) + const uint64_t in_chunk = p % kChunkPages, bit = in_chunk % 64; + const uint64_t word_last = p - bit + 63, to = word_last < last ? word_last : last; + const uint64_t n = to - p + 1; + const uint64_t mask = (n == 64 ? ~0ull : ((1ull << n) - 1)) << bit; + c[in_chunk / 64].fetch_or(mask, std::memory_order_seq_cst); + p = to + 1; + } + } + /// whether a page with a byte in [lo, lo + size) was ever marked; true for a range that reaches 512 GB or wraps + bool any(uint64_t lo, uint64_t size) const { + if (!size) return false; + if (lo >= kSpan || size > kSpan - lo) return true; + const uint64_t last = (lo + size - 1) >> kPageShift; + for (uint64_t p = lo >> kPageShift; p <= last;) { + const std::atomic* c = chunks[p / kChunkPages].load(std::memory_order_seq_cst); + if (!c) { // (a chunk never marked: skipped whole) + p = (p / kChunkPages + 1) * kChunkPages; + continue; + } + const uint64_t in_chunk = p % kChunkPages, bit = in_chunk % 64; + const uint64_t word_last = p - bit + 63, to = word_last < last ? word_last : last; + const uint64_t n = to - p + 1; + const uint64_t mask = (n == 64 ? ~0ull : ((1ull << n) - 1)) << bit; + if (c[in_chunk / 64].load(std::memory_order_seq_cst) & mask) return true; + p = to + 1; + } + return false; + } +}; + +namespace inv_quick { // (the rule of vk_replay.cpp's invalidate_quick) + +/// Whether the question is asked at all. quick: RX_VK_INVALIDATE_QUICK (a number other than 0 turns it on; off unless set). The +/// three others are switches that make invalidate() log every call or wait for every range: with one set, every call is made +inline bool on(const char* quick, const char* log_invalidate, const char* log_readback, const char* wait_all) { + return quick && std::atoi(quick) != 0 && !log_invalidate && !log_readback && !wait_all; +} + +/// What is known of a range when the question is asked +struct Look { + uint64_t gpu = 0, size = 0; + bool ticket_pending = false; // (a fast fence poll's order ticket not yet left: invalidate() waits for that section first) + bool tex_storage = false; // (a live texture's storage in the range's pages: tex_pages_any) + bool vw_marked = false; // (a texture the GPU wrote ever listed there) + bool wb_marked = false; // (a write-back ever made into its pages in the game's memory) +}; +/// true: invalidate(gpu, size) would return without waiting for anything and without work: the call may be left out +inline bool nothing_to_wait_for(const Look& l) { + if (!l.size) return true; + if (l.gpu >= PageMarks::kSpan || l.size > PageMarks::kSpan - l.gpu) return false; + return !l.ticket_pending && !l.tex_storage && !l.vw_marked && !l.wb_marked; +} + +} // namespace inv_quick +} // namespace mkwvk diff --git a/gpu/vk/pcache_bound.h b/gpu/vk/pcache_bound.h index 13d5b35..8798e89 100644 --- a/gpu/vk/pcache_bound.h +++ b/gpu/vk/pcache_bound.h @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -87,6 +88,7 @@ enum class Read { OverBound, // (longer than the bound: not a byte of it read) NoMemory, // (no block of its size) Cut, // (the read ended early: nothing of it is used) + Foreign, // ((Triforce) its header is another GPU's or driver's: not read past it) }; inline const char* why(Read r) { @@ -96,13 +98,19 @@ inline const char* why(Read r) { case Read::OverBound: return "it is over the bound"; case Read::NoMemory: return "there is no memory for a block of its size"; case Read::Cut: return "reading it ended early"; + case Read::Foreign: return "it is another GPU's or driver's"; default: return ""; } } /// The whole file into out, or nothing: out is empty unless the answer is Ok. *len: the file's length whenever it could be -/// asked, read or not. least: a file of at most this many bytes is refused; bound: one longer than this too (0: none) -inline Read read(const char* path, uint64_t least, uint64_t bound, std::vector& out, uint64_t* len) { +/// asked, read or not. least: a file of at most this many bytes is refused; bound: one longer than this too (0: none). +/// ((Triforce) head_ok: when given, the file's first kHead bytes (a pipeline cache's header) are read alone and handed to it +/// with the file's length; refused, the rest is not read (Foreign). The file's stamp names only the translated shaders, so +/// after a driver update a file of 0.65-1.6 GB was read whole only for its header to be refused) +constexpr size_t kHead = 32; +inline Read read(const char* path, uint64_t least, uint64_t bound, std::vector& out, uint64_t* len, + bool (*head_ok)(const uint8_t* head, uint64_t size) = nullptr) { out.clear(); *len = 0; FILE* f = fopen(path, "rb"); @@ -120,7 +128,14 @@ inline Read read(const char* path, uint64_t least, uint64_t bound, std::vector SIZE_MAX) r = Read::NoMemory; - else { + uint8_t head[kHead]; + size_t got = 0; + if (r == Read::Ok && head_ok && (uint64_t)n >= kHead) { + got = fread(head, 1, kHead, f); + if (got != kHead) r = Read::Cut; + else if (!head_ok(head, (uint64_t)n)) r = Read::Foreign; + } + if (r == Read::Ok) { try { out.resize((size_t)n); } catch (const std::bad_alloc&) { @@ -128,7 +143,7 @@ inline Read read(const char* path, uint64_t least, uint64_t bound, std::vector 12,029 and 12,638 -> 11,822 MB over a tour, fps 54.3 -> 54.6, a pipeline's load 1.9 -> +// 5.0 ms. Triforce cannot drop the file: a stage library asked under the compile-required flag must be in the cache it hands +// the driver (21 of 21 cold otherwise). So the file keeps the libraries and the compute pipelines, and only whole graphics +// pipelines are left to the driver's cache, with no play file. +// +// Decided only when a fresh file begins (none read for these shaders), and kept in .lean ("lean ") for that +// file's life: a full file stays full and a lean one lean until the next fresh one. Not in a start that only prepares. +// +// RX_VK_PCACHE_LIBS_ONLY: unset or 0, off (every pipeline in the file, as before); 1, by the rule; 2, at every fresh file +// whatever the PC (the lab's and the tests'). Off by default: the driver's cache is shared and capped, and on a 32 GB PC +// Source A measured p99 17.23 -> 18.58 ms. +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace pcache_libs { + +inline constexpr uint64_t kRamMost = 20ull << 30; // (20 GB of RAM or less) +inline constexpr uint64_t kGlCacheLeast = 512ull << 20; // (512 MB or more in the driver's own cache) + +/// RX_VK_PCACHE_LIBS_ONLY: 0 off (unset too), 1 by the rule, 2 forced +inline int mode(const char* env) { + if (!env || !*env) return 0; + const int v = std::atoi(env); + return v <= 0 ? 0 : v >= 2 ? 2 : 1; +} + +struct Pc { + bool nvidia_proprietary = false; // (VK_DRIVER_ID_NVIDIA_PROPRIETARY) + uint64_t ram = 0; // (the PC's RAM, bytes; 0 unknown) + uint64_t glcache = 0; // (bytes in the driver's own cache folder) + bool gpl = false; // (the stage libraries are wanted and kept in our cache: gpl_wanted, gpl_libs_cached) + bool prepare_only = false; // (a start that only prepares) +}; +/// Whether the fresh file is a lean one +inline bool choose(int mode, const Pc& pc) { + if (mode <= 0 || pc.prepare_only) return false; + if (mode >= 2) return true; + return pc.nvidia_proprietary && pc.gpl && pc.ram && pc.ram <= kRamMost && pc.glcache >= kGlCacheLeast; +} + +/// The folder NVIDIA's driver keeps its Vulkan and OpenGL cache in: on Windows \NVIDIA\GLCache; elsewhere +/// __GL_SHADER_DISK_CACHE_PATH, else /nvidia/GLCache, else /.cache/nvidia/GLCache. Empty: unknown +inline std::string nvidia_cache_dir(bool windows, const char* localappdata, const char* gl_path, const char* xdg, const char* home) { + if (windows) return localappdata && *localappdata ? std::string(localappdata) + "\\NVIDIA\\GLCache" : std::string(); + if (gl_path && *gl_path) return gl_path; + if (xdg && *xdg) return std::string(xdg) + "/nvidia/GLCache"; + if (home && *home) return std::string(home) + "/.cache/nvidia/GLCache"; + return {}; +} +/// The bytes of the files under a folder, at most most_entries looked at (a folder that is not there: 0) +inline uint64_t folder_bytes(const std::string& dir, size_t most_entries = 200000) { + uint64_t sum = 0; + if (dir.empty()) return 0; + std::error_code ec; + const std::filesystem::path root(std::u8string(reinterpret_cast(dir.data()), dir.size())); // (UTF-8) + std::filesystem::recursive_directory_iterator it(root, std::filesystem::directory_options::skip_permission_denied, ec); + for (size_t n = 0; !ec && it != std::filesystem::recursive_directory_iterator() && n < most_entries; it.increment(ec), n++) { + std::error_code e2; + if (it->is_regular_file(e2)) { + const uint64_t s = it->file_size(e2); + if (!e2) sum += s; + } + } + return sum; +} + +// ---- .lean: "lean " + +inline bool lean_read(const std::string& path, const char* stamp16) { + FILE* f = fopen(path.c_str(), "rb"); + if (!f) return false; + char line[40]{}; + const size_t n = fread(line, 1, sizeof line - 1, f); + fclose(f); + line[n] = 0; + return n >= 21 && !memcmp(line, "lean ", 5) && !memcmp(line + 5, stamp16, 16); +} +inline bool lean_write(const std::string& path, const char* stamp16) { + FILE* f = fopen(path.c_str(), "wb"); + if (!f) return false; + const bool ok = fprintf(f, "lean %.16s\n", stamp16) > 0; + return fclose(f) == 0 && ok; +} + +/// What a start does: fresh (no file read for these shaders), whether the rule says lean now, and the record of the file +/// read. Lean for the session: a fresh file the rule makes lean, or a file read whose record says lean (and the switch on) +inline bool lean_now(int mode, bool fresh, bool chose, bool recorded) { + if (mode <= 0) return false; + return fresh ? chose : recorded; +} + +} // namespace pcache_libs diff --git a/gpu/vk/pcache_play.h b/gpu/vk/pcache_play.h index 3a252d1..069759a 100644 --- a/gpu/vk/pcache_play.h +++ b/gpu/vk/pcache_play.h @@ -22,6 +22,7 @@ #pragma once #include +#include namespace pcache_play { @@ -33,6 +34,13 @@ inline bool routes(bool main_cache, bool play_cache, bool preparing) { return main_cache && play_cache && !preparing; } +/// ((Triforce, 11-22, the performance audit's PIPE-95, after Source A's) RX_VK_PCACHE_OWN_PASSES: whether the back end's own +/// compute passes (FSR 3.1's, the anti-aliasing's, the texture converters': vk_draw.cpp shared_cache_compute) are made +/// through the play cache as the game's are, on unless set to 0 (0: bare into the main cache, as before) +inline bool own_passes(const char* env) { + return !env || std::atoi(env) != 0; +} + /// At the start, the play file read: whether it is folded into the main cache now (fold_bytes 0: never) inline bool fold_due(uint64_t play_bytes, uint64_t fold_bytes) { return fold_bytes && play_bytes >= fold_bytes; diff --git a/gpu/vk/piece_idle.h b/gpu/vk/piece_idle.h new file mode 100644 index 0000000..86574f1 --- /dev/null +++ b/gpu/vk/piece_idle.h @@ -0,0 +1,62 @@ +// (Triforce, 11-22, the performance audit's MEM-26, its first step, after Source A's) What a mirror in pieces holds that +// no game pool covers any more, and how much of that no binding has reached for a while: counted, nothing released. +// +// A mirror in pieces gives a piece its memory when a binding first reaches it, and keeps it until the whole mirror goes. +// The mirror is an import's (a chunk of guest memory around what was bound), and the game's pools inside it come and go: +// a long-lived import keeps the memory of every piece any area reached, also where no game pool is left. Source A gives +// such memory back after two waits of 600 submissions. Before that is ported (the unbind and the free must wait for the +// GPU and for the bind timeline: memory freed under a compute pass that reads it lost devices before) this counts what +// it would find, every 1200 submissions, in the "mirrors in pieces" line: under about 100 MB at the end of a tour on a +// 6 GB card, the release is not worth having. +// +// Counted by piece. A release frees whole runs of pieces (one allocation a run), so this is the most it could give back. +// The rule alone, no Vulkan: tests/test_mirror_piece_idle.py compiles it against a page-by-page model. +#pragma once +#include +#include +#include +#include +#include + +namespace piece_idle { + +using Ranges = std::vector>; // ([first, second)) + +/// The ranges sorted and merged: disjoint, ascending, none empty (touching ones joined) +inline void merge(Ranges& r) { + std::sort(r.begin(), r.end()); + size_t m = 0; + for (const auto& e : r) { + if (e.second <= e.first) continue; + if (m && e.first <= r[m - 1].second) r[m - 1].second = std::max(r[m - 1].second, e.second); + else r[m++] = e; + } + r.resize(m); +} + +struct Count { + uint64_t held = 0; // (bytes of the pieces that have memory) + uint64_t uncovered = 0; // (of them, the pieces no game pool touches) + uint64_t idle = 0; // (of those, the ones no binding reached for idle_for submissions; 0 when that is not tracked) +}; + +/// A mirror at guest address lo of `size` bytes, in pieces of 1 << shift bytes: resident(i) says piece i has memory, +/// last(i) the submission a binding last reached it at. pools: the game's pools, merged (merge above). now: this +/// submission; idle_for: 0, or how many submissions without a binding make a piece idle. Adds to out +template +inline void count(uint64_t lo, uint64_t size, uint32_t shift, const Resident& resident, const Last& last, const Ranges& pools, uint64_t now, + uint64_t idle_for, Count& out) { + const uint64_t piece = 1ull << shift, n = (size + piece - 1) >> shift; + size_t at = 0; // (the first pool that ends after the piece's start: the pieces come ascending) + for (uint64_t i = 0; i < n; i++) { + if (!resident(i)) continue; + const uint64_t a = lo + (i << shift), b = lo + std::min(size, (i + 1) << shift); + out.held += b - a; + while (at < pools.size() && pools[at].second <= a) at++; + if (at < pools.size() && pools[at].first < b) continue; // (a game pool touches it) + out.uncovered += b - a; + if (idle_for && now - last(i) >= idle_for) out.idle += b - a; + } +} + +} // namespace piece_idle diff --git a/gpu/vk/pipe_ahead.h b/gpu/vk/pipe_ahead.h new file mode 100644 index 0000000..00c44bf --- /dev/null +++ b/gpu/vk/pipe_ahead.h @@ -0,0 +1,48 @@ +// Where the pipelines the draws first need stand in the last sessions' order, against where the draws are (no game, no GPU: +// the rules; vk_draw.cpp order_met, graphics_pipeline, draw_left_out_take). Apart from Vulkan so that tests/test_pipe_ahead.py +// runs them. +// +// (Triforce, 11-22, the performance audit's PIPE-81, after Source B's: its first step.) Where the precompile keeps no +// pipelines (light mode: precompile_keeps_all), every pipeline of a load is new at its first draw: linked there from its +// stage libraries, or by the warm a moment earlier (pipe_warm.h, pipe_order.h). The entry proposes building whole pipelines +// just ahead of the draws on light frames, from the order's next places, and keeping them; and it says to count first: +// each pipeline first drawn, its place ahead of where the draws are (pipe_order.h Front) or none, and whether the warm had +// linked it. Under a third within the window: stop, the builder is not worth its RAM and its compiles. +// +// This is that count, and only it: nothing is built or kept. A line every 600 presents while it grows (with the order used): +// within the window ahead, further ahead, behind, not in the order, and how many of those within the window the warm had +// linked; and whether the builder would be worth it by the entry's rule. +// +// RX_VK_PIPE_AHEAD=: the window, 1024 unset (1 to 4096); 0: not counted. +#pragma once + +#include +#include + +namespace pipe_ahead { + +inline constexpr int64_t kWindow = 1024; +inline constexpr int64_t kMost = 4096; + +/// RX_VK_PIPE_AHEAD: the window in places, kWindow unset; 0 off; clamped to 1..kMost +inline int64_t window(const char* env) { + if (!env || !*env) return kWindow; + const long long v = std::atoll(env); + return v <= 0 ? 0 : v > kMost ? kMost : (int64_t)v; +} + +enum class Where { None, Near, Far, Behind }; +/// A pipeline first drawn at place in the order (-1: not in it), the draws at front (Front::furthest; -1: not yet placed, +/// counted from the order's start) +inline Where where(int64_t place, int64_t front, int64_t window_places) { + if (place < 0) return Where::None; + const int64_t from = front < 0 ? 0 : front; + if (place < from) return Where::Behind; + return place - from <= window_places ? Where::Near : Where::Far; +} +/// The entry's rule for the builder: worth it when a third or more of the pipelines first drawn were within the window +inline bool worth(int64_t within, int64_t all) { + return all > 0 && within * 3 >= all; +} + +} // namespace pipe_ahead diff --git a/gpu/vk/pipe_order.h b/gpu/vk/pipe_order.h index b40b253..b51f4b6 100644 --- a/gpu/vk/pipe_order.h +++ b/gpu/vk/pipe_order.h @@ -32,6 +32,7 @@ #pragma once #include +#include #include #include #include @@ -45,7 +46,11 @@ namespace mkwvk::pipe_order { -constexpr size_t kKeysMax = program_order::kKeysMax; // (pipelines a session's order keeps, and the file) +// ((Triforce, 11-21) 16384, was the programs' 4096: one ten-area tour needs 3918 pipelines, so an adult tour pushed half of the +// child tour's out and its next tour linked 828 of 2274 ahead where it had linked 1455 of 2012 (pipeline stalls 1.3 -> 2.6 s, +// the worst 174 -> 464 ms). The file only: what the warm holds stays bounded by kLinksMax and kLibsMax below, and the programs' +// read-ahead (its RAM) by program_order's own 4096. 8 bytes a pipeline: 128 KB at most) +constexpr size_t kKeysMax = 16384; // (pipelines a session's order keeps, and the file) constexpr size_t kPerVariant = 32; // (the order's pipelines made for a program variant, at most: as the notes') constexpr size_t kLinksMax = 1024; // (links held for draws that have not come: a load's, beside the title's taken) constexpr size_t kLibsMax = 2048; // (stage libraries the warm made that no draw has had: two a program variant of 1024) @@ -58,9 +63,13 @@ inline bool on(const char* env) { return !(env && env[0] == '0' && !env[1]); } /// The file: program_order.h's form, its own magic inline std::vector write(const std::vector& keys) { return program_order::write(keys, kMagic); } -inline std::vector read(const uint8_t* data, size_t n) { return program_order::read(data, n, kMagic); } -using program_order::merge; -using Record = program_order::Record; +inline std::vector read(const uint8_t* data, size_t n) { return program_order::read(data, n, kMagic, kKeysMax); } +inline std::vector merge(const std::vector& now, const std::vector& before, size_t max = kKeysMax) { + return program_order::merge(now, before, max); +} +struct Record : program_order::Record { + Record() { max = kKeysMax; } +}; /// One pipeline of the order: its place, its note's record in the notes file struct Entry { @@ -154,12 +163,19 @@ public: by_place_.emplace(std::make_pair(place, key), o); place_of_.emplace(key, place); while (by_place_.size() > max_) { - const auto last = std::prev(by_place_.end()); - gone.push_back(last->second); - place_of_.erase(last->first.second); - by_place_.erase(last); + // ((Triforce, 11-21, soft) one behind the draws first (the farthest behind), then the farthest ahead as before + const bool back = soft_ && by_place_.begin()->first.first < behind_at_; + const auto out = back ? by_place_.begin() : std::prev(by_place_.end()); + gone.push_back(out->second); + place_of_.erase(out->first.second); + by_place_.erase(out); } } + /// ((Triforce, 11-21) RX_VK_ORDER_SOFT_BEHIND) Soft: behind() lets nothing go, it marks where behind begins; those held there + /// stay for a draw that comes back to them (a tour's shared pipelines sit in another session's region of the merged order, + /// and the draws went back and forth: 1265 of 3048 links let go as behind in a child tour after an adult one) and go first + /// when a new link needs the room. The bound (max) is the same + void soft(bool on) { soft_ = on; } /// The link held for key, taken out (a draw's from now on): false when none is bool take(uint64_t key, One& out) { const auto it = place_of_.find(key); @@ -172,6 +188,10 @@ public: } /// Those placed before at, taken out into gone void behind(int64_t at, std::vector& gone) { + if (soft_) { + behind_at_ = std::max(behind_at_, at); + return; + } const auto end = by_place_.lower_bound({at, 0}); for (auto it = by_place_.begin(); it != end;) { gone.push_back(it->second); @@ -193,7 +213,8 @@ public: } /// Whether a link at place would stay held: room left, or nearer than the farthest held (made, not let go at once) bool room(int64_t place) const { - return by_place_.size() < max_ || (!by_place_.empty() && place < std::prev(by_place_.end())->first.first); + return by_place_.size() < max_ || (!by_place_.empty() && place < std::prev(by_place_.end())->first.first) || + (soft_ && !by_place_.empty() && by_place_.begin()->first.first < behind_at_); } /// Whether a link held was linked from library lib bool uses(uint64_t lib) const { @@ -216,6 +237,8 @@ private: return false; } size_t max_; + bool soft_ = false; + int64_t behind_at_ = INT64_MIN; std::map, One> by_place_; // ((place, key): the nearest first) std::unordered_map place_of_; }; diff --git a/gpu/vk/pipe_wait.h b/gpu/vk/pipe_wait.h index 129f131..5c5e052 100644 --- a/gpu/vk/pipe_wait.h +++ b/gpu/vk/pipe_wait.h @@ -39,6 +39,13 @@ // libraries after it, so the draws of such a pipeline waited for the whole build. The same objects are made either way. // A draw that waits runs the first job itself (vk_jobs.cpp jobs_wait): the libraries and the link, not the whole build. // RX_VK_PIPE_FAST_WORKERS=0: the whole build first, as before. +// - choose(), depth_new ((Triforce, 11-22, the performance audit's PIPE-29, after Source B's): a depth-tested draw into a +// pass drawn once whose shaders write no memory, outside a samples-passed count and the item icon capture's kept targets): +// from the frame's wait pool, then left out, and a pipeline that ran out of time is left out at once until it is built +// (vk_draw.cpp). Source B froze 11-13 s twice on an Arc B580 compiling some fifty new pipelines for such passes, and made +// them wait at most 50 ms a frame. RX_VK_PIPE_DEPTH_BUDGET=1; off by default on every driver, Intel's too, where the +// entry would have it on: a render into a new depth target that is left out can stay missing until it is drawn again, +// and that trade is the owner's to make (depth_budget). // A wait from a budget is vk_jobs.cpp jobs_wait_for: the pipeline's job goes to the front of the queue, and is not run on the // draw's thread (a compile there would outlast the bound). #pragma once @@ -46,6 +53,7 @@ #include #include #include +#include namespace pipe_wait { @@ -139,7 +147,12 @@ private: // ---- the draw's choice enum class Act { Skip, Wait, WaitAll }; -enum Kind { K_NONE, K_ONESHOT, K_FULL, K_LAST }; +enum Kind { K_NONE, K_ONESHOT, K_FULL, K_LAST, K_DEPTH }; + +/// ((Triforce, 11-22, PIPE-29) RX_VK_PIPE_DEPTH_BUDGET: on when set to anything but 0; unset, off on every driver) +inline bool depth_budget(const char* env) { + return env && *env && std::atoi(env) != 0; +} struct Rules { bool always_wait = false; // (RX_VK_PIPE_SKIP=0: every pipeline waited for) @@ -155,9 +168,16 @@ struct Choice { /// A draw whose pipeline is not ready. kept: its pass (kept()); writes: its shaders write memory; full: full_screen(); held: /// the hold of the last picture would take this frame if it lacks a draw (present_hold.h DEFERS, the pipelines counted); -/// pool_ns, last_ns: what the frame's wait pool and its last wait have left -inline Choice choose(const Rules& r, bool kept, bool writes, bool full, bool held, int64_t pool_ns, int64_t last_ns) { +/// pool_ns, last_ns: what the frame's wait pool and its last wait have left. depth_new ((Triforce, 11-22, PIPE-29) +/// RX_VK_PIPE_DEPTH_BUDGET, the caller's: a depth-tested draw into a pass drawn once whose shaders write no memory): from the +/// pool, then left out +inline Choice choose(const Rules& r, bool kept, bool writes, bool full, bool held, int64_t pool_ns, int64_t last_ns, + bool depth_new = false) { if (r.always_wait) return {Act::WaitAll, 0, K_NONE}; + if (!kept && !writes && depth_new) { + if (pool_ns >= kMinWaitNs) return {Act::Wait, pool_ns, K_DEPTH}; + return {Act::Skip, 0, K_NONE}; + } if (!kept || writes) { if (!r.oneshot_budget) return {Act::WaitAll, 0, K_NONE}; if (pool_ns >= kMinWaitNs) return {Act::Wait, pool_ns, K_ONESHOT}; diff --git a/gpu/vk/precompile_save.h b/gpu/vk/precompile_save.h new file mode 100644 index 0000000..45c4afb --- /dev/null +++ b/gpu/vk/precompile_save.h @@ -0,0 +1,92 @@ +// The pipeline cache saved before the title after a long precompile, and after a preparation without stage libraries +// (no game, no GPU: a rule; vk_draw.cpp precompile_run, prepare_run, prepare_compute_pipeline, pipeline_cache_save). +// +// (Triforce, 11-22, the performance audit's PIPE-23, after Source A's.) A precompile that keeps every pipeline (enough +// RAM: precompile_keeps_all) ended without a save: what it compiled was first written in play, at present 300 and only +// after 32 more builds, and on Intel only at a clean close. A crash, a kill or a power cut before that cost the whole +// build again at the next start, minutes of it when the driver's own cache did not have them either. Source A wrote its +// cache before the title once the precompile had built for 30 s or more (after a 1,454 s compile on a handheld: 439 MB +// in 821 ms, and the next start's precompile took 13 s). +// +// Here the same, with two differences that are this project's: +// - the save is the play file's alone while there is a play cache (pcache_play.h): the precompile's compiles went into +// it. The main cache's hits count as builds too (cached_create), and on NVIDIA the main cache grows by the driver's +// copies, so an ordinary forced save would write the whole main file, 2 to 3 GB, before the title. With no play +// cache (a start that folded the play file, or no pipelineCreationCacheControl) the main file is what there is, and +// it is saved; +// - not where something else saves or the workers still build: a precompile the game went on from (its workers build +// behind the game: no fetch while they do, Intel's driver crashed there), a stopped one (the stop's own save), a +// start that only prepares (its end saves), and the light precompile (it saves before its record, as before). +// +// The second gap: a preparation without stage libraries compiled its compute pipelines into the main cache, counted +// them nowhere the save looks (pcache_main_built) and ended without a save, where the one with libraries saves after +// them (warm_libraries). Now they are counted, and that preparation saves before its record says it is made. +// +// RX_VK_PRECOMPILE_SAVE_S=: the build time from which the precompile's end saves, 30 by default, at most a day; +// 0: never, and the preparation's count and save are off too, all as before. +#pragma once +#include + +namespace mkwvk::precompile_save { + +inline constexpr int kAfterS = 30; // Source A's +inline constexpr int kMostS = 86400; + +/// RX_VK_PRECOMPILE_SAVE_S: the seconds of building from which the precompile's end saves (0: never) +inline int after_s(const char* env) { + if (!env || !*env) return kAfterS; + const long v = std::atol(env); + return v <= 0 ? 0 : v > kMostS ? kMostS : (int)v; +} + +/// Whether the rule is on at all (the preparation's count and its save go by this too) +constexpr bool on(int save_s) { + return save_s > 0; +} + +/// Whether a precompile that built for build_s seconds is a long one: only those say anything +constexpr bool long_build(double build_s, int save_s) { + return save_s > 0 && build_s >= (double)save_s; +} + +/// Whether the keep-all precompile's end saves now. gone_on: the game went on while workers still build; stopped: a stop +/// was asked; prepare_only: a start that only prepares +constexpr bool due(double build_s, int save_s, bool gone_on, bool stopped, bool prepare_only) { + return long_build(build_s, save_s) && !gone_on && !stopped && !prepare_only; +} + +enum class Outcome { + Saved, // (written: the line gives its size and time) + Nothing, // (the cache holds nothing its files lack) + GoneOn, + Stopped, + PrepareOnly, + NoCache, // (no pipeline cache this session) +}; + +/// Why a long precompile's end does not save (only asked when due() said no, or with no cache) +constexpr Outcome why_not(bool gone_on, bool stopped, bool prepare_only, bool cache) { + return !cache ? Outcome::NoCache : stopped ? Outcome::Stopped : gone_on ? Outcome::GoneOn : + prepare_only ? Outcome::PrepareOnly : Outcome::Nothing; +} + +/// The log line's words after "vk: precompile: N s of building: " +constexpr const char* said(Outcome o) { + switch (o) { + case Outcome::Saved: return "the pipeline cache saved before the title"; + case Outcome::Nothing: return "the pipeline cache holds nothing new to save before the title"; + case Outcome::GoneOn: return "the pipeline cache is not saved before the title: the game went on and the workers still build"; + case Outcome::Stopped: return "the pipeline cache is not saved here: the stop's own save writes it"; + case Outcome::PrepareOnly: return "the pipeline cache is not saved here: this start only prepares and saves at its end"; + case Outcome::NoCache: return "there is no pipeline cache to save"; + } + return ""; +} + +/// Whether a preparation without stage libraries saves the cache before its record is written: the rule on, not behind +/// the game, and compute pipelines built into the main cache +constexpr bool prepare_saves(int save_s, bool libraries_warmed, bool background, unsigned long long computes_built) { + return on(save_s) && !libraries_warmed && !background && computes_built > 0; +} + +} // namespace mkwvk::precompile_save diff --git a/gpu/vk/predict_check.h b/gpu/vk/predict_check.h new file mode 100644 index 0000000..a0bdbaf --- /dev/null +++ b/gpu/vk/predict_check.h @@ -0,0 +1,211 @@ +// Whether the driver still has the predicted pipelines, at a start whose record says they were all built (no game, no +// GPU: the rules; vk_draw.cpp predict_check_cold, predict_background, predicted_play_build). After lib_check.h, which +// asks the same of the stage libraries. +// +// (Triforce, 11-22, the performance audit's PIPE-68, after Source B's and Source A's.) The predicted pipelines (the ones +// the notes predict for programs no note names) are built once for the driver's own cache and let go, and a record +// (.predicted) says which programs are done; once it says all, no start looks again. But the driver's cache is +// the driver's: NVIDIA's has a size cap, AMD's Windows driver kept 30 to 44% of what Source B had prepared, and a cache +// folder can be cleared. Source B found 105 dispatches waiting 115 s in all on prepared pipelines the driver had dropped, +// while its record still said built. Then each of those programs' first draw compiles in play: the freezes on arrival +// the prediction is there to take away. +// +// The check. At a start whose record says all built, on the builder's idle thread after its delay and, as the builder, +// only while the frame shown is a light one: up to kSample predicted pipelines from the plan, spread over it from +// another place each start, each built for the driver's cache alone and timed, then built once more at once. The second +// comes from the driver's memory by construction, so it is this driver's warm time on this PC and no fixed time is +// needed: a first build over kRatio times the second and kSlackMs more was compiled. Not asked with +// FAIL_ON_PIPELINE_COMPILE_REQUIRED: under that flag a driver answers only from the cache it is handed, and these are in +// none of ours. The sample ends once its majority is decided. +// +// Cold (more than half of a sample): the record is removed, so the builder builds them all again behind the game at +// light frames, this session and the next, and .predicted.check takes a strike for this stamp. Cold again at a +// start after they were built again: this driver does not keep them. That is said once and recorded, and no later start +// checks for this stamp (the compiles of the check itself are not free), so the pipelines are left to their first draw +// as before the prediction. Warm: the strike is forgotten. Another stamp (a new driver, new shaders) begins anew. +// +// The counter, for what the check is worth: builds in play of programs the plan predicts that took over 20 ms, which a +// driver that had them does not need. Logged at most every 10 s while it grows. +// +// RX_VK_PREDICT_CHECK: 1 by default; 0: no check and no counter, as before; 2: checked at every start, a recorded +// "does not keep them" too (the lab's). +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace predict_check { + +inline constexpr int kSample = 12; // (pipelines sampled, as lib_check's) +inline constexpr int kLook = 2 * kSample; // (plan entries looked at for them: one may have lost its notes or its program) +inline constexpr double kRatio = 4.0; // (the first build against the second: compiled above) +inline constexpr double kSlackMs = 3.0; // (and this much more: lookups of a few milliseconds say nothing) +inline constexpr int kStrikes = 1; // (the record removed once; cold again after that: stop) +inline constexpr int64_t kSlowUs = 20000; // (a play build of a predicted program over this was compiled) +inline constexpr int64_t kSayMs = 10000; // (the counter's line: at most this often) + +/// RX_VK_PREDICT_CHECK: 0 off, 1 on (unset too), 2 every start +inline int mode(const char* env) { + if (!env || !*env) return 1; + const int v = std::atoi(env); + return v <= 0 ? 0 : v >= 2 ? 2 : 1; +} + +/// One pipeline sampled: made both times, and what each took (ms). One the driver refuses says nothing of its cache +struct Probe { + bool made = false; + double first_ms = 0, again_ms = 0; +}; +inline bool cold(const Probe& p) { + return p.made && p.first_ms > kRatio * p.again_ms + kSlackMs; +} + +struct Tally { + int got = 0, cold = 0; + double first_median = 0, again_median = 0; +}; +inline double median(std::vector v) { + if (v.empty()) return 0; + std::sort(v.begin(), v.end()); + return v[v.size() / 2]; +} +inline Tally tally(const std::vector& probes) { + Tally t; + std::vector first, again; + for (const Probe& p : probes) { + if (!p.made) continue; + t.got++, t.cold += cold(p); + first.push_back(p.first_ms), again.push_back(p.again_ms); + } + t.first_median = median(first), t.again_median = median(again); + return t; +} + +enum class Verdict { Unknown, Warm, Cold }; +/// Cold when more than half of the sample is; under half a sample says nothing +inline Verdict verdict(const Tally& t) { + if (t.got * 2 < kSample) return Verdict::Unknown; + return t.cold * 2 > t.got ? Verdict::Cold : Verdict::Warm; +} +/// Whether the sample needs no more probes: it is full, or what is still to come cannot change its verdict +inline bool decided(const Tally& t) { + return t.got >= kSample || t.cold * 2 > kSample || (t.got - t.cold) * 2 >= kSample; +} + +/// k places spread over n, the first at seed's place in the first step: each start samples other pipelines +inline std::vector spots(size_t n, size_t k, uint64_t seed) { + std::vector at; + if (!n || !k) return at; + const size_t step = std::max(1, n / k); + for (size_t i = (size_t)(seed % step); i < n && at.size() < k; i += step) at.push_back(i); + return at; +} + +/// The records of some notes in the notes file f, read from its start: a head of {stamp, rec_bytes}, then records of +/// rec_bytes whose first 8 bytes are the key (0: the first after the head). Only the keys are read, each record's rest +/// stepped over, and the walk ends once every wanted key is found. A key twice keeps its first record. Empty when the +/// head is another build's +inline std::unordered_map find(FILE* f, uint64_t stamp, uint64_t rec_bytes, const std::vector& keys) { + std::unordered_map at; + uint64_t head[2] = {0, 0}; + if (!f || rec_bytes < 8 || rec_bytes > (1u << 30) || fread(head, 8, 2, f) != 2 || head[0] != stamp || head[1] != rec_bytes) return at; + std::unordered_map want; + for (const uint64_t k : keys) want.emplace(k, false); + size_t left = want.size(); + uint64_t key = 0; + for (uint32_t i = 0; left && fread(&key, 8, 1, f) == 1; i++) { + const auto w = want.find(key); + if (w != want.end() && !w->second) w->second = true, at.emplace(key, i), left--; + if (fseek(f, (long)(rec_bytes - 8), SEEK_CUR) != 0) break; + } + return at; +} + +// ---- .predicted.check: "predict-check " + +struct State { + bool found = false; + uint64_t stamp = 0; + int strikes = 0; +}; +inline State state_read(const std::string& path) { + State s; + FILE* f = fopen(path.c_str(), "rb"); + if (!f) return s; + char line[96]{}; + const size_t n = fread(line, 1, sizeof line - 1, f); + fclose(f); + line[n] = 0; + unsigned long long stamp = 0; + int strikes = 0; + if (sscanf(line, "predict-check %16llx %d", &stamp, &strikes) == 2 && strikes >= 0) + s.found = true, s.stamp = stamp, s.strikes = strikes; + return s; +} +/// Written whole beside the old one, then put in place (a start that ends in the middle leaves the old file) +inline bool state_write(const std::string& path, uint64_t stamp, int strikes) { + char line[96]; + const int n = snprintf(line, sizeof line, "predict-check %016llx %d\n", (unsigned long long)stamp, strikes); + const std::string tmp = path + ".tmp"; + FILE* f = fopen(tmp.c_str(), "wb"); + if (!f) return false; + bool ok = fwrite(line, 1, (size_t)n, f) == (size_t)n; + ok = fclose(f) == 0 && ok; + if (ok) { + remove(path.c_str()); // (rename does not replace on Windows) + ok = rename(tmp.c_str(), path.c_str()) == 0; + } + if (!ok) remove(tmp.c_str()); + return ok; +} + +/// Whether the record says this driver does not keep them, for this stamp: cold again after they were built again +inline bool stopped(const State& had, uint64_t stamp) { + return had.found && had.stamp == stamp && had.strikes > kStrikes; +} +/// Whether a start checks at all: the switch, and (but for mode 2) not once the record says stop for this stamp +inline bool runs(int mode, const State& had, uint64_t stamp) { + return mode >= 2 || (mode == 1 && !stopped(had, stamp)); +} + +enum class Do { Nothing, Again, Stop }; +/// What a start does with a sample's verdict, the record of the last starts and the stamp now. *next: the strikes to +/// record (Again and Stop). Again: the record of what is built is removed, and the builder builds them again +inline Do decide(Verdict v, const State& had, uint64_t stamp, int* next) { + if (v != Verdict::Cold) return Do::Nothing; + const bool same = had.found && had.stamp == stamp; // (found cold for this stamp before, and built again since) + *next = same ? had.strikes + 1 : 1; + return same && had.strikes >= kStrikes ? Do::Stop : Do::Again; +} + +// ---- the counter: play builds of predicted programs the driver compiled + +/// Whether a build of that many microseconds was compiled rather than found +inline bool slow(int64_t us) { + return us > kSlowUs; +} +/// Whether the counter's line is due: said_ms, when it was last written (0: never) +inline bool say(int64_t now_ms, int64_t said_ms) { + return !said_ms || now_ms - said_ms >= kSayMs; +} +/// The plan's programs as a sorted list without repeats, for has() (Plan: entries with .prog) +template +std::vector programs(const std::vector& plan) { + std::vector keys; + keys.reserve(plan.size()); + for (const Plan& p : plan) keys.push_back(p.prog); + std::sort(keys.begin(), keys.end()); + keys.erase(std::unique(keys.begin(), keys.end()), keys.end()); + return keys; +} +inline bool has(const std::vector& sorted, uint64_t prog) { + return std::binary_search(sorted.begin(), sorted.end(), prog); +} + +} // namespace predict_check diff --git a/gpu/vk/predict_stream.h b/gpu/vk/predict_stream.h new file mode 100644 index 0000000..23bec47 --- /dev/null +++ b/gpu/vk/predict_stream.h @@ -0,0 +1,74 @@ +// (Triforce, 11-21, the performance audit's PIPE-83, after Source B's) The predicted builder's plan as record numbers, its +// notes read when their program's turn comes. +// +// predict_background (vk_draw.cpp) builds, behind the game, the predicted pipelines the start did not get to: on the starts +// after a translator or driver update that is most of them, and it rests while the game is in play, so it runs for much of +// a session. It read every note of the notes file (a PipeNote is ~27 KB), made a full note of every predicted pipeline, and +// held both until it ended: about 1 GB with the game's 14,535 notes and 24,182 predicted pipelines. The draws, the +// read-ahead and the warm read a note by its record already (pipe_note_at). Here: the notes file is gone through once into +// one record's buffer for each key's record number, the plan becomes {variant, the vertex note's record, the output note's +// record} for each program still to build, 16 bytes a pipeline, and a worker reads the two notes through its own handle +// when it builds that pipeline. The same programs and pipelines; a note gone since drops its pipeline, as before. +// RX_VK_PREDICT_BG_STREAM=0: every note and predicted note held, as before. +// +// No Vulkan and no PipeNote here: tests/test_predict_stream.py compiles it with a file of its own. +#pragma once +#include +#include +#include +#include +#include +#include + +namespace predict_stream { + +/// One predicted pipeline: its program's variant, and the records of the note whose vertex layout and of the note whose +/// targets and state it takes +struct Entry { + uint64_t variant; + uint32_t vertex_rec, output_rec; +}; +/// A program still to build and its pipelines, in the plan's order +struct Program { + uint64_t prog; + std::vector entries; +}; + +/// Each note's key to its record (0: the first after the head), of the notes file f at its start: a head of {stamp, +/// rec_bytes}, then records of rec_bytes whose first 8 bytes are the key. rec: a buffer of rec_bytes each is read into; +/// sane(): whether the one in it is used (it may mend it). A key twice keeps its first record, as pipe_notes_read. Empty +/// when the head is another build's +template +std::unordered_map index(FILE* f, uint64_t stamp, uint64_t rec_bytes, void* rec, Sane&& sane) { + std::unordered_map at; + uint64_t head[2] = {0, 0}; + if (!f || rec_bytes < 8 || fread(head, 8, 2, f) != 2 || head[0] != stamp || head[1] != rec_bytes) return at; + for (uint32_t i = 0; fread(rec, (size_t)rec_bytes, 1, f) == 1; i++) { + if (!sane()) continue; + uint64_t key; + std::memcpy(&key, rec, 8); + at.emplace(key, i); + } + return at; +} + +/// The plan (entries with prog, variant, vertex_note, output_note: PredictPlan) as the programs still to build: those not +/// in had, in the order the plan first names them, each with its pipelines in the plan's order. A pipeline one of whose +/// notes is not in the index is dropped, and a program left with none is not listed +template +std::vector group(const std::vector& plan, const std::unordered_map& at, + const std::unordered_set& had) { + std::vector out; + std::unordered_map where; + for (const Plan& p : plan) { + if (had.count(p.prog)) continue; + const auto v = at.find(p.vertex_note), o = at.find(p.output_note); + if (v == at.end() || o == at.end()) continue; + const auto [it, fresh] = where.try_emplace(p.prog, out.size()); + if (fresh) out.push_back(Program{p.prog, {}}); + out[it->second].entries.push_back(Entry{p.variant, v->second, o->second}); + } + return out; +} + +} // namespace predict_stream diff --git a/gpu/vk/predict_types.h b/gpu/vk/predict_types.h new file mode 100644 index 0000000..214e57f --- /dev/null +++ b/gpu/vk/predict_types.h @@ -0,0 +1,96 @@ +// The prediction's checks (no game, no GPU: vk_predict.cpp predict_pipelines and vk_draw.cpp predict_run, +// predict_background): whether a predicted vertex layout can feed a program's vertex inputs, and the switch. +// +// (Triforce, 11-22, the performance audit's PIPE-44, after Source A's.) The predicted pipelines take a program no note names +// and give it the vertex layout of notes of the same vertex module, or else the layout most programs with its vertex inputs +// had. A program's key already says which of its inputs it reads as integers (vk_shader.cpp graphics_key), so a layout +// borrowed from another program can hand an integer input a float attribute, or the other signedness: such a pipeline is +// invalid (VUID-VkGraphicsPipelineCreateInfo-Input-08733), built for nothing or worse. Source A reads each vertex input's +// base type from the module's SPIR-V and drops a predicted layout whose attribute types definitely differ: 8,209 of 64,797 +// predicted pipelines on its PC, none by colour. Scaled formats count as integers here (the translator reads them so). +// +// And a program's predictions can give the same pipeline twice (two layouts that differ only in what is dynamic, a stride): +// Source A found 420 built twice. Built once each now, by the pipeline's key, as play keys them: the same pipelines. +// +// RX_VK_PREDICT_TYPES: unset, each pipeline built once (the same pipelines as before, fewer builds); 1, that and the type +// check, which drops pipelines (off by default: it changes which pipelines exist, and a plan is made once for the shaders, +// so it applies to a plan made after it is set); 0, as before. +#pragma once + +#include +#include +#include +#include +#include + +namespace predict_types { + +/// RX_VK_PREDICT_TYPES: 0 as before, 1 (unset) built once, 2 built once and the type check +inline int mode(const char* env) { + if (!env || !*env) return 1; + return std::atoi(env) == 0 ? 0 : std::atoi(env) == 1 ? 2 : 1; +} +inline bool once(int mode) { return mode >= 1; } +inline bool typed(int mode) { return mode >= 2; } + +/// A module's vertex inputs by location (under 32): read, read as an integer, read as a signed integer +struct Locations { + uint32_t mask = 0, ints = 0, sign = 0; +}; + +/// The Input (storage class 1) variables with a Location in a SPIR-V module, and each one's base type: through its +/// pointer type to a vector's or an array's element, an OpTypeInt (signedness) or anything else (float). Built-ins carry +/// no Location and are not counted +inline Locations locations(const std::vector& w) { + std::unordered_map loc, pointee, element; + std::unordered_map int_sign; // (OpTypeInt id: 1 signed, 0 unsigned) + std::vector> inputs; // (variable id, its pointer type) + for (size_t i = 5; i < w.size();) { + const uint32_t op = w[i] & 0xFFFF, n = w[i] >> 16; + if (!n || i + n > w.size()) break; + if (op == 71 && n >= 4 && w[i + 2] == 30) loc[w[i + 1]] = w[i + 3]; // OpDecorate Location + else if (op == 59 && n >= 4 && w[i + 3] == 1) inputs.push_back({w[i + 2], w[i + 1]}); // OpVariable Input + else if (op == 32 && n >= 4) pointee[w[i + 1]] = w[i + 3]; // OpTypePointer + else if ((op == 23 || op == 28) && n >= 4) element[w[i + 1]] = w[i + 2]; // OpTypeVector, OpTypeArray + else if (op == 21 && n >= 4) int_sign[w[i + 1]] = w[i + 3] ? 1 : 0; // OpTypeInt + i += n; + } + Locations out; + for (const auto& [var, ptr] : inputs) { + const auto l = loc.find(var); + if (l == loc.end() || l->second >= 32) continue; + const auto p = pointee.find(ptr); + if (p == pointee.end()) continue; + uint32_t t = p->second; + for (int depth = 0; depth < 4; depth++) { // (a vector of, an array of vectors of) + const auto e = element.find(t); + if (e == element.end()) break; + t = e->second; + } + const uint32_t bit = 1u << l->second; + out.mask |= bit; + if (const auto s = int_sign.find(t); s != int_sign.end()) { + out.ints |= bit; + if (s->second) out.sign |= bit; + } + } + return out; +} + +/// What an attribute of the layout hands its location: nothing known (unset: it reads zeros, as play's), a float, an +/// unsigned or a signed integer (scaled formats are read as integers) +enum class Kind : uint8_t { Unknown, Float, Uint, Sint }; + +/// Whether a layout's attributes (kinds by location) can feed the module's inputs: false only where a read location's +/// attribute is definitely of the other kind (integer against float, or the other signedness) +inline bool fits(const Locations& in, const Kind kinds[16]) { + for (int i = 0; i < 16; i++) { + if (!(in.mask >> i & 1) || kinds[i] == Kind::Unknown) continue; + const bool want_int = in.ints >> i & 1, want_sign = in.sign >> i & 1; + if ((kinds[i] != Kind::Float) != want_int) return false; + if (want_int && (kinds[i] == Kind::Sint) != want_sign) return false; + } + return true; +} + +} // namespace predict_types diff --git a/gpu/vk/prepare_again.h b/gpu/vk/prepare_again.h new file mode 100644 index 0000000..712fcb8 --- /dev/null +++ b/gpu/vk/prepare_again.h @@ -0,0 +1,79 @@ +// (Triforce, 11-22, after Casual Gamer's #dev-fixes post) Two ways a start took its shaders for prepared when they were not, apart +// from Vulkan so that tests/test_prepare_again.py runs them (vk_draw.cpp prepare_run). +// +// The program cache made again for another GPU. triforce_prepared.bin says the shaders were translated for this stamp +// (program_cache_stamp: the translator and what of the GPU and its driver changes a translation). A start on another GPU or +// with other settings that did not prepare (or did not rewrite the record) makes triforce_programs.bin again for its own +// stamp: back here the record still matches, prepare_check says nothing to do, the program cache opens with none of these +// shaders (vk_shader.cpp disk_init throws the other stamp's away), and every program is translated in play. So when the +// record says nothing to do, the program cache's head (kDiskMagic, its stamp) and size are read: a cache of another stamp, or +// of its head alone, holds none of them, and the shaders are prepared again. A cache carried over an update keeps the stamp in +// its head (its second word, the build that wrote it, is another: program_carry.h) and passes. +// +// The mark of the driver's part without stage libraries, .warming: "this step began and did not end", an empty file. +// A start that found it left the part out for good, on whatever GPU and driver came next. Now it holds what the part is made +// for (vk_draw.cpp libraries_stamp: the GPU, the driver, the programs), as the libraries' journal does (gpl_rules.h +// journal_left): a mark of another stamp is another driver's, and the part goes on. An empty one (an older build's) is this +// driver's, as before. +// RX_VK_PREPARE_RECHECK=0: neither, as before. +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace prepare_again { + +inline constexpr uint32_t kProgramsMagic = 0x31505348; // (vk_shader.cpp kDiskMagic: the program cache's head) +inline constexpr uint64_t kProgramsHead = 16; // (the magic, the build that wrote it, the stamp) + +/// RX_VK_PREPARE_RECHECK: on unless it is 0 +inline bool on(const char* env) { return !(env && env[0] == '0' && !env[1]); } + +/// Whether a program cache, by its first n bytes (head) and its size, holds none of the shaders of stamp: no program cache's +/// head, another stamp's, or its head alone +inline bool programs_none(const uint8_t* head, size_t n, uint64_t size, uint64_t stamp) { + if (n < kProgramsHead || size <= kProgramsHead) return true; + uint32_t magic; + uint64_t was; + std::memcpy(&magic, head, 4), std::memcpy(&was, head + 8, 8); + return magic != kProgramsMagic || was != stamp; +} +/// The same for the file at path (none there: none of them). *size: its size, 0 when there is none +inline bool programs_none_at(const std::string& path, uint64_t stamp, uint64_t* size = nullptr) { + std::error_code ec; + const uint64_t n = std::filesystem::is_regular_file(path, ec) && !ec ? (uint64_t)std::filesystem::file_size(path, ec) : 0; + const uint64_t whole = ec ? 0 : n; + if (size) *size = whole; + uint8_t head[kProgramsHead]{}; + size_t got = 0; + if (FILE* f = std::fopen(path.c_str(), "rb")) { + got = std::fread(head, 1, sizeof head, f); + std::fclose(f); + } + return programs_none(head, got, whole, stamp); +} + +/// What the last start left of the driver's part without stage libraries: no mark, a mark of this stamp (or an empty one, an +/// older build's; or any, with by_stamp false: RX_VK_PREPARE_RECHECK=0), or one of another stamp +enum class Mark { None, Ours, Other }; +inline Mark mark_left(const std::string& path, uint64_t stamp, bool by_stamp) { + FILE* f = std::fopen(path.c_str(), "rb"); + if (!f) return Mark::None; + uint64_t was = 0; + const bool stamped = std::fread(&was, 8, 1, f) == 1; + std::fclose(f); + return !by_stamp || !stamped || was == stamp ? Mark::Ours : Mark::Other; +} +/// The mark written before the part: the stamp it is made for (empty with by_stamp false, as before). False: not written +inline bool mark_write(const std::string& path, uint64_t stamp, bool by_stamp) { + FILE* f = std::fopen(path.c_str(), "wb"); + if (!f) return false; + const bool ok = !by_stamp || std::fwrite(&stamp, 8, 1, f) == 1; + return std::fclose(f) == 0 && ok; +} + +} // namespace prepare_again diff --git a/gpu/vk/prepare_threads.h b/gpu/vk/prepare_threads.h index 4455dab..809bba5 100644 --- a/gpu/vk/prepare_threads.h +++ b/gpu/vk/prepare_threads.h @@ -58,4 +58,14 @@ inline unsigned translate(unsigned all, bool amd_older, bool asked) { return amd_older && !asked ? std::min(all, kAmdOlderMost) : all; } +/// (Triforce) Which heap the process is on (Windows), by the signature word at byte 16 of GetProcessHeap()'s header: the +/// segment heap's (an exe built with HOOKSHOT_SEGMENT_HEAP=1, nx2/win_manifest.py) or the NT heap's; said in the log's line +/// on the preparation's threads, so that a run says which of the two exes it was +inline constexpr uint32_t kSegmentHeapSignature = 0xDDEEDDEEu; +inline constexpr uint32_t kNtHeapSignature = 0xFFEEFFEEu; +inline const char* heap_name(uint32_t signature) { + return signature == kSegmentHeapSignature ? "segment" : signature == kNtHeapSignature ? "NT" : "not known"; +} +inline bool segment_heap(uint32_t signature) { return signature == kSegmentHeapSignature; } + } // namespace prepare_threads diff --git a/gpu/vk/program_order.h b/gpu/vk/program_order.h index aa420a7..fcac207 100644 --- a/gpu/vk/program_order.h +++ b/gpu/vk/program_order.h @@ -100,11 +100,11 @@ inline std::vector write(const std::vector& keys, uint32_t ma } /// The keys of a file's bytes; none when they are not a whole file of this version (cut short, another file, a bad check word) -inline std::vector read(const uint8_t* data, size_t n, uint32_t magic = kMagic) { +inline std::vector read(const uint8_t* data, size_t n, uint32_t magic = kMagic, size_t max_keys = kKeysMax) { uint32_t head[4]; if (!data || n < 24) return {}; std::memcpy(head, data, 16); - if (head[0] != magic || head[1] != kVersion || head[2] > kKeysMax || n != 16 + (size_t)head[2] * 8 + 8) return {}; + if (head[0] != magic || head[1] != kVersion || head[2] > max_keys || n != 16 + (size_t)head[2] * 8 + 8) return {}; std::vector keys(head[2]); if (!keys.empty()) std::memcpy(keys.data(), data + 16, keys.size() * 8); uint64_t c; @@ -129,8 +129,9 @@ struct Record { std::vector keys; std::unordered_set seen; size_t saved = 0; // (keys.size() at the last write) + size_t max = kKeysMax; // ((Triforce, 11-21) the pipelines' order keeps more: pipe_order.h kKeysMax) bool add(uint64_t key) { - if (!key || keys.size() >= kKeysMax || !seen.insert(key).second) return false; + if (!key || keys.size() >= max || !seen.insert(key).second) return false; keys.push_back(key); return true; } diff --git a/gpu/vk/range_walk.h b/gpu/vk/range_walk.h new file mode 100644 index 0000000..b493cea --- /dev/null +++ b/gpu/vk/range_walk.h @@ -0,0 +1,37 @@ +// (Triforce, 11-22, the performance audit's MEM-42, after Source A's) Is an address in a sorted list of ranges, asked for +// the pages of a whole loop: a cursor that only moves forward. +// +// The capture and collection loops ask of every page whether it lies in a range list sorted by first address (the +// captures' ranges, the texture storage's), and each asked with a binary search. A loop's pages come ascending, or +// nearly, so one pass over the ranges serves the whole loop. An address below the last one asked searches again, so any +// order gives the answers the search gave: tests/test_range_walk.py compares the two. +// +// A walk lives inside one loop, under the lock of its list: the list must not change while it is used. +#pragma once +#include +#include +#include +#include +#include + +namespace range_walk { + +using Ranges = std::vector>; // ([first, second), sorted by first) + +struct RangeWalk { + const Ranges& r; + bool search = false; // (always search, as before: RX_VK_RANGE_WALK=0, the lab knob norangewalk) + size_t next = 0; // (the first range that begins after the last address asked) + uint64_t last = 0; + + /// a lies in one of the ranges: std::upper_bound(r, {a, ~0}) != begin && a < prev->second, as the loops asked + bool in(uint64_t a) { + if (search || a < last) next = (size_t)(std::upper_bound(r.begin(), r.end(), std::make_pair(a, ~uint64_t(0))) - r.begin()); + else + while (next < r.size() && r[next].first <= a) next++; + last = a; + return next != 0 && a < r[next - 1].second; + } +}; + +} // namespace range_walk diff --git a/gpu/vk/readback_late.h b/gpu/vk/readback_late.h new file mode 100644 index 0000000..d07c6c1 --- /dev/null +++ b/gpu/vk/readback_late.h @@ -0,0 +1,123 @@ +// (Triforce, 11-22, the performance audit's SYNC-19, after Source A's) Late readbacks: ranges the CPU reads back without +// the wait for their write-back. The replay's invalidate (vk_replay.cpp) waits for the write-back of every range the game is +// about to read: without it the game read 0 from its exposure meter one frame and twice the sum the next (the white flashes). +// Some ranges the game reads right after it submitted the work that fills them, so the console cannot count on that work +// either: there the wait is the GPU's whole latency for nothing but a value one frame newer. Source A names such ranges and +// reads them from what guest memory holds, the last write-back made: +// a size in hex every range of that size (Source A's default is 3980: the culling camera block, 0x3980 bytes) +// newest a range whose writer is the newest submission handed out (the game cannot have waited for it) +// RX_VK_READBACK_LATE=[,...]. Unset: NO rule here until the lab has run it (the audit's lab proof: a handheld or an +// AMD iGPU, where the waits are long); =0: none either. The lab knob rblate (lab_ab.h) turns the rules a later default would +// have (default_rules: 3980) on in its phases when the variable is unset. A late range keeps its note for the CPU's +// write-back (memory_note_readback) and skips the catch-up and the wait, nothing else. +// What it risks: a late read hands the game the frame before's values; with mirrors, when the CPU resets a range read late, +// the pending write-back lands over its reset (the exposure meter's 2x flashes). `newest` can catch that meter, so it is +// never a default. RX_VK_LOG_READBACK_RANGES=1 counts the readbacks by range (RangeLog) to find such ranges on other hardware. +// tests/test_readback_late.py +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include + +namespace mkwvk::readback_late { + +constexpr int kMaxSizes = 8; +/// The size Source A reads late by default: the culling camera block +constexpr uint64_t kCameraBlock = 0x3980; + +struct Rules { + uint64_t sizes[kMaxSizes] = {}; + int n = 0; + bool newest = false; + int bad = 0; // (parts of the text that were neither a hex size nor "newest", or sizes past kMaxSizes: left out) + bool any() const { return n > 0 || newest; } + bool has(uint64_t size) const { + for (int i = 0; i < n; i++) + if (sizes[i] == size) return true; + return false; + } +}; + +/// The rules a default would have once the lab has passed them (and the knob rblate has now): the camera block's size alone +inline Rules default_rules() { + Rules r; + r.sizes[r.n++] = kCameraBlock; + return r; +} + +/// RX_VK_READBACK_LATE's text. Unset or empty: no rule (until the lab passes; then default_rules()). "0": none. Otherwise parts +/// separated by commas: "newest", or a size in hex with or without 0x. A part that is neither is left out and counted in bad +inline Rules parse(const char* text) { + Rules r; + if (!text || !*text) return r; + for (const char* p = text; *p;) { + const char* end = std::strchr(p, ','); + const size_t len = end ? (size_t)(end - p) : std::strlen(p); + if (len == 6 && !std::strncmp(p, "newest", 6)) { + r.newest = true; + } else if (len > 0 && len < 20) { + char part[24] = {}; + std::memcpy(part, p, len); + char* stop = nullptr; + const unsigned long long v = std::strtoull(part, &stop, 16); + if (stop == part || *stop) r.bad++; + else if (v == 0) {} // ("0": none, and no rule of its own) + else if (r.has(v)) {} + else if (r.n < kMaxSizes) r.sizes[r.n++] = v; + else r.bad++; + } else if (len) { + r.bad++; + } + if (!end) break; + p = end + 1; + } + return r; +} + +/// Whether a readback of `size` bytes is read late. writer_newest: its write-back's value is at or past the newest value +/// handed out (the fence a game's submit got last) +inline bool wanted(const Rules& r, uint64_t size, bool writer_newest) { + return (r.newest && writer_newest) || r.has(size); +} +/// The decision invalidate makes: only a range whose write-back is still ahead of the GPU (pending) can be late; one whose +/// write-back is done waits for nothing anyway +inline bool late(const Rules& r, uint64_t size, uint64_t writer_value, uint64_t gpu_done, uint64_t newest_handed_out) { + return r.any() && writer_value > gpu_done && wanted(r, size, writer_value >= newest_handed_out); +} + +/// RX_VK_LOG_READBACK_RANGES: the readbacks by range. Not thread safe: the caller locks +struct RangeLog { + struct Row { + int64_t readbacks = 0, waits = 0, wait_ns = 0, newest = 0, late = 0; + }; + std::map, Row> rows; // ((address, size)) + size_t cap = 4096; // (the game reads a ring of slots at new addresses: past this, the rest is counted together) + Row other; + void note(uint64_t gpu, uint64_t size, bool waited, int64_t ns, bool writer_newest, bool was_late) { + Row* row = &other; + auto it = rows.find({gpu, size}); + if (it != rows.end()) row = &it->second; + else if (rows.size() < cap) row = &rows[{gpu, size}]; + row->readbacks++, row->waits += waited, row->wait_ns += waited ? ns : 0, row->newest += writer_newest, row->late += was_late; + } + struct Line { + uint64_t gpu = 0, size = 0; + Row row; + }; + /// The n ranges that waited longest in all, longest first (ties: the lower address first) + std::vector top(size_t n) const { + std::vector all; + all.reserve(rows.size()); + for (const auto& [key, row] : rows) all.push_back({key.first, key.second, row}); + std::stable_sort(all.begin(), all.end(), [](const Line& a, const Line& b) { return a.row.wait_ns > b.row.wait_ns; }); + if (all.size() > n) all.resize(n); + return all; + } +}; + +} // namespace mkwvk::readback_late diff --git a/gpu/vk/recorder_auto.h b/gpu/vk/recorder_auto.h new file mode 100644 index 0000000..9bb9a9a --- /dev/null +++ b/gpu/vk/recorder_auto.h @@ -0,0 +1,56 @@ +// (Triforce) The recorder thread's default (vk_recorder.h g_auto_thread, RX_VK_RECORDER unset), chosen once the device is known +// (vk_main.cpp pick_device). Plain code with no Vulkan or system calls: tests/test_recorder_auto.py runs it on made-up PCs. +// +// - NVIDIA: on with 12 logical processors or more, as since 11-16 (lab, RTX 4070 SUPER + i7-13700K: Market +11%; held to 8 +// logical processors -10%: the recorder needs a core of its own). +// - AMD: on with 6 physical cores or more in the process's affinity. AMD's driver spends about twice NVIDIA's CPU time on a call, +// so the replay thread's vkCmd* calls cost it more; taken out of a renderer of the same per-call design, the densest Market view +// (~2,500 draws) fell 16-25% behind on RX 7900 XTX, 9070 XT and 9060 XT cards. Physical cores, not logical processors: the +// recorder spins on a core while the replay records, and an SMT sibling of a busy core is not one (a 4-core Steam Deck: off). +// Not on AMD's older Windows driver (vk_vendor.h amd_older_compiler, RX 5000 and RX 6000): it lost the device with commands +// recorded beside the replay thread (replay_threads.h), so it waits for a lab of its own. +// - Anything else (Intel, a GPU not named): off, nothing measured. +// RX_VK_RECORDER=thread or =off goes over this either way. +#pragma once + +#include + +namespace mkwvk::recorder_default { + +enum Why : int { kNvidia, kNvidiaFew, kAmd, kAmdFew, kAmdUnknown, kAmdOlder, kOther }; + +/// Why the recorder is on or off by default. logical: cpu_count() (cpu_count.h: hardware_concurrency, or the lab's HOS_SIM_CPU); physical: physical_cores (0: unknown) +constexpr Why why(uint32_t vendor_id, unsigned logical, unsigned physical, bool amd_older) { + if (vendor_id == 0x10DE) return logical >= 12 ? kNvidia : kNvidiaFew; + if (vendor_id == 0x1002) { + if (amd_older) return kAmdOlder; + if (!physical) return kAmdUnknown; + return physical >= 6 ? kAmd : kAmdFew; + } + return kOther; +} +constexpr bool on(Why w) { return w == kNvidia || w == kAmd; } +constexpr bool rule(uint32_t vendor_id, unsigned logical, unsigned physical, bool amd_older) { + return on(why(vendor_id, logical, physical, amd_older)); +} +constexpr const char* text(Why w) { + switch (w) { + case kNvidia: return "NVIDIA, 12 logical processors or more"; + case kNvidiaFew: return "NVIDIA, under 12 logical processors"; + case kAmd: return "AMD, 6 physical cores or more"; + case kAmdFew: return "AMD, under 6 physical cores"; + case kAmdUnknown: return "AMD, the physical cores not known"; + case kAmdOlder: return "AMD's driver for RX 5000 and RX 6000 (not tried there)"; + default: return "not an NVIDIA or AMD GPU (not measured)"; + } +} + +/// The physical cores with at least one logical processor in affinity: masks[i] is core i's logical processors (one +/// RelationProcessorCore entry each: GetLogicalProcessorInformation, which on Linux reads sysfs's package and core ids) +inline unsigned physical_cores(const uint64_t* masks, int n, uint64_t affinity) { + unsigned cores = 0; + for (int i = 0; i < n; i++) cores += (masks[i] & affinity) != 0; + return cores; +} + +} // namespace mkwvk::recorder_default diff --git a/gpu/vk/redraw_state.h b/gpu/vk/redraw_state.h new file mode 100644 index 0000000..780b352 --- /dev/null +++ b/gpu/vk/redraw_state.h @@ -0,0 +1,57 @@ +// (Triforce, 11-21, the upscaler audit's UPS-03) What a redraw of the stored draws has to set before it draws. +// +// The stored draws are drawn again into the motion vectors in one pass (vk_temporal.cpp capture_motion), about 900 of +// them a frame in the owner's log of 11-21. Each one set the primitive topology, the primitive restart and the index +// buffer before it drew, whatever the draw before had set: three commands a draw beside its push constants and the draw +// itself, and nearly every draw of the scene is a list of triangles without restart. Dynamic state and the index buffer +// stay as they were set until they are set again in the same command buffer, so a command that would set what is set +// already says nothing new. Here is what the pass has set so far, and whether a draw's own differs from it. +// +// A pass starts knowing nothing (a new Pass): its first draw sets everything, as before. +// RX_VK_REDRAW_STATE=0: every draw sets all three, as before (every()). +// +// No Vulkan: tests/test_redraw_state.py compiles this. +#pragma once +#include +#include + +namespace redraw_state { + +/// RX_VK_REDRAW_STATE (value, null when unset): 0, every draw sets its state whatever is set, as before +inline bool every(const char* value) { return value && *value && std::atoi(value) == 0; } + +/// What one pass has set so far, from its first draw on +struct Pass { + bool all = false; // (every(): each asks true) + bool have_topology = false, have_restart = false, have_index = false; + uint32_t topology = 0; + bool restart = false; + uint64_t index_buffer = 0, index_offset = 0; + uint32_t index_type = 0; + uint64_t set = 0, left = 0; // (commands recorded, and left out: for a count in the log) +}; + +/// Whether the topology is to be set for a draw of `topology` (and it is then what the pass has set) +inline bool topology(Pass& p, uint32_t topology) { + if (!p.all && p.have_topology && p.topology == topology) return p.left++, false; + p.have_topology = true, p.topology = topology; + return p.set++, true; +} + +/// The same for primitive restart +inline bool restart(Pass& p, bool restart) { + if (!p.all && p.have_restart && p.restart == restart) return p.left++, false; + p.have_restart = true, p.restart = restart; + return p.set++, true; +} + +/// The same for an indexed draw's index buffer: the buffer's handle as a number, the offset into it and the index type, +/// all three the same as the one bound +inline bool index(Pass& p, uint64_t buffer, uint64_t offset, uint32_t type) { + if (!p.all && p.have_index && p.index_buffer == buffer && p.index_offset == offset && p.index_type == type) + return p.left++, false; + p.have_index = true, p.index_buffer = buffer, p.index_offset = offset, p.index_type = type; + return p.set++, true; +} + +} // namespace redraw_state diff --git a/gpu/vk/release_queue.h b/gpu/vk/release_queue.h new file mode 100644 index 0000000..03f817c --- /dev/null +++ b/gpu/vk/release_queue.h @@ -0,0 +1,64 @@ +// (Triforce, 11-22, the performance audit's MEM-85, after Source B's idea) Texture and program releases queued for the +// replaying thread, not waited for. +// +// The game's nvnTextureFinalize and nvnProgramFinalize reach vk_replay.cpp texture_released_of and program_released_locked. +// Without the replay thread (RX_VK_ASYNC=0, the default) a release is held for the replaying thread only while a queue is +// stopped at a wait (PR 430: after the stopped work, which may still use it); otherwise the game's thread takes replay_mu, +// which a replay holds through whole submissions: the finalizing thread waited 45% of its time in Kokiri before PR 430. +// +// RX_VK_RELEASE_QUEUE=1 (OFF unless set; the lab knob relq in its phases while it is unset): every such release is pushed to +// a list under held_mu, stopped queue or not, and applied by the thread that replays next, at the start of a submission's +// replay or a present (replay_mu held there already), when no queue is stopped; with one stopped, update_blocked applies +// them after the stopped work, as PR 430's. A texture's release carries the write generation of its call, so the CPU's +// writes after the call still count as written after the release (vk_texture.cpp release_gen). The RX_VK_ASYNC job path is +// not touched. Unset, a release goes the way it went: way() is the old decision then (tests/test_release_queue.py). +// +// And a count, on by default: the releases that went the old way, their time in sync_drain and the replay's lock, the +// longest; a line every 600 releases when one of them waited 0.5 ms or more. +// +// The rule alone: no Vulkan, no game, no clock. +#pragma once +#include +#include +#include + +namespace release_queue { + +/// RX_VK_RELEASE_QUEUE: a number other than 0 on, 0 off; unset: the lab knob relq (on only in its phases) +inline bool on(const char* env, bool knob) { + if (env) return std::atoi(env) != 0; + return knob; +} + +/// Where a release without the replay thread goes. Now: sync_drain(1) and the replay's lock on the caller's thread, as +/// before; Held: PR 430's list, while a queue is stopped (hold: RX_VK_RELEASE_DRAIN unset; sync_waits: the event ordering +/// on); Queued: the switch's list, whatever is stopped +enum class Way { Now, Held, Queued }; +inline Way way(bool queue_on, bool hold, bool sync_waits, bool blocked) { + if (queue_on) return Way::Queued; + return hold && sync_waits && blocked ? Way::Held : Way::Now; +} +/// The old decision, for the test: held or not +inline bool held_before(bool hold, bool sync_waits, bool blocked) { return hold && sync_waits && blocked; } + +/// Whether the queued releases are applied at a submission's replay start or a present: something queued, no queue stopped +inline bool apply_now(size_t queued, int blocked) { return queued != 0 && blocked == 0; } + +/// The releases that went the old way: how many, how many waited kWaitNs or more, their time, the longest +struct Waits { + static constexpr int64_t kWaitNs = 500000; // (0.5 ms: a replay met, not the release's own work) + static constexpr int64_t kEvery = 600; + int64_t n = 0, waited = 0, ns = 0, max_ns = 0, queued = 0; + void note(int64_t took_ns) { + n++, ns += took_ns; + if (took_ns >= kWaitNs) waited++; + if (took_ns > max_ns) max_ns = took_ns; + } + void note_queued() { queued++; } + /// Every kEvery releases (either way) the counts start again; a line when one of them waited, or went to the switch's list + bool due() const { return n + queued >= kEvery; } + bool worth_a_line() const { return waited > 0 || queued > 0; } + void reset() { *this = Waits{}; } +}; + +} // namespace release_queue diff --git a/gpu/vk/replay_pin.h b/gpu/vk/replay_pin.h new file mode 100644 index 0000000..b68dbdd --- /dev/null +++ b/gpu/vk/replay_pin.h @@ -0,0 +1,77 @@ +// (Triforce, 11-22, the performance audit's THREAD-09, after Source A's) Where a thread may run while it replays, on a processor with +// cores of more than one kind. +// +// A thread that replays is pinned for that time to the core the host keeps for the replay (vk_replay.cpp ReplayPin, +// hos_reserved_cpus). That core is kept from the game's threads, not from other programs': on a 6+8-core hybrid processor Source A +// saw Windows park one of its two logical processors while another program's thread at priority 15 held the other 30 to 40 ms at a +// time, and the pinned replay, raised only to the highest of the normal levels, waited for it ready to run: 14 to 37 frames over +// 40 ms in a 40 s run. So on a processor of two kinds of core the pin is soft: the thread may run on the reserved core and on the +// fastest-kind cores among its own processors, its ideal processor is the reserved core's last logical processor, and the core it +// is on when the pin starts is left out for that pin, so that it moves at once, to the reserved core when that is free. Widening +// alone would leave it where it is: that is the "only preferring the core" that gained nothing. After the replay it has the +// affinity and the ideal processor it had. +// +// A processor of one kind of core keeps the strict pin: there the pin itself is what was measured to help, and nothing changes. +// RX_VK_REPLAY_PIN_SOFT unset or 1: soft where the cores are of more than one kind; 0: strict everywhere, as before; +// 2: soft everywhere (the lab). +// Plain rules over masks, no system calls: tests/test_replay_pin_soft.py runs them over made-up processors. +#pragma once + +#include +#include + +namespace mkwvk::replay_pin { + +constexpr int kStrict = 0, kHybrid = 1, kAlways = 2; + +/// RX_VK_REPLAY_PIN_SOFT's value: unset (or empty, or 1) kHybrid, 0 (or anything that is no number above 0) kStrict, 2 and more kAlways +inline int mode(const char* env) { + if (!env || !*env) return kHybrid; + const int v = std::atoi(env); + return v <= 0 ? kStrict : v == 1 ? kHybrid : kAlways; +} + +struct Kinds { + uint64_t fastest = 0; // (the logical processors of the cores with the highest efficiency class) + bool hybrid = false; // (some core is of a lower class) +}; +/// The fastest kind of core among n cores (each core's logical processors and its EfficiencyClass, higher the faster, as Windows +/// lists them; a core with no processors is not one). hybrid only when a listed core is not of that kind +inline Kinds kinds(const uint64_t* masks, const unsigned char* classes, int n) { + Kinds k; + int best = -1; + uint64_t all = 0; + for (int i = 0; i < n; i++) { + if (!masks[i]) continue; + all |= masks[i]; + if ((int)classes[i] > best) best = classes[i], k.fastest = masks[i]; + else if ((int)classes[i] == best) k.fastest |= masks[i]; + } + k.hybrid = k.fastest != all; + return k; +} + +/// Whether the pin is soft: by the switch and the processor's kinds +inline bool soft(int mode, bool hybrid) { + return mode >= kAlways || (mode == kHybrid && hybrid); +} + +/// The soft pin's processors: the reserved core, and the fastest-kind ones among the thread's own (own: its affinity before the +/// pin, which keeps a thread held to one die, or to anything else, inside what it was held to). Never fewer than the reserved core +inline uint64_t wide(uint64_t reserved, uint64_t fastest, uint64_t own) { + return reserved | (fastest & own); +} + +/// The same without the core the thread is on now (now: that core's logical processors, 0 when not known), so that the thread +/// moves at once. Left whole when it is on the reserved core already or when nothing else would be left +inline uint64_t moved(uint64_t wide, uint64_t reserved, uint64_t now) { + if (!now || (now & reserved) || !(wide & ~now)) return wide; + return wide & ~now; +} + +/// The ideal processor of a replaying thread: the reserved core's last logical processor (-1: no core reserved) +inline int ideal(uint64_t reserved) { + return reserved ? 63 - __builtin_clzll(reserved) : -1; +} + +} // namespace mkwvk::replay_pin diff --git a/gpu/vk/replay_threads.h b/gpu/vk/replay_threads.h index 04cde74..9f807a7 100644 --- a/gpu/vk/replay_threads.h +++ b/gpu/vk/replay_threads.h @@ -25,7 +25,8 @@ // they record only those trial frames. (Triforce, 11-15) And its rule (rule below): where the replay is plainly the frame's // limit they go on without a trial, and where they cannot pay no trial is made. (Triforce, 11-16) And once a way has held, // trials come ever rarer (up to some 14,400 presents apart) and shorter, and early only when the scene clearly changes (the -// back-off, at Tuner). +// back-off, at Tuner). (Triforce, 11-22) And both judge settled scenes only: the rule no window of a load or an arrival, and a +// trial the scene changed under decides nothing (settled, at Tuner). // // The parallel replay was audited for this by reading (nothing could be run). What was found and is fixed, each with a // check in tests/test_parallel_fixes.py: @@ -236,6 +237,18 @@ inline Verdict rule(const Frame& m, int threads) { /// trial whose first and last parts (the same way) moved so had the scene change under it: the windows after it are the /// new scene's reference, so the back-off starts again there too, the next trial kEvery on. /// A change of way, by a trial or by the rule, starts the back-off again. tests/test_replay_threads_tuner.py "backoff" +/// (Triforce, 11-22) Settled (settled; vk_replay.cpp par_settled_on, RX_VK_TUNER_SETTLED=0: off, as before), after +/// Kaefergeneral's #dev-fixes post: on a 5900X with an RTX 3090, where the helpers lost in every settled scene (1.2 to 4.5 ms a +/// frame), Automatic was 6 to 10% slower than Off. The rule turned them on in the window of an area's load (long frames, the +/// replay 71 to 82% of them, nearly every draw cuttable), not in the settled Market (the replay 62%), and the first trial after +/// the arrival straddled the load ("parallel 4.76 ms a frame, the other way 24.10, parallel again 17.25: as it was"): the +/// scene's change was seen (moved) but the verdict stood, for kEvery presents or more. So: +/// - the rule judges a window only if the one before it, on the replay thread alone, describes the same scene (scene_moved, +/// the back-off's test): a load's window and the first after it get no verdict, neither "on" nor "no trial". After a trial +/// or a change of way the first window has none before it, so the rule's "on" comes a window later than it did; +/// - a trial whose first and last parts moved so decides nothing, either way (not better, no recheck), and the next one comes +/// kWindow after a window of the way in use like the one before it (settle_look), not kEvery later, and never while the rule +/// says no trial. kHold and the thresholds as they were. tests/test_replay_threads_tuner.py "settled" class Tuner { public: static constexpr int kEvery = 900, kTake = 80, kSettle = 24, kWindow = 120, kHold = 4 * kEvery; @@ -243,6 +256,7 @@ public: static constexpr double kClear = 1.10, kSceneDraws = 0.25, kSceneDrawsMin = 200, kScenePart = 0.30, kScenePartMs = 1.0; bool use_rule = true; bool backoff = true; // (the back-off above; false: every step as before it) + bool settled = true; // (the rule and the trials only on a settled scene, above; false: as before) int threads = 4; // (the threads that record segments: rule()'s) struct Step { bool flip = false; // change the way in use @@ -262,6 +276,9 @@ public: bool moved = false; // (the scene changed under the trial that ended: the back-off starts again) bool sure = false; // (the trial kept the way, the other way longer than both its parts: the back-off grows) int64_t next = 0; // (the present the next trial may begin at: set at a trial's end and at a scene change) + // (settled's) + bool unsettled = false; // (a window the rule did not judge, or a trial that decided nothing: the scene moved) + bool settle = false; // (the scene settled after such a trial: the next trial at next) }; /// f: the frame just presented; on: the helper threads in use for it; order_only: both ways are the replay thread alone Step step(const Frame& f, bool on, bool order_only) { @@ -272,31 +289,44 @@ public: if (f.frame <= 60.0) win_.add(f); if (++win_frames_ >= kWindow) { if (win_.n >= kWindow * 3 / 4 && !order_only) { - s.ruled = true, s.window = win_.mean(), s.verdict = rule(s.window, threads); - const bool turn_on = s.verdict.kind == Verdict::kOn && frames_ >= hold_until_; - s.rule_new = s.verdict.kind != last_ || turn_on; - last_ = s.verdict.kind; - if (turn_on) { - s.flip = true; - next_trial_ = frames_ + kEvery; - if (backoff) held_ = 0, fresh_ = true, scene_restart(); - } else if (s.verdict.kind == Verdict::kOff) { - next_trial_ = std::max(next_trial_, frames_ + kEvery); - no_trial_until_ = frames_ + kEvery; + // (settled: a window is judged only if the one before it, on the replay thread alone too, describes + // the same scene) + s.window = win_.mean(); + const bool same = !settled || (win_prev_set_ && !scene_moved(win_prev_, s.window)); + win_prev_ = s.window, win_prev_set_ = true; + if (!same) { + s.unsettled = true; + } else { + s.ruled = true, s.verdict = rule(s.window, threads); + const bool turn_on = s.verdict.kind == Verdict::kOn && frames_ >= hold_until_; + s.rule_new = s.verdict.kind != last_ || turn_on; + last_ = s.verdict.kind; + if (turn_on) { + s.flip = true; + next_trial_ = frames_ + kEvery; + waiting_ = false, win_prev_set_ = false; // (the way changed: a trial kEvery on, as after a change) + if (backoff) held_ = 0, fresh_ = true, scene_restart(); + } else if (s.verdict.kind == Verdict::kOff) { + next_trial_ = std::max(next_trial_, frames_ + kEvery); + no_trial_until_ = frames_ + kEvery; + } } + } else { + win_prev_set_ = false; // (a load, or DLSS / FSR kept in order: the next window has none before it) } win_ = Part{}, win_frames_ = 0; if (s.flip) return s; } } else { - win_ = Part{}, win_frames_ = 0; + win_ = Part{}, win_frames_ = 0, win_prev_set_ = false; } - if (backoff) scene_look(f, s); + if (waiting_) settle_look(f, s); + else if (backoff) scene_look(f, s); if (frames_ < next_trial_) return s; if (order_only) return s; phase_ = 1, n_ = 0; short_ = backoff && held_ >= 2; - win_ = Part{}, win_frames_ = 0; // (the rule's next window begins after the trial) + win_ = Part{}, win_frames_ = 0, win_prev_set_ = false; // (the rule's next window begins after the trial) sum_[0] = sum_[1] = sum_[2] = 0, cnt_[0] = cnt_[1] = cnt_[2] = 0; part_[0] = part_[1] = part_[2] = Part{}; return s; @@ -318,6 +348,11 @@ public: s.a = cnt_[0] ? sum_[0] / cnt_[0] : 0, s.b = cnt_[1] ? sum_[1] / cnt_[1] : 0, s.c = cnt_[2] ? sum_[2] / cnt_[2] : 0; s.better = s.ok && s.b < std::min(s.a, s.c) * 0.96; s.short_trial = short_; + // (settled: a trial whose first and last parts, the same way, describe two scenes measured the change of scene, not + // the way: it decides nothing, either way, and the next one waits for the scene to settle) + const bool moved = (backoff || settled) && cnt_[0] && cnt_[2] && scene_moved(part_[0].mean(), part_[2].mean()); + s.moved = moved; + if (settled && moved) s.better = false, s.unsettled = true; if (s.better && short_) s.better = false, s.recheck = true; // (a full trial decides: at once, below) if (s.better) { s.flip = true, changes_++; @@ -334,7 +369,6 @@ public: // put off ever longer; such a trial costs little, its two ways' frames being about as long. The first trial // after a change of scene or way that is too close to tell: kEvery, a second look soon. The trials' means // drift with the scene, and right after a change the first is the one most often wrong) - s.moved = cnt_[0] && cnt_[2] && scene_moved(part_[0].mean(), part_[2].mean()); const double top = std::max(s.a, s.c); s.sure = s.ok && !s.better && !s.recheck && !s.moved && s.b > top; const bool again = !s.sure && fresh_; @@ -345,6 +379,10 @@ public: last_end_ = frames_; scene_restart(); } + if (s.unsettled) { // (the next trial kWindow after a window like the one before it: settle_look) + waiting_ = true, next_trial_ = INT64_MAX; + settle_win_ = Part{}, settle_frames_ = 0, settle_prev_set_ = false; + } s.next = next_trial_; s.trial = true, s.parts = part_; } @@ -367,6 +405,22 @@ public: private: void scene_restart() { scene_win_ = Part{}, scene_frames_ = 0, ref_set_ = false, strikes_ = 0; } + /// (settled: after a trial the scene moved under, the way's windows each against the one before; the first like it, and + /// the next trial kWindow on, never while the rule says no trial. A load breaks the run) + void settle_look(const Frame& f, Step& s) { + if (f.frame <= 60.0) settle_win_.add(f); + if (++settle_frames_ < kWindow) return; + const bool counted = settle_win_.n >= kWindow * 3 / 4; + const Frame m = settle_win_.mean(); + settle_win_ = Part{}, settle_frames_ = 0; + const bool same = counted && settle_prev_set_ && !scene_moved(settle_prev_, m); + settle_prev_ = m, settle_prev_set_ = counted; + if (!same) return; + waiting_ = false, s.settle = true; + next_trial_ = std::max(frames_ + kWindow, no_trial_until_); + s.next = next_trial_; + if (backoff) scene_restart(); // (the settled scene is the back-off's reference) + } /// (between trials: the scene's windows against the reference, the next trial brought forward when two in a row moved) void scene_look(const Frame& f, Step& s) { if (f.frame <= 60.0) scene_win_.add(f); @@ -406,6 +460,11 @@ private: int scene_frames_ = 0, strikes_ = 0; bool ref_set_ = false; Frame ref_; + // (settled's: the rule's window before, and after a trial the scene moved under, the wait for it to settle and its windows) + Frame win_prev_, settle_prev_; + bool win_prev_set_ = false, waiting_ = false, settle_prev_set_ = false; + Part settle_win_; + int settle_frames_ = 0; }; } // namespace replay_threads diff --git a/gpu/vk/retired_give_back.h b/gpu/vk/retired_give_back.h new file mode 100644 index 0000000..5ba5810 --- /dev/null +++ b/gpu/vk/retired_give_back.h @@ -0,0 +1,85 @@ +// (Triforce, 11-22, the performance audit's MEM-21, after Source A's) When a retired pool's import is given back. +// +// A pool remade larger is retired: never bound again, and once the GPU is past it its import is pinned RAM nothing uses. +// vk_memory.cpp retired_imports_give_back gives such imports back at once on Linux and with NVIDIA's driver on Windows, and +// keeps them for the session with AMD's and Intel's drivers on Windows: given back at once there, an Intel Arc B390, an +// Ally X and an RX 6600 lost the device right after. It never gives back the import of a pool whose mirror was refused +// (RX_VK_NO_MIRROR_POOLS, a pool over 1 GB, VRAM refused), on any OS. +// +// Source A gives them back on Windows AMD and Intel after a wait instead: 30 s after the pool retired, or 2 s while the +// system heap is past 85% of its budget; and the refused mirrors' imports too. Here both are OFF unless asked: +// RX_VK_KEEP_RETIRED=wait the wait, on any driver (RX_VK_RETIRE_SECS, 30; RX_VK_RETIRE_MIN_SECS, 2) +// RX_VK_RETIRED_REFUSED=1 the refused mirrors' imports too, by the mode in force (with `wait`: on unless =0) +// Unset, and every value RX_VK_KEEP_RETIRED took before, decide exactly as before: tests/test_retired_give_back.py compares +// want() with the old condition on random inputs. NVIDIA's driver on Windows keeps its behaviour. +// +// The rule alone, no Vulkan, no clock: the caller passes the times in milliseconds. +#pragma once +#include +#include +#include + +namespace retired_give_back { + +enum class Mode { AtOnce, Wait, Keep }; + +/// RX_VK_KEEP_RETIRED: "wait" the wait; any other value as it was read before (a value beginning with 0 gives back at once, +/// anything else keeps, the empty text too); unset: kept on Windows unless the card is NVIDIA's, given back at once elsewhere +inline Mode mode(const char* env, bool windows, bool nvidia) { + if (env) { + if (std::strcmp(env, "wait") == 0) return Mode::Wait; + return env[0] != '0' ? Mode::Keep : Mode::AtOnce; + } + return windows && !nvidia ? Mode::Keep : Mode::AtOnce; +} +/// The old reading, for the test: whether retired imports were kept +inline bool keep_before(const char* env, bool windows, bool nvidia) { + if (windows) return env ? env[0] != '0' : !nvidia; + return env && env[0] != '0'; +} + +/// RX_VK_RETIRED_REFUSED: a number other than 0 on, 0 off; unset: on with the wait (Source A's rule whole), off otherwise +inline bool refused_too(const char* env, Mode m) { + if (env) return std::atoi(env) != 0; + return m == Mode::Wait; +} + +/// RX_VK_RETIRE_SECS, RX_VK_RETIRE_MIN_SECS: whole seconds, 0 or more; unset or not a number of 0 or more: the default +inline int64_t secs(const char* env, int64_t fallback) { + if (!env || !*env) return fallback; + char* end = nullptr; + const long long v = std::strtoll(env, &end, 10); + return end && *end == 0 && v >= 0 ? (int64_t)v : fallback; +} + +/// Whether the system heap is short: used past 85% of its budget (no budget known: not short) +inline bool pressure(uint64_t used, uint64_t budget) { + return budget && used > budget / 100 * 85; +} + +/// The wait over: now_ms - retired_ms at least min_s seconds while the heap is short, wait_s seconds otherwise +inline bool due(int64_t now_ms, int64_t retired_ms, int64_t wait_s, int64_t min_s, bool short_heap) { + return now_ms - retired_ms >= (short_heap ? min_s : wait_s) * 1000; +} + +/// Whether a retired pool's import goes back now. mirror_none, mirror_refused: its mirror's state; inherit: the pool that took +/// it in still copies from its mirror; gpu_past: the GPU is past its retirement and its last use; is_due: due() (the wait only) +inline bool want(Mode m, bool refused, bool mirror_none, bool mirror_refused, bool inherit, bool gpu_past, bool is_due) { + if (m == Mode::Keep) return false; + if (!mirror_none && !(refused && mirror_refused)) return false; + if (inherit || !gpu_past) return false; + return m == Mode::AtOnce || is_due; +} +/// The old condition, for the test +inline bool want_before(bool keep, bool mirror_none, bool inherit, bool gpu_past) { + return !keep && mirror_none && !inherit && gpu_past; +} + +/// The system heap's use as the pressure give-back (imports_give_back) should read it with the wait on: less the imports given +/// back and not freed yet (dead_imports, freed once the GPU is past them), which the driver still counts. One overshoot, +/// one valve: without this a burst of retired imports given back made it give back live ones as well +inline uint64_t settled(uint64_t used, uint64_t dead_bytes) { + return used > dead_bytes ? used - dead_bytes : 0; +} + +} // namespace retired_give_back diff --git a/gpu/vk/settle_gate.h b/gpu/vk/settle_gate.h new file mode 100644 index 0000000..de9e6a7 --- /dev/null +++ b/gpu/vk/settle_gate.h @@ -0,0 +1,58 @@ +// (Triforce, 11-22, the performance audit's SYNC-29, after Source A's) A CPU read looks at the stopped queues only after a +// new CPU signal. A read that settles (vk_replay.cpp sync_settle: the stopped queues whose waits are met go on, none forced) +// takes sync_order_mu whenever a queue is stopped, and that lock is held through whole replays and presents: the readback's +// look (RX_VK_RB_SETTLE, off) and the two looks invalidate makes for a range a texture may be in. +// +// What can make a stopped queue's wait met: +// a signal in the stream replayed by a thread that holds sync_order_mu, and every such holder runs schedule() and ends its +// burst (burst_end: finish_recording) before it lets the lock go: it looked itself; +// a signal of the CPU's event_signal_queued: queued, then counted (signals); +// a plain store into the memory of an event nothing has set yet (the wait reads its value from guest memory, +// event_value_in): not seen by anything, so such a wait keeps every look (wait_in_memory). +// So a look is needed only while a queue is stopped and the CPU has signalled since the last look, or a stopped queue waits +// on an event read from memory. A look notes the signals' count at its start (burst_begin) and publishes it once its recording +// is finished (burst_end, after finish_recording): a reader that sees signals == settled finds the work those signals let on +// recorded. Publishing the count as it is at the look's END would cover a signal that came after the look's schedule() and +// was never acted on; counting a signal BEFORE it is queued would let a look publish it without having seen it. The test runs +// both mistakes and finds them. +// +// RX_VK_SETTLE_GATE: unset or 0, no gate (every look as before); 1 the readback's look (the one RX_VK_RB_SETTLE turns on); +// 2 the texture sites' looks too. Those two order the exposure meter's readback: the trial that skipped them outright made +// the Market flicker at once, and Source A left them ungated, so 2 is new ground and needs the lab's flicker count. The lab +// knobs settlegate (1) and texgate (2) in their phases. tests/test_settle_gate.py +#pragma once +#include + +namespace mkwvk::settle_gate { + +/// The sites that settle +enum Site { kReadback = 0, kTexture = 1, kSites = 2 }; + +/// RX_VK_SETTLE_GATE's text: "1" and "2"; unset, empty, "0" and anything else: 0, no gate +inline int level(const char* text) { + if (!text || !text[0] || text[1]) return 0; + return text[0] == '1' ? 1 : text[0] == '2' ? 2 : 0; +} +/// The level in force: the variable's, raised by the lab knobs in their phases +inline int level_now(int env, bool knob_settlegate, bool knob_texgate) { + int l = env; + if (knob_settlegate && l < 1) l = 1; + if (knob_texgate && l < 2) l = 2; + return l; +} +/// Whether a site's look goes through the gate at this level +inline bool gated(int level, int site) { return site == kReadback ? level >= 1 : level >= 2; } +/// Whether a gated site looks: a queue stopped, and a CPU signal since the last look or a stopped queue waiting on an event +/// whose value is read from guest memory +inline bool look(bool blocked, uint64_t signals, uint64_t settled, bool wait_in_memory) { + return blocked && (signals != settled || wait_in_memory); +} +/// The whole decision of a site: 0 nothing to do (no queue stopped: as before), 1 look, 2 skipped by the gate +enum What { kNothing = 0, kLook = 1, kSkip = 2 }; +inline int decide(int level, int site, bool blocked, uint64_t signals, uint64_t settled, bool wait_in_memory) { + if (!blocked) return kNothing; + if (gated(level, site) && !look(true, signals, settled, wait_in_memory)) return kSkip; + return kLook; +} + +} // namespace mkwvk::settle_gate diff --git a/gpu/vk/shaders/sim_gpu_burn.comp b/gpu/vk/shaders/sim_gpu_burn.comp new file mode 100644 index 0000000..adb1459 --- /dev/null +++ b/gpu/vk/shaders/sim_gpu_burn.comp @@ -0,0 +1,16 @@ +#version 460 +// (Triforce, 11-22, the performance audit's OS-49, after Source B's) RX_VK_SIM_GPU_SLOW's burn (gpu/vk/sim_gpu.h): one +// invocation runs a chain of dependent multiply-adds `loops` times, so that a dispatch takes a time the loop count sets; the +// result goes to a word nothing reads, so that the chain is not left out. +// glslangValidator -V --target-env vulkan1.3 --vn SIM_GPU_BURN_COMP_SPV -o sim_gpu_burn_comp_spv.h sim_gpu_burn.comp +layout(local_size_x = 1) in; +layout(push_constant) uniform P { + uint loops; + float seed; +} pc; +layout(std430, set = 0, binding = 0) buffer Sink { float v; } sink; +void main() { + float x = pc.seed; + for (uint i = 0u; i < pc.loops; i++) x = x * 0.99999994 + 0.5; + sink.v = x; +} diff --git a/gpu/vk/shaders/sim_gpu_burn_comp_spv.h b/gpu/vk/shaders/sim_gpu_burn_comp_spv.h new file mode 100644 index 0000000..ffb0b3d --- /dev/null +++ b/gpu/vk/shaders/sim_gpu_burn_comp_spv.h @@ -0,0 +1,43 @@ + // 1116.6.0 + #pragma once +const uint32_t SIM_GPU_BURN_COMP_SPV[] = { + 0x07230203,0x00010600,0x0008000b,0x00000031,0x00000000,0x00020011,0x00000001,0x0006000b, + 0x00000001,0x4c534c47,0x6474732e,0x3035342e,0x00000000,0x0003000e,0x00000000,0x00000001, + 0x0007000f,0x00000005,0x00000004,0x6e69616d,0x00000000,0x0000000c,0x0000002a,0x00060010, + 0x00000004,0x00000011,0x00000001,0x00000001,0x00000001,0x00030003,0x00000002,0x000001cc, + 0x00040005,0x00000004,0x6e69616d,0x00000000,0x00030005,0x00000008,0x00000078,0x00030005, + 0x0000000a,0x00000050,0x00050006,0x0000000a,0x00000000,0x706f6f6c,0x00000073,0x00050006, + 0x0000000a,0x00000001,0x64656573,0x00000000,0x00030005,0x0000000c,0x00006370,0x00030005, + 0x00000013,0x00000069,0x00040005,0x00000028,0x6b6e6953,0x00000000,0x00040006,0x00000028, + 0x00000000,0x00000076,0x00040005,0x0000002a,0x6b6e6973,0x00000000,0x00030047,0x0000000a, + 0x00000002,0x00050048,0x0000000a,0x00000000,0x00000023,0x00000000,0x00050048,0x0000000a, + 0x00000001,0x00000023,0x00000004,0x00030047,0x00000028,0x00000002,0x00050048,0x00000028, + 0x00000000,0x00000023,0x00000000,0x00040047,0x0000002a,0x00000021,0x00000000,0x00040047, + 0x0000002a,0x00000022,0x00000000,0x00020013,0x00000002,0x00030021,0x00000003,0x00000002, + 0x00030016,0x00000006,0x00000020,0x00040020,0x00000007,0x00000007,0x00000006,0x00040015, + 0x00000009,0x00000020,0x00000000,0x0004001e,0x0000000a,0x00000009,0x00000006,0x00040020, + 0x0000000b,0x00000009,0x0000000a,0x0004003b,0x0000000b,0x0000000c,0x00000009,0x00040015, + 0x0000000d,0x00000020,0x00000001,0x0004002b,0x0000000d,0x0000000e,0x00000001,0x00040020, + 0x0000000f,0x00000009,0x00000006,0x00040020,0x00000012,0x00000007,0x00000009,0x0004002b, + 0x00000009,0x00000014,0x00000000,0x0004002b,0x0000000d,0x0000001b,0x00000000,0x00040020, + 0x0000001c,0x00000009,0x00000009,0x00020014,0x0000001f,0x0004002b,0x00000006,0x00000022, + 0x3f7fffff,0x0004002b,0x00000006,0x00000024,0x3f000000,0x0003001e,0x00000028,0x00000006, + 0x00040020,0x00000029,0x0000000c,0x00000028,0x0004003b,0x00000029,0x0000002a,0x0000000c, + 0x00040020,0x0000002c,0x0000000c,0x00000006,0x00040017,0x0000002e,0x00000009,0x00000003, + 0x0004002b,0x00000009,0x0000002f,0x00000001,0x0006002c,0x0000002e,0x00000030,0x0000002f, + 0x0000002f,0x0000002f,0x00050036,0x00000002,0x00000004,0x00000000,0x00000003,0x000200f8, + 0x00000005,0x0004003b,0x00000007,0x00000008,0x00000007,0x0004003b,0x00000012,0x00000013, + 0x00000007,0x00050041,0x0000000f,0x00000010,0x0000000c,0x0000000e,0x0004003d,0x00000006, + 0x00000011,0x00000010,0x0003003e,0x00000008,0x00000011,0x0003003e,0x00000013,0x00000014, + 0x000200f9,0x00000015,0x000200f8,0x00000015,0x000400f6,0x00000017,0x00000018,0x00000000, + 0x000200f9,0x00000019,0x000200f8,0x00000019,0x0004003d,0x00000009,0x0000001a,0x00000013, + 0x00050041,0x0000001c,0x0000001d,0x0000000c,0x0000001b,0x0004003d,0x00000009,0x0000001e, + 0x0000001d,0x000500b0,0x0000001f,0x00000020,0x0000001a,0x0000001e,0x000400fa,0x00000020, + 0x00000016,0x00000017,0x000200f8,0x00000016,0x0004003d,0x00000006,0x00000021,0x00000008, + 0x00050085,0x00000006,0x00000023,0x00000021,0x00000022,0x00050081,0x00000006,0x00000025, + 0x00000023,0x00000024,0x0003003e,0x00000008,0x00000025,0x000200f9,0x00000018,0x000200f8, + 0x00000018,0x0004003d,0x00000009,0x00000026,0x00000013,0x00050080,0x00000009,0x00000027, + 0x00000026,0x0000000e,0x0003003e,0x00000013,0x00000027,0x000200f9,0x00000015,0x000200f8, + 0x00000017,0x0004003d,0x00000006,0x0000002b,0x00000008,0x00050041,0x0000002c,0x0000002d, + 0x0000002a,0x0000001b,0x0003003e,0x0000002d,0x0000002b,0x000100fd,0x00010038 +}; diff --git a/gpu/vk/shadow_prev.h b/gpu/vk/shadow_prev.h index 44b7e1e..07637d1 100644 --- a/gpu/vk/shadow_prev.h +++ b/gpu/vk/shadow_prev.h @@ -53,6 +53,8 @@ #include #include +#include "new_wait.h" // ((Triforce, 11-22, OS-04) a nothrow new that must answer at once: NoWait) + namespace shadow_prev { inline constexpr uint64_t kPage = 4096; @@ -217,7 +219,10 @@ inline bool keep_old(std::unique_ptr& b, uint64_t& hash, Vers& v, int v.n = (uint8_t)(maxn - 1); } else { if (alive().load(std::memory_order_relaxed) >= cap) return false; - fresh.reset(new (std::nothrow) uint8_t[kPage]); + { + mkwvk::new_wait::NoWait at_once; // ((Triforce, OS-04) no page: none kept; not a wait at the commit limit) + fresh.reset(new (std::nothrow) uint8_t[kPage]); + } if (!fresh) return false; alive().fetch_add(1, std::memory_order_relaxed); } diff --git a/gpu/vk/sim_gpu.h b/gpu/vk/sim_gpu.h new file mode 100644 index 0000000..96f1462 --- /dev/null +++ b/gpu/vk/sim_gpu.h @@ -0,0 +1,56 @@ +// (Triforce, 11-22, the performance audit's OS-49, after Source B's) A lab switch: a slower GPU, so that a lab PC's renderer +// meets the GPU-bound frames a weaker card's player has (the write-back waits, the capture's late pages, the recording slots, +// the pacing), on the same 5090. RX_VK_SIM_GPU_SLOW= (over 1, at most 10; 2.5 is read): for each ms of work the +// recordings measure, (factor - 1) ms more, a one-invocation compute loop (shaders/sim_gpu_burn.comp) at the end of the +// replay's own recordings, between full barriers, inside the span the GPU time is measured over, so that the frame's GPU time +// and Resolution Auto see the slower card (vk_replay.cpp sim_burn). What a burn took is measured by a third timestamp between +// the recording's work and the burn, and the difference is carried: the debt below. It repeats the fast card's relative +// answer only: no bandwidth, no other driver, no upscaler's cost. Unset: nothing of it runs, two timestamps a recording. +// +// The rules alone, no Vulkan: tests/test_sim_cores.py compiles them. +#pragma once +#include +#include +#include + +namespace sim_gpu { + +inline constexpr int64_t kLeastNs = 1'000'000; // (a burn is planned for 1 ms at least: less owed waits) +inline constexpr int64_t kMostNs = 16'000'000; // (and 16 ms at most: the rest goes to the next recordings) +inline constexpr int64_t kDebtLeast = -50'000'000; // (the time owed is held between these: a burn that ran long is not +inline constexpr int64_t kDebtMost = 100'000'000; // carried for seconds, nor a backlog after a hitch) +inline constexpr double kFirstNsPerLoop = 4.0; // (a loop's time before the first measure) + +/// RX_VK_SIM_GPU_SLOW: the factor, over 1 and at most 10; 0 (off) for anything else +inline double factor_of(const char* e) { + if (!e || !*e) return 0; + char* end = nullptr; + const double f = std::strtod(e, &end); + if (end == e || *end || !(f > 1.0) || f > 10.0) return 0; + return f; +} +/// The time owed after a recording measured `work_ns` of work and `added_ns` of burn, before the bounds +inline double owed(int64_t debt, int64_t work_ns, int64_t added_ns, double factor) { + return (double)debt + (double)work_ns * (factor - 1.0) - (double)added_ns; +} +/// And within them +inline int64_t held(double owed_ns) { return (int64_t)std::clamp(owed_ns, (double)kDebtLeast, (double)kDebtMost); } +/// The time the next burn is planned for: what is owed, less the burns planned and not measured yet; 0 under kLeastNs +inline int64_t plan_ns(int64_t debt, int64_t in_flight) { + const int64_t o = debt - in_flight; + return o < kLeastNs ? 0 : std::min(o, kMostNs); +} +/// The loop count for a planned time, at the measured time a loop takes +inline uint32_t loops_for(int64_t ns, double ns_per_loop) { + if (ns <= 0) return 0; + const double l = (double)ns / (ns_per_loop > 0.01 ? ns_per_loop : 0.01); + return (uint32_t)std::clamp(l, 1.0, (double)(1u << 30)); +} +/// The time a loop takes after a burn of `loops` took `added_ns`: 0.8 of what it was, 0.2 of what this one measured +inline double calibrated(double ns_per_loop, int64_t added_ns, uint32_t loops) { + if (!loops || added_ns <= 0) return ns_per_loop; + const double m = (double)added_ns / loops; + return ns_per_loop > 0 ? 0.8 * ns_per_loop + 0.2 * m : m; +} + +} // namespace sim_gpu diff --git a/gpu/vk/slot_retire.h b/gpu/vk/slot_retire.h new file mode 100644 index 0000000..0686f6e --- /dev/null +++ b/gpu/vk/slot_retire.h @@ -0,0 +1,49 @@ +// (Triforce, 11-21, the performance audit's TEX-76, after Source B's) When a depth target that another size has replaced at its +// start is let go. +// +// The game draws each character's shadow map (D32, one level) at a fixed address, sized from the camera every frame. An +// image is kept by its storage, width and height, so each new size is a new texture, and the one before it has been drawn +// over. The sweep took such a texture only once it was unused for 120 presents (60 over a fixed budget): a pile of dead maps +// at every slot, in VRAM for two seconds each. Source B's rule, as it ended after its earlier ones retired targets the game +// drew into again (after 1 frame; colour targets; depth of the same size): a depth target of one level and layer at the +// same start and of the same format as a newer target of another width or height, the newer one written after its last +// use, goes once it is unused for kWait presents. Its VRAM peak in the night market fell from 5,972 to 5,482 MiB, and no +// retired image was wanted again within 16 frames (29 were with a wait of 1). Nothing is written back: the newer target's +// texels are in that storage. The image goes to the render targets' pool by its create info, as any dead target's. +// RX_VK_SLOT_RETIRE=: the wait, 16 when not set; 0: only after the sweep's own 120, as before. +// +// The rule alone, no Vulkan: tests/test_slot_retire.py compiles it. +#pragma once +#include +#include + +namespace slot_retire { + +inline constexpr uint64_t kWait = 16; + +/// The wait in presents: asked, RX_VK_SLOT_RETIRE's value; not set or empty kWait; 0 or less (and what is no number): 0, off +inline uint64_t wait(const char* asked) { + if (!asked || !*asked) return kWait; + const long v = atol(asked); + return v > 0 ? (uint64_t)v : 0; +} + +/// What the rule looks at of a texture +struct Target { + bool gpu_written, depth; // (the GPU drew it; its aspect is depth) + uint32_t levels, layers, deep; // (deep: its depth in texels, 1 for a 2D one) + int nvn_format; + uint32_t width, height; + uint64_t last_frame, last_written; // (the present it was last used in; the frame the GPU last wrote it in) +}; + +/// Whether old, unused for `unused` presents, is let go because newer took its place: both at the same start +inline bool retired_by(const Target& old, const Target& newer, uint64_t unused, uint64_t wait_presents) { + if (!wait_presents || unused < wait_presents) return false; + if (!old.gpu_written || !old.depth || old.levels != 1 || old.layers != 1 || old.deep > 1) return false; + if (newer.nvn_format != old.nvn_format) return false; + if (newer.width == old.width && newer.height == old.height) return false; // (the same size: the game may draw into it again) + return newer.last_written > old.last_frame; // (written after its last use: drawn over) +} + +} // namespace slot_retire diff --git a/gpu/vk/staging_keep.h b/gpu/vk/staging_keep.h new file mode 100644 index 0000000..acd3a3f --- /dev/null +++ b/gpu/vk/staging_keep.h @@ -0,0 +1,41 @@ +// How much of the idle staging arenas a present may keep (no game, no GPU: rules). tests/test_staging_burst.py +// +// (Triforce, 11-22, the performance audit's TEX-28, after Source A's.) The uploads stage their bytes in host-visible +// arenas of 64 MB (vk_texture.cpp staging()). An arena is taken again once the GPU is done with the submission that read +// it, which texture_frame learns a present later: frames that stage one after another need two arenas in turn. Each +// present gives the idle ones back beyond what is kept, 256 MB, and 64 MB on a card of 6.5 GB or less, where an arena may +// be VRAM behind a resizable BAR. With 64 MB kept, the arena that had its turn was idle at every second present of a load, +// was freed, and was made again the present after: Source A counted 16 arenas made, 225 ms of them on the thread that +// replays, in one arrival on a handheld (15.5 ms each), and 4 with this rule. +// +// The rule: an idle arena that something staged into less than RX_VK_STAGING_BURST_S seconds ago (2) is one of a burst's, +// and while such an arena is looked at the idle ones may add up to 256 MB instead of what is kept. Not while VRAM is over +// 90% of the allowance (the render targets' pool's own flag, pool_pressed): a full card keeps what it kept. Where 256 MB +// are kept anyway, every card over 6.5 GB, nothing changes. The arenas over 64 MB still go at once, and the 30 s rule +// (RX_VK_STAGING_IDLE_S) stays. RX_VK_STAGING_BURST_S=0: as before. +#pragma once +#include +#include +#include + +namespace staging_keep { + +inline constexpr uint64_t kBurstKeep = 256ull << 20; // (what the idle arenas of a burst may add up to) +inline constexpr int64_t kBurstS = 2; // (how long after its last staging an arena is one of a burst's) + +/// RX_VK_STAGING_BURST_S: unset or empty kBurstS; a number of seconds, at most an hour; 0, less or not a number: no rule +inline int64_t burst_s(const char* env) { + if (!env || !*env) return kBurstS; + const long long v = std::atoll(env); + return v > 0 ? (int64_t)std::min(v, 3600) : 0; +} + +/// Whether an arena last staged into `since_used_ms` ago is one of a burst's: `pressed` is VRAM over 90% of the allowance +constexpr bool recent(int64_t burst_s, bool pressed, int64_t since_used_ms) { + return burst_s > 0 && !pressed && since_used_ms >= 0 && since_used_ms < burst_s * 1000; +} + +/// What the arenas looked at so far and this one may add up to for this one to stay: `keep` is what a present keeps anyway +constexpr uint64_t allowance(uint64_t keep, bool recent) { return recent ? std::max(keep, kBurstKeep) : keep; } + +} // namespace staging_keep diff --git a/gpu/vk/stall_watch.h b/gpu/vk/stall_watch.h index 009a94c..b33d40c 100644 --- a/gpu/vk/stall_watch.h +++ b/gpu/vk/stall_watch.h @@ -80,6 +80,9 @@ struct Frame { double wb_unsent_ms = 0; int64_t wb_held = 0; // ((Triforce, wp20) how many went on after the writer's recording was done on the GPU, and double wb_held_ms = 0; // that part of them: the submissions' timeline held by work stopped at a wait, wb_rec_wait.h) + // ((Triforce, 11-22, the performance audit's SYNC-122) that part by caller, ms: a capture's take, a read of a page put off, + // a binding of one, a readback's fallback (wb_rec_wait.h Caller; the readbacks' is in none of the counts above) + double wb_held_by[4] = {0, 0, 0, 0}; double pipe_wait = 0; // (waits for shaders and pipelines) int64_t pipe_waits = 0, left_out = 0; // (how many; draws left out while theirs were built) int64_t pipes_first = 0, pipes_ahead = 0, pipes_order = 0; // ((Triforce, wp18) vk_draw.cpp graphics_pipeline: the pipelines the @@ -186,6 +189,10 @@ inline int stall_line(const Frame& f, char* out, size_t cap) { (long long)f.wb_held, f.wb_held_ms); put("%s", ")"); } + // ((Triforce, 11-22, SYNC-122) the held part by caller, when there is any: the reads' side of RX_VK_WB_WAIT_REC) + if (f.wb_held_by[0] + f.wb_held_by[1] + f.wb_held_by[2] + f.wb_held_by[3] > 0) + put("; the timeline held after a writer's recording was done, by caller: takes %.1f, reads %.1f, binds %.1f, readbacks %.1f ms", + f.wb_held_by[0], f.wb_held_by[1], f.wb_held_by[2], f.wb_held_by[3]); if (f.pipe_waits || f.left_out) put("; %lld waits for shaders or pipelines (%.1f ms), %lld draws left out", (long long)f.pipe_waits, f.pipe_wait, (long long)f.left_out); diff --git a/gpu/vk/stub_same_size.h b/gpu/vk/stub_same_size.h new file mode 100644 index 0000000..a0e8050 --- /dev/null +++ b/gpu/vk/stub_same_size.h @@ -0,0 +1,53 @@ +// A gameplay upscale network that cannot run: its input drawn into its output, also where the two are one size (no +// game, no GPU: the rule; vk_draw.cpp upscale_stub). +// +// (Triforce, 11-22, the performance audit's GPUWORK-21, after Source B's and Source A's.) The game draws its 3D through +// a small network every frame: fd029a317a9a4b94 from 3/4 size, 9dc98871eb7fc101 at full size. upscale_stub stands in +// for a network that was left out or did not translate (RX_VK_NO_F16, a card or driver that cannot build it), by +// drawing the network's input into its output. It declined a dispatch whose output is no larger than its input, as not +// an upscale's last pass, so at Render resolution 100% a 9dc98871 that could not run was skipped whole: nothing wrote +// the picture, a black 3D under the HUD. Source B turns its bypass on by itself where its network cannot be built, and +// Source A lets that pass through for the gameplay networks. +// +// So, for the two gameplay networks only: +// - an output as large as its input is accepted too (any other program is still declined: a pass of that shape that +// is no gameplay network is not an upscale's last one); +// - both take the typed upscale first, which reads the scene's shared-exponent picture as what it is and undoes the +// network's square-root encoding (shaders/taau_upscale.comp). It was fd029a317a9a4b94's alone, so a 9dc98871 that +// upscaled went to the blit, which copies the encoded bits as they lie; +// - the blit, where the typed upscale cannot be made, warns of wrong colours for both. +// The stand-in looks like the upscaler Off (GPUWORK-20): softer, none of the network's anti-aliasing. Only the path of +// a network that cannot run changes; where it runs, nothing does. +// +// RX_VK_STUB_SAME_SIZE=0: the decline and the first network alone in the typed upscale, as before. +// RX_VK_NO_UPSCALE_STUB still turns the whole stub off. +#pragma once +#include +#include + +namespace stub_same_size { + +/// RX_VK_STUB_SAME_SIZE: on unless set to 0 +inline bool on(const char* env) { + return !env || std::atoi(env) != 0; +} + +/// Whether the stub takes a dispatch by its sizes: an output larger than its input, or (the rule on) a gameplay +/// network's as large as it. An output smaller than its input never +constexpr bool sized(uint64_t out_area, uint64_t in_area, bool gameplay, bool on) { + return out_area > in_area || (on && gameplay && out_area == in_area && out_area != 0); +} + +/// Whether the typed upscale is tried first: any program that is not a gameplay network, the 3/4 network, and (the +/// rule on) the other gameplay network too +constexpr bool typed(bool gameplay, bool taau, bool on) { + return !gameplay || taau || on; +} + +/// Whether the blit that follows a typed upscale that could not be made warns of wrong colours: the 3/4 network's, and +/// (the rule on) every gameplay network's +constexpr bool warns(bool gameplay, bool taau, bool on) { + return taau || (on && gameplay); +} + +} // namespace stub_same_size diff --git a/gpu/vk/surface_lost.h b/gpu/vk/surface_lost.h new file mode 100644 index 0000000..0ac5ff1 --- /dev/null +++ b/gpu/vk/surface_lost.h @@ -0,0 +1,39 @@ +// (Triforce, 11-22, the performance audit's PRESENT-45, after Source A's) Presentation made again after the window's +// surface is lost. +// +// An acquire, a present or a swapchain's creation can answer VK_ERROR_SURFACE_LOST_KHR: the surface itself is gone (a +// display driver reset, a monitor change, a remote session), and no swapchain can be made for it again. Nothing handled +// that result: every acquire after it failed without a word, so the picture stayed frozen while the game ran on, until a +// restart. +// +// Now the renderer lets go of what hangs on the surface, in the order Vulkan asks (vk_main.cpp invalidate_lost_surface: +// the device idle, the swapchain's views, its semaphores, the swapchain, then the surface), and the next swapchain's +// creation first makes a surface for the window again with the startup's own calls (open_window_surface): only while the +// window has a size (a minimized window waits), one try each kRetryMs at most, and with the presents' queue chosen again +// for the new surface. +// +// RX_VK_SURFACE_RECOVER=0: as before, the result is not handled. Nor is it while NVIDIA DLSS frame generation runs: the +// swapchain is Streamline's there, and that path has not been tried. +// +// No Vulkan: tests/test_present_surface_lost.py compiles this, and runs vk_main.cpp's and vk_replay.cpp's side against a +// model of a driver that loses its surface. +#pragma once +#include +#include + +namespace surface_lost { + +/// RX_VK_SURFACE_RECOVER (value, null when unset): on unless it is 0 +inline bool on(const char* value) { return !(value && *value && std::atoi(value) == 0); } + +/// A Vulkan result is a lost surface that is recovered from: lost (the result is VK_ERROR_SURFACE_LOST_KHR), the switch +/// on, and the swapchain the renderer's own (not Streamline's) +inline bool recovers(bool lost, bool on, bool dlssg_active) { return lost && on && !dlssg_active; } + +/// The least time between two tries to make the surface again +constexpr int64_t kRetryMs = 250; + +/// A try is due at now_ms: none made yet (last_ms 0), or the last one kRetryMs ago or more +inline bool retry_due(int64_t now_ms, int64_t last_ms) { return !last_ms || now_ms - last_ms >= kRetryMs; } + +} // namespace surface_lost diff --git a/gpu/vk/table_stream.h b/gpu/vk/table_stream.h new file mode 100644 index 0000000..0fd2531 --- /dev/null +++ b/gpu/vk/table_stream.h @@ -0,0 +1,97 @@ +// (Triforce, 11-22, the performance audit's SYNC-04, after Source A's) The guest page table's changes of one recording, +// streamed into transfers of at most 64 KiB. +// +// A change to the table is a run: `count` pages from page `first`, each given `address` plus its offset (or 0 when the +// address is 0: unmapped). The recording's flush made a (page, value) pair for every page of every run, sorted them all +// and kept the last of each page: 16 bytes a page and a sort, on the recording thread, for ranges that are hundreds of +// MB when a pool is imported ahead or merged (150,000 pages: about 8 ms, and 2.4 MB of scratch). +// +// stream() writes the same transfers in the same order without the pairs: runs that come in page order and do not +// overlap, which is nearly always, are appended one after the other; any other set is swept by page with a heap of +// the runs covering the page, in which the newest run (the highest index) owns it. One batch of 8,192 entries is reused +// for every transfer. The transfers are those of the sort it replaces, entry for entry: tests/test_page_table_stream.py +// compares the two on random runs. RX_VK_TABLE_STREAM=0 (vk_memory.cpp memory_table_flush): the sort, as before. +// +// No Vulkan here: emit(first_page, values, n) is the caller's vkCmdUpdateBuffer. +#pragma once +#include +#include +#include +#include +#include + +namespace table_stream { + +constexpr size_t kBatch = 8192; // (entries a transfer: 64 KiB, the most vkCmdUpdateBuffer takes) + +struct Scratch { + std::vector batch; // (the transfer being filled, kept from flush to flush) + std::vector order, heap; // (the sweep's: an index a run) +}; + +/// The runs' changes (the later run of a page wins) as transfers of consecutive pages, ascending, each of kBatch entries +/// at most: emit(first_page, values, n). Run: {first, count, address}. How many entries were written +template +uint64_t stream(const Run* runs, size_t n, Scratch& s, Emit&& emit) { + s.batch.clear(); + if (s.batch.capacity() < kBatch) s.batch.reserve(kBatch); + uint64_t batch_first = 0, written = 0; + const auto send = [&] { + if (!s.batch.empty()) emit(batch_first, (const uint64_t*)s.batch.data(), s.batch.size()); + s.batch.clear(); + }; + // (count pages from `page`, the first one's value `address`, 0: all zero. A transfer ends at a gap or when it is full) + const auto append = [&](uint64_t page, uint64_t count, uint64_t address) { + written += count; + while (count) { + if (!s.batch.empty() && (page != batch_first + s.batch.size() || s.batch.size() == kBatch)) send(); + if (s.batch.empty()) batch_first = page; + const uint64_t room = kBatch - s.batch.size(), take = count < room ? count : room; + const size_t at = s.batch.size(); + s.batch.resize(at + (size_t)take); + uint64_t* const out = s.batch.data() + at; + if (address) + for (uint64_t i = 0; i < take; i++) out[i] = address + (i << 12); + else + std::fill(out, out + take, uint64_t(0)); + page += take, count -= take; + if (address) address += take << 12; + } + }; + // (in page order and apart: each run ends at or before the next one's first page) + const bool in_order = std::adjacent_find(runs, runs + n, [](const Run& a, const Run& b) { return a.first + a.count > b.first; }) == runs + n; + if (in_order) { + for (size_t i = 0; i < n; i++) append(runs[i].first, runs[i].count, runs[i].address); + send(); + return written; + } + // (overlapping or out of order: the runs by first page; of those covering a page the newest owns it. A run stays in + // the heap past its end until it is the top, and is dropped there) + s.order.resize(n); + std::iota(s.order.begin(), s.order.end(), uint32_t(0)); + std::sort(s.order.begin(), s.order.end(), [&](uint32_t a, uint32_t b) { return runs[a].first != runs[b].first ? runs[a].first < runs[b].first : a < b; }); + s.heap.clear(); + uint64_t pos = 0; + size_t k = 0; + while (k < n || !s.heap.empty()) { + if (s.heap.empty()) pos = runs[s.order[k]].first; + while (k < n && runs[s.order[k]].first <= pos) { + s.heap.push_back(s.order[k++]); + std::push_heap(s.heap.begin(), s.heap.end()); + } + while (!s.heap.empty() && runs[s.heap.front()].first + runs[s.heap.front()].count <= pos) { + std::pop_heap(s.heap.begin(), s.heap.end()); + s.heap.pop_back(); + } + if (s.heap.empty()) continue; + const Run& r = runs[s.heap.front()]; + uint64_t stop = r.first + r.count; // (this run owns the pages up to its end, or to where a run begins that may be newer) + if (k < n && runs[s.order[k]].first < stop) stop = runs[s.order[k]].first; + append(pos, stop - pos, r.address ? r.address + ((pos - r.first) << 12) : 0); + pos = stop; + } + send(); + return written; +} + +} // namespace table_stream diff --git a/gpu/vk/target_overlap.h b/gpu/vk/target_overlap.h index fe13088..1a3a72e 100644 --- a/gpu/vk/target_overlap.h +++ b/gpu/vk/target_overlap.h @@ -17,4 +17,32 @@ inline bool shifted_over(uint64_t lo, uint64_t hi, uint64_t last_used, uint64_t return b > a && (b - a) * 8 >= hi - lo; } +/// ((Triforce, 11-21, the performance audit's TEX-30, after Source A's) The same question for all the other render targets +/// together. Judged one by one, an eighth each, a target that two others had drawn over between them, one over its head +/// and one over its tail, each less than an eighth, was kept for good: its texels are as gone from that memory. Here the +/// parts the others cover inside [lo, hi) are added up, each byte once: add() takes the others in the order of their starts +/// (vk_texture.cpp's by_start), so a part never starts before the one before it and one pass merges them. A target with +/// the same start, one inside it and one not written after its last use count nothing, as in shifted_over +struct ShiftedOver { + uint64_t lo, hi, last_used; + uint64_t closed = 0, run_lo = 0, run_hi = 0; // (the parts merged and done with, and the one still growing) + + void add(uint64_t olo, uint64_t ohi, uint64_t o_written) { + if (hi <= lo || ohi <= olo || olo == lo || o_written <= last_used) return; + if (olo >= lo && ohi <= hi) return; + const uint64_t a = std::max(lo, olo), b = std::min(hi, ohi); + if (b <= a) return; + if (a <= run_hi) { + run_hi = std::max(run_hi, b); + } else { + closed += run_hi - run_lo; + run_lo = a, run_hi = b; + } + } + /// The bytes of it the others cover + uint64_t covered() const { return closed + (run_hi - run_lo); } + /// Whether that is an eighth of it or more + bool an_eighth() const { return hi > lo && covered() * 8 >= hi - lo; } +}; + } // namespace mkwvk::target_overlap diff --git a/gpu/vk/target_pool.h b/gpu/vk/target_pool.h new file mode 100644 index 0000000..4ec0ffa --- /dev/null +++ b/gpu/vk/target_pool.h @@ -0,0 +1,58 @@ +// (Triforce, 11-21, the performance audit's TEX-29, after Source A's and Source B's) The render targets' pool: how much it +// may hold, what goes first, and when. +// +// The pool gives a dead render target's image to the next target of its shape, so that a target the game makes again each +// frame costs no image creation. It held a fixed 256 MB, kept every image 120 presents, was trimmed in the order of its +// hash table, and was emptied whenever VRAM use passed 90% of the allowance. Source A found it starved at 2560 x 1440: of +// 22,000 images it was given in 100 s it destroyed 21,879 for being over the bound, and with a larger bound, an age order +// and tiers the images made new fell from about 20,000 to about 4,300 a run and the 1% low rose from 36 to 44 fps. +// - The bound: a twelfth of what the driver allows, 256 MB at least and 768 at most (RX_VK_TARGET_POOL_MB sets it; 0: no +// pool). +// - Trimmed oldest first, through one list in the order the images came in: over the bound, the image that has waited +// longest goes, whatever its shape. +// - The wait: a shape never taken from the pool goes after RX_VK_POOL_IDLE presents (8): the game's per-character shadow +// maps come in sizes of their own that nothing asks for again, and at 120 presents they filled the bound. A shape that +// was taken before keeps 120. +// - Tiers by the VRAM in use: above 90% of the allowance half the bound, above 97% none. With a floor of blocks kept for +// the targets (vk_vma.cpp target_floor_kept) the tiers do nothing, as the pool was not given back there before either: +// what it gave back went into blocks the floor keeps. +// RX_VK_TARGET_POOL_TRIM=0: no tiers (the pool emptied above 90%), as before; with RX_VK_TARGET_POOL_MB=256 and +// RX_VK_POOL_IDLE=120 the pool is the one it was, but for the order. +// +// The rules alone, no Vulkan: tests/test_target_pool_rules.py compiles them. +#pragma once +#include +#include +#include + +namespace target_pool { + +inline constexpr uint64_t kLeastMb = 256, kMostMb = 768; +inline constexpr uint64_t kIdle = 8, kIdleTaken = 120; // (presents: a shape never taken from the pool; one taken before) +inline constexpr size_t kTrim = 16, kTrimPressed = 64; // (images destroyed a present; while a tier holds) +inline constexpr size_t kTakenKeys = 4096; // (the shapes remembered as taken) + +/// The bound in MB with no switch: a twelfth of the allowance (bytes), kLeastMb to kMostMb; not known (0): kLeastMb +inline uint64_t default_mb(uint64_t allowed) { + return allowed ? std::clamp((allowed >> 20) / 12, kLeastMb, kMostMb) : kLeastMb; +} +/// The tier: 0, 1 with the VRAM in use above 90% of the allowance, 2 above 97%. known: the driver gave both numbers +inline int tier(bool known, uint64_t in_use, uint64_t allowed) { + if (!known || !allowed) return 0; + return in_use > allowed / 100 * 97 ? 2 : in_use > allowed / 10 * 9 ? 1 : 0; +} +/// What the pool may hold under a tier. tiers: RX_VK_TARGET_POOL_TRIM (off: any tier empties it, as before). floor_kept: a +/// floor of the targets' blocks is kept, and the pool is not given back at all +inline uint64_t cap_for(uint64_t cap, int tier, bool tiers, bool floor_kept) { + if (tier <= 0 || floor_kept) return cap; + return tiers && tier == 1 ? cap / 2 : 0; +} +/// How many images a present may destroy +inline size_t trim_most(int tier, bool floor_kept) { return tier > 0 && !floor_kept ? kTrimPressed : kTrim; } +/// Whether an image put in at present since has waited out its time at present now. idle: RX_VK_POOL_IDLE; taken_before: +/// its shape was taken from the pool lately +inline bool idle_out(uint64_t now, uint64_t since, uint64_t idle, bool taken_before) { + return now - since > (taken_before ? std::max(idle, kIdleTaken) : idle); +} + +} // namespace target_pool diff --git a/gpu/vk/tex_bc1.h b/gpu/vk/tex_bc1.h index 6473136..6349630 100644 --- a/gpu/vk/tex_bc1.h +++ b/gpu/vk/tex_bc1.h @@ -13,7 +13,9 @@ // same call with the same arguments as BC3's colour half. And BC3's alpha half for such a block is (255, 255, indices 0), // which decodes to 255, what the BC1 RGB format gives every texel. Which textures are all opaque: astc_opaque.h. // -// A row on the Graphics tab, Off unless set, because what a test cannot show is whether the GPU decoder agrees with +// ((Triforce, 11-21, TEX-50) On unless the row says off, since options_version 5; and with no options file too, which +// started it at 0 until then: vk_menu.cpp tex_bc1_at_start.) It was a row on the Graphics tab, off unless set, because what +// a test cannot show is whether the GPU decoder agrees with // astc_opaque.h on every block of the game's textures, and the image changes in vk_texture.cpp have not run in the game. #pragma once #include diff --git a/gpu/vk/tex_count.h b/gpu/vk/tex_count.h new file mode 100644 index 0000000..08f2aa8 --- /dev/null +++ b/gpu/vk/tex_count.h @@ -0,0 +1,40 @@ +// What a block-compressed image the GPU samples as it is (BC, or ASTC where the driver samples it) takes: whole blocks +// (no game, no GPU: a rule). tests/test_tex_count_blocks.py +// +// (Triforce, 11-22, the performance audit's TEX-86, after Source A's.) vk_texture.cpp counted such an image by its texels: +// a byte a texel for BC, and for ASTC the 16 bytes of a block for every texel, which is 16 times its size at 4 x 4, 64 +// times at 8 x 8 and 144 times at 12 x 12; BC1 and BC4, eight bytes a block, twice. The texture budget then let textures +// go for memory nothing held (Source A's Intel UHD 770, whose driver samples ASTC: 17,608 MB counted against a budget of +// 2,560, 2,241 MB by blocks), and the first size estimate left out the top levels of arrays that fit. A level is +// ceil(w / bw) x ceil(h / bh) blocks a slice, each block its bytes, times the layers. The textures the renderer encodes +// itself (decoded ASTC kept as BC3 or BC1) were counted right already and are not asked here. +// RX_VK_COUNT_BY_TEXELS=1: the count as it was. +#pragma once +#include +#include + +namespace tex_count { + +/// RX_VK_COUNT_BY_TEXELS: unset, empty or 0 counts by blocks; anything else brings the count by texels back +inline bool by_texels(const char* env) { return env && *env && std::atoi(env) != 0; } + +/// The blocks of one slice of level `l`: the level's size rounded up to whole blocks each way +constexpr uint64_t level_blocks(uint32_t width, uint32_t height, uint32_t l, uint32_t bw, uint32_t bh) { + const uint32_t w = l < 32 && (width >> l) ? (width >> l) : 1, h = l < 32 && (height >> l) ? (height >> l) : 1; + const uint32_t x = bw ? bw : 1, y = bh ? bh : 1; + return (uint64_t)((w + x - 1) / x) * ((h + y - 1) / y); +} + +/// The bytes of the levels `first` .. `levels - 1` of an image of blocks of `bw` x `bh` texels and `block_bytes` bytes: +/// a 3D image's level has its depth in slices of blocks (a block is flat), every layer has the whole chain +constexpr uint64_t bytes(uint32_t width, uint32_t height, uint32_t depth, uint32_t first, uint32_t levels, uint32_t layers, + uint32_t bw, uint32_t bh, uint32_t block_bytes) { + uint64_t blocks = 0; + for (uint32_t l = first; l < levels; l++) { + const uint32_t d = l < 32 && (depth >> l) ? (depth >> l) : 1; + blocks += level_blocks(width, height, l, bw, bh) * d; + } + return blocks * block_bytes * (layers ? layers : 1); +} + +} // namespace tex_count diff --git a/gpu/vk/tex_warm.h b/gpu/vk/tex_warm.h index 6b08528..7cfc533 100644 --- a/gpu/vk/tex_warm.h +++ b/gpu/vk/tex_warm.h @@ -18,6 +18,9 @@ #include #include #include +#include +#include +#include namespace tex_warm { @@ -49,4 +52,109 @@ inline bool ripe(uint64_t noted, uint64_t frame) { return frame >= noted + kAgeF /// Whether the look goes on after a texture: neither the time spent nor the frame's uploads took the budget inline bool goes_on(int64_t spent_ns, int64_t frame_upload_ns) { return spent_ns < kBudgetNs && frame_upload_ns < kBudgetNs; } +// ---- (Triforce, 11-22, the performance audit's TEX-23, after Source A's) The list of what was drawn ---- +// Every registered slot made ahead needs the 40% rule above, and on a card of 8 GB that rule stops the warming at once (40% is +// about 2.8 GB, below the 4 GB a stand in the lab drew from). Source A keeps a list of the textures a draw used within 90 s of +// the game making them, between runs, and makes those ahead with a room of their own: while the textures and 256 MB more fit +// their budget. A texture is named in the list by its place: its shape and eight 64-byte samples of its bytes (the signature), +// mixed with its storage's address, so that a texture at a new address is not taken for one the list has (it is made at its +// draw as before, and learnt then). RX_VK_TEX_WARM_LIST=1: the listed slots first, the others after them while the 40% rule +// allows; 2: the listed slots alone; unset or 0: every slot as above. The list is learnt only while it is on. The file is +// triforce_textures_drawn.bin beside the game, its own magic and version; one cut short or of another version is not read. +inline int list_mode_of(const char* env) { + if (!env || !*env) return 0; + const int v = std::atoi(env); + return v == 1 || v == 2 ? v : 0; +} + +inline constexpr int64_t kLearnNs = 90'000'000'000; // (a texture drawn within this of the game making it is learnt) +inline constexpr size_t kListMost = 32768; // (places in the list: past it the list starts again) +inline constexpr uint64_t kListRoom = 256ull << 20; // (listed textures are made while the textures and this fit their budget) +inline constexpr uint64_t kSaveEvery = 300; // (presents between saves of a list that changed) +inline constexpr int kSamples = 8; +inline constexpr uint64_t kSampleBytes = 64; +inline constexpr char kListMagic[8] = {'T', 'F', 'D', 'R', 'A', 'W', 'N', 0}; +inline constexpr uint32_t kListVersion = 1; + +/// What a texture's signature takes of its shape (its descriptor and its storage's size) +struct Shape { + uint32_t width, height, depth, levels, format, target, layers; + uint64_t size; +}; + +/// Whether a texture the game made at born_ns (0: not known) and drawn at now_ns is learnt: within kLearnNs of it +inline bool learnable(int64_t born_ns, int64_t now_ns) { return born_ns > 0 && now_ns >= born_ns && now_ns - born_ns <= kLearnNs; } + +/// Where sample i (0 to kSamples - 1) of a storage of `size` bytes starts: spread from its start to its end, 64-byte aligned +inline uint64_t sample_at(uint64_t size, int i) { + if (size <= kSampleBytes) return 0; + return ((size - kSampleBytes) * (uint64_t)i / (kSamples - 1)) & ~(kSampleBytes - 1); +} +/// How many bytes each sample reads: kSampleBytes, or the whole of a smaller storage +inline uint64_t sample_len(uint64_t size) { return size < kSampleBytes ? size : kSampleBytes; } + +/// The signature: the shape and the samples (kSamples * kSampleBytes bytes, sample i at i * kSampleBytes), FNV-1a +inline uint64_t signature(const Shape& s, const uint8_t* samples) { + uint64_t h = 0xCBF29CE484222325ull; + const auto add = [&h](uint64_t v) { + for (int i = 0; i < 8; i++) h = (h ^ ((v >> (8 * i)) & 0xFF)) * 1099511628211ull; + }; + add(s.width), add(s.height), add(s.depth), add(s.levels), add(s.format), add(s.target), add(s.layers), add(s.size); + for (uint64_t i = 0; i < kSamples * kSampleBytes; i++) h = (h ^ samples[i]) * 1099511628211ull; + return h; +} +/// A signature at a storage address: what the list holds (never 0) +inline uint64_t place(uint64_t sig, uint64_t gpu) { + uint64_t h = sig ^ (gpu * 0x9E3779B97F4A7C15ull); + h ^= h >> 31, h *= 0xBF58476D1CE4E5B9ull, h ^= h >> 29, h *= 0x94D049BB133111EBull, h ^= h >> 32; + return h | 1; +} +/// A place learnt: true when the list changed. A full list (kListMost) starts again, so that it follows what the game draws now +inline bool remember(std::unordered_set& list, uint64_t p) { + if (!p || list.count(p)) return false; + if (list.size() >= kListMost) list.clear(); + list.insert(p); + return true; +} +/// Whether a listed texture may be made: the target pool is not being given back (VRAM near the allowance), and the textures +/// with kListRoom more fit their budget (0 or all ones: none is set, no room is known) +inline bool listed_room(bool pressed, uint64_t texture_bytes, uint64_t budget) { + if (pressed || !budget || budget == ~0ull) return false; + return texture_bytes <= budget && budget - texture_bytes >= kListRoom; +} + +inline uint64_t list_check(const uint8_t* b, size_t n) { + uint64_t h = 0xCBF29CE484222325ull; + for (size_t i = 0; i < n; i++) h = (h ^ b[i]) * 1099511628211ull; + return h; +} +/// The file's bytes: the magic, the version, the count, the places, and a check of the places +inline std::vector encode(const std::unordered_set& list) { + std::vector o(16 + list.size() * 8 + 8); + const uint32_t head[2] = {kListVersion, (uint32_t)list.size()}; + std::memcpy(o.data(), kListMagic, 8), std::memcpy(o.data() + 8, head, 8); + size_t at = 16; + for (const uint64_t p : list) std::memcpy(o.data() + at, &p, 8), at += 8; + const uint64_t check = list_check(o.data() + 16, list.size() * 8); + std::memcpy(o.data() + at, &check, 8); + return o; +} +/// The places of a file's bytes into `out`; false, and `out` left empty, for anything that is not a whole file of this version +inline bool decode(const uint8_t* b, size_t n, std::unordered_set& out) { + out.clear(); + if (!b || n < 24 || std::memcmp(b, kListMagic, 8)) return false; + uint32_t head[2]; + std::memcpy(head, b + 8, 8); + if (head[0] != kListVersion || head[1] > kListMost || n != 16 + (size_t)head[1] * 8 + 8) return false; + uint64_t check; + std::memcpy(&check, b + n - 8, 8); + if (check != list_check(b + 16, (size_t)head[1] * 8)) return false; + for (uint32_t i = 0; i < head[1]; i++) { + uint64_t p; + std::memcpy(&p, b + 16 + (size_t)i * 8, 8); + if (p) out.insert(p); + } + return true; +} + } // namespace tex_warm diff --git a/gpu/vk/translate_wait.h b/gpu/vk/translate_wait.h new file mode 100644 index 0000000..49ac6e3 --- /dev/null +++ b/gpu/vk/translate_wait.h @@ -0,0 +1,71 @@ +// A bounded wait for a draw whose program is still translating (no game, no GPU: the switch, the choice and the notes of +// which programs write memory; vk_draw.cpp draw, translate_note). Apart from Vulkan so that tests/test_translate_wait.py runs +// it alone. +// +// (Triforce, 11-22, the performance audit's PIPE-30, after Source A's.) A draw whose program is still being translated +// waits for it, its thread running the job if it is still queued (jobs_wait); RX_VK_PIPE_SKIP=2 leaves such a draw out +// when its targets hold an earlier frame, with no bound and no regard for what its shaders write. Source A, with its switch +// at 8 ms: a draw into targets that hold an earlier frame waits at most that much in all a frame, never running the job +// itself, then is left out (new targets and compute still wait). On its RTX 4070 SUPER, title to Castle Town to the field +// on a cold bank, fps 44.9/45.3 -> 50.0/47.4, the summed wait 4.94/4.85 -> 1.80/1.70 s, frames over 100 ms 39/34 -> 22/23, +// but 4.4k -> 21-23k draws left out, mostly in black load frames. +// +// With the guard the audit asked for: an untranslated program cannot be asked whether it writes memory, so only a program +// key whose variants translated before are known to write none is bounded (the variants of a key share their stage code). +// A key never translated, or one with a variant that writes, waits as before, and so does a draw into new targets. +// +// RX_VK_TRANSLATE_WAIT_MS=: 1 to 100 ms a frame in all, digits only (8 to try); unset, 0 or anything else: off, every +// such draw waits, as before. RX_VK_PIPE_SKIP=0 and =2 keep their meaning. +#pragma once + +#include +#include +#include + +namespace translate_wait { + +inline constexpr int64_t kMs = 1000000; // (ns) +inline constexpr int kMostMs = 100; + +/// RX_VK_TRANSLATE_WAIT_MS: the milliseconds a frame, strictly: digits only, 1 to kMostMs; anything else 0 (off) +inline int parse(const char* env) { + if (!env || !*env) return 0; + int v = 0; + for (const char* c = env; *c; c++) { + if (*c < '0' || *c > '9') return 0; + v = v * 10 + (*c - '0'); + if (v > kMostMs) return 0; + } + return v; +} + +enum class Act { AsBefore, Bounded, Skip }; +/// A draw whose program is still translating. kept: its targets hold an earlier frame (pass_kept); clean: its program key is +/// known to write no memory; left_ns: what the frame's budget has left. Bounded: wait at most left_ns, then leave it out; +/// Skip: the budget is spent, left out at once; AsBefore: waited for +inline Act choose(int ms, bool kept, bool clean, int64_t left_ns) { + if (ms <= 0 || !kept || !clean) return Act::AsBefore; + return left_ns >= kMs ? Act::Bounded : Act::Skip; +} + +/// Which program keys write memory, from their variants once translated: a key is clean only when every variant noted +/// wrote none; one that writes sticks +class Notes { +public: + void note(uint64_t key, bool writes) { + std::lock_guard lk(mu_); + bool& w = by_[key]; + w = w || writes; + } + bool clean(uint64_t key) const { + std::lock_guard lk(mu_); + const auto it = by_.find(key); + return it != by_.end() && !it->second; + } + +private: + mutable std::mutex mu_; + std::unordered_map by_; // (key -> some variant writes) +}; + +} // namespace translate_wait diff --git a/gpu/vk/ub_batch.h b/gpu/vk/ub_batch.h new file mode 100644 index 0000000..6ac87e1 --- /dev/null +++ b/gpu/vk/ub_batch.h @@ -0,0 +1,52 @@ +// (Triforce, 11-22, the performance audit's SYNC-21, after Source A's) A recording's uniform updates as copies of many +// regions, ordered where they overlap: when a run of them ends. +// +// A recording's uniform updates are written into guest memory at its end, between two full barriers: one +// vkCmdUpdateBuffer an update (about 80 a frame), and nothing between two updates of the same bytes, whose order on the +// GPU is then not defined (transfer writes with no dependency between them; so far a validation finding only). With +// RX_VK_UB_BATCH=1 the updates' bytes go into the recording's stream and each run of them into one buffer is one +// vkCmdCopyBuffer of many regions. The regions of one copy may not overlap, and two writes of the same bytes need an +// order: a run ends where a piece would write bytes the run already wrote, or where the destination buffer changes, and +// the next run starts behind a barrier. This header is that rule alone (no Vulkan): vk_replay.cpp flush_ub_batched asks +// it, tests/test_ub_update_batch.py runs both and takes each part of the rule away in turn. +#pragma once +#include +#include +#include +#include + +namespace ub_batch { + +/// What has to happen before a piece joins the run: nothing; the regions so far copied (their source buffer is another, +/// no barrier); the run ended by an overlap; the run ended by another destination. The last two: the regions so far +/// copied, then a barrier, then a new run +enum Before : int { JOIN = 0, COPY = 1, RUN_OVERLAP = 2, RUN_IMPORT = 3 }; + +template +struct Run { + Buffer src{}, dst{}; // (the stream buffer the pending regions' bytes are in; the buffer the run writes) + bool any = false; // (a run is open: dst is set) + size_t pending = 0; // (regions not copied yet) + std::vector> written; // ([at, end) in dst written since the run began, copied or not) + + bool overlaps(uint64_t at, uint64_t end) const { + for (const auto& w : written) + if (at < w.second && w.first < end) return true; + return false; + } + /// Before a piece from stream buffer s into [at, end) of d joins + Before before(Buffer s, Buffer d, uint64_t at, uint64_t end) const { + if (!any) return JOIN; + if (d != dst) return RUN_IMPORT; + if (overlaps(at, end)) return RUN_OVERLAP; + if (pending && s != src) return COPY; + return JOIN; + } + void clear() { any = false, pending = 0, written.clear(); } + void begin(Buffer d) { dst = d, any = true, pending = 0, written.clear(); } // (a new run, into d) + void add(Buffer s, uint64_t at, uint64_t end) { src = s, pending++, written.push_back({at, end}); } // (a region joins) + void wrote(uint64_t at, uint64_t end) { written.push_back({at, end}); } // (bytes of dst written some other way in this run) + void copied() { pending = 0; } // (the pending regions went into a copy) +}; + +} // namespace ub_batch diff --git a/gpu/vk/upscale_off.h b/gpu/vk/upscale_off.h new file mode 100644 index 0000000..84ef8fb --- /dev/null +++ b/gpu/vk/upscale_off.h @@ -0,0 +1,34 @@ +// Which of the game's network dispatches the plain stretch stands in for, and how it draws (no game, no GPU: a rule). +// +// (Triforce, 11-22, the performance audit's GPUWORK-20, after Source A's.) The game upscales its 3D with a small neural +// network every frame: fd029a317a9a4b94 from 3/4 size, 9dc98871eb7fc101 at full size, and one network of its own for each +// other share of the Render resolution row. vk_draw.cpp taau_stand_in already draws the first one's output with a +// stretch of its input (shaders/taau_upscale.comp, which undoes the network's square-root encoding) where the network +// runs on software matrix multiplies and costs most of a frame; 9dc98871 still ran every frame there (2.6 ms of GPU +// time at 2560x1440 on a mid-range AMD card, as Source A measured its like). +// +// "Off" leaves every gameplay upscale network out: its input stretched linearly into its output, no FXAA of the +// stand-in's and no history bound (the game's own SMAA has already run on the input and stays). It is opt-in +// (RX_VK_UPSCALE_OFF=1, or the lab's "upoff" phases): the picture is softer below 100% and has none of the network's +// sharpening. The networks are still compiled as before, so leaving Off again needs nothing made. +#pragma once + +namespace upscale_off { + +struct Choice { + bool stand_in; // the stretch draws the dispatch's output; the network is not dispatched + bool temporal; // as the stand-in was before: its history bound and FXAA. false: the plain linear stretch + bool by_shape; // taken for its shape alone: the caller must still see the scene's formats at both ends +}; + +/// listed: the program is one RX_VK_TAAU_KEYS names (by default fd029a317a9a4b94 on software multiplies); +/// off: the upscaler is Off; +/// by_key: the program is one of the two gameplay networks known by key; +/// shaped: it has a gameplay upscale's shape (a network sampling one texture into one image). +constexpr Choice choose(bool listed, bool off, bool by_key, bool shaped) { + if (off && by_key) return {true, false, false}; + if (off && shaped && !listed) return {true, false, true}; + return {listed, true, false}; +} + +} // namespace upscale_off diff --git a/gpu/vk/upscaler_rows.h b/gpu/vk/upscaler_rows.h index 0d89a0f..82efeeb 100644 --- a/gpu/vk/upscaler_rows.h +++ b/gpu/vk/upscaler_rows.h @@ -2,6 +2,8 @@ Upscaler, and the names it shows for DLSS's (and FSR 3.1's and FSR 4's) modes. No Vulkan, no menu: tests/test_upscaler_rows.cpp checks them. */ #pragma once +#include + namespace mkwvk::upscaler_rows { /// The Upscaler row (opts.dlss, RX_VK_DLSS): the game's own network, NVIDIA DLSS, NVIDIA DLAA, AMD FSR 3.1 (built from AMD's @@ -189,7 +191,18 @@ inline int window_filter(unsigned picture_w, unsigned picture_h, unsigned dest_w /// generation transformer, preset M (NVIDIA's own pick for Performance); 2 the same generation as tuned for Ultra /// Performance, preset L; 3 NVIDIA's default, preset 0 (the runtime picks per mode, and a runtime update can change it) enum { MODEL_K = 0, MODEL_M = 1, MODEL_L = 2, MODEL_DEFAULT = 3, kModels = 4 }; -inline unsigned model_preset(int model) { +/// (Triforce, 11-22, the performance audit's PRESENT-36, after Source A's and Source B's) The K row on a card with less +/// than 7 GB of video memory asks for preset E, NVIDIA's last CNN model: less GPU time and less of NGX's own memory than +/// the transformer there (Source A's, a 6 GB laptop RTX 3050 at 1080p DLSS Performance: 15.75 to 14.20 ms of GPU a frame, +/// 85 to 58 MB), with more ghosting on grass and thin objects at the smallest render sizes, and NVIDIA marks it deprecated. +/// vram: the card's video memory in bytes; 0 when the rule is off (vk_dlss.cpp dlss_cnn_vram: RX_VK_DLSS_CNN_VRAM is not +/// 1, the default) or the size is not known: K as chosen. M, L and NVIDIA's default stay as chosen, and RX_VK_DLSS_PRESET +/// goes over all of it +constexpr unsigned kPresetE = 5; +constexpr uint64_t kCnnVramBytes = 7ull << 30; +inline bool cnn_for_vram(int model, uint64_t vram) { return model == MODEL_K && vram && vram < kCnnVramBytes; } +inline unsigned model_preset(int model, uint64_t vram = 0) { + if (cnn_for_vram(model, vram)) return kPresetE; switch (model) { case MODEL_M: return 13; case MODEL_L: return 12; @@ -202,8 +215,8 @@ inline unsigned model_preset(int model) { inline bool preset_env(const char* env) { return env && ((*env >= 'A' && *env <= 'Z') || (*env >= 'a' && *env <= 'z') || (env[0] == '0' && !env[1])); } -inline unsigned dlss_preset(int model, const char* env) { - if (!preset_env(env)) return model_preset(model); +inline unsigned dlss_preset(int model, const char* env, uint64_t vram = 0) { + if (!preset_env(env)) return model_preset(model, vram); if (*env == '0') return 0; return (unsigned)(*env >= 'a' ? *env - 'a' + 1 : *env - 'A' + 1); } diff --git a/gpu/vk/vk_api.h b/gpu/vk/vk_api.h index 9104a28..1f3233a 100644 --- a/gpu/vk/vk_api.h +++ b/gpu/vk/vk_api.h @@ -150,6 +150,27 @@ enum { HOS_MODS_PRESS_UP = 1, HOS_MODS_PRESS_DOWN = 2, HOS_MODS_PRESS_LEFT = 4, text. HOS_OPT_TEXT_CHAR, option_set (the window's thread, host/display.c): a code point typed, or 8 Backspace, 127 Ctrl+Backspace (a pasted text comes as its characters) */ enum { HOS_OPT_TEXT_CHAR = 159, HOS_OPT_TEXT_WANTED }; +/* ((Triforce) 11-20) The Mods screen's music (host/mods_music.c). HOS_OPT_MODS_MUSIC, option: 0 none; 1 the title's list + is there with its Mods entry (the tune is read from the game's files ahead of its use); 2 the Mods screen is open from + the title's list: the game's shop theme plays quietly and the game's own sound, the title's music, is faded out + under it, and back when the screen shuts */ +enum { HOS_OPT_MODS_MUSIC = 161 }; +/* ((Triforce) 11-20) The Mods entry with the game's own buttons, not by a clock of its own. HOS_OPT_MODS_ALPHA, + option_set: how much of the list's last button the game shows this frame, in thousandths (0 to 1000: the button's own + alpha as the game animates it, host/oot_menu.c), told before that frame's HOS_OPT_MODS_LIST. A HOS_OPT_MODS_LIST without + HOS_MODS_LIST_SHOWN says that the list is there no longer for the entry (another screen has the game's cursor): the + entry goes at once, as the game's buttons did */ +enum { HOS_OPT_MODS_ALPHA = 162 }; +/* (Triforce) Achievements (host/achievements.c keeps them, on unless HOS_ACHIEVEMENTS=0; gpu/vk/vk_menu.cpp and + gpu/vk/achievement_rows.h show them: the System tab's Achievements page, and a notice when one is earned; 165 to 169 are + theirs). HOS_OPT_ACH_ASK, option: 1 from a renderer that shows them. Once it answers the host names its list, in the list's + order: HOS_OPT_ACH_TEXT, option_set: the next byte of an achievement's name, UTF-8, then a line feed and the line about it, + then a line feed and the day it was earned when it was; HOS_OPT_ACH_ADD, option_set: the bytes sent so far are the next + achievement's, the value its icon (0 to 255, host/achievements.h ach_icon) and HOS_ACH_EARNED when it is earned; + HOS_OPT_ACH_COUNT, option_set: the list is whole (0: none). HOS_OPT_ACH_EARNED, option_set: the achievement at that place + in the list was earned now, the bytes sent so far the day: the notice shows it. One at a time, the host waits between two */ +enum { HOS_OPT_ACH_TEXT = 165, HOS_OPT_ACH_ADD, HOS_OPT_ACH_COUNT, HOS_OPT_ACH_EARNED, HOS_OPT_ACH_ASK }; +enum { HOS_ACH_EARNED = 256 }; /* (hookshot) the mouse over the window while it is free (not captured): the Insert menu's (host/display.c, host/posix/display_sdl.c). x and y: the cursor in the window's pixels, of a window width x height pixels (the diff --git a/gpu/vk/vk_dlss.cpp b/gpu/vk/vk_dlss.cpp index 994f215..c4d86aa 100644 --- a/gpu/vk/vk_dlss.cpp +++ b/gpu/vk/vk_dlss.cpp @@ -29,6 +29,8 @@ #endif #include "dlss_checks.h" +#include "dlss_feature.h" // ((Triforce, the upscaler audit's UPS-01) the render size the feature is made for) +#include "lab_ab.h" // ((Triforce, 11-22, PRESENT-38) the lab's "dlssstrict" phases) #include "upscaler_rows.h" #include "vk_menu.h" #include "vk_paths.h" @@ -76,7 +78,13 @@ struct State { NVSDK_NGX_Handle* feature = nullptr; uint32_t feature_rw = 0, feature_rh = 0, feature_ow = 0, feature_oh = 0; // (its largest render size, its output) uint32_t min_rw = 0, min_rh = 0; + // ((Triforce, 11-21, the upscaler audit's UPS-01) made for the one render size it is given (dlss_feature.h: a fixed share), + // not for the largest of its mode's range; and whether the render size was seen changing under one output in this run + // (the game's dynamic resolution), after which every feature is made for the range, as all were before) + bool feature_exact = false; + bool range_seen = false; unsigned feature_preset = ~0u; // (the preset it was made with: another one from the menu makes it again) + dlss_checks::PresetSettle settle; // ((Triforce, 11-21, PRESENT-37) the row's preset, taken once it has held: dlss_checks.h) bool feature_hdr = false; // (made for linear HDR colour, NGX's IsHDR: the other kind of colour makes it again) uint64_t features_made = 0; // (how many times: a feature made again in play costs a stall, so each says why) // (a feature not made for want of video memory: tried again from retry_frame on, kMemoryRetries times before it is given @@ -287,7 +295,17 @@ bool ngx_init() { /// DLSS's preset: the DLSS model row's, RX_VK_DLSS_PRESET over it (upscaler_rows.h dlss_preset) unsigned wanted_preset() { static const char* const env = getenv("RX_VK_DLSS_PRESET"); - return upscaler_rows::dlss_preset(opts.dlss_model, env && *env ? env : nullptr); + // ((Triforce, 11-22, the performance audit's PRESENT-36, after Source A's and Source B's) with RX_VK_DLSS_CNN_VRAM=1 the + // K row on a card under 7 GB asks for preset E, NVIDIA's CNN model (upscaler_rows.h cnn_for_vram); said once) + const uint64_t vram = dlss_cnn_vram(); + const unsigned preset = upscaler_rows::dlss_preset(opts.dlss_model, env && *env ? env : nullptr, vram); + static bool told; + if (!told && vram && preset != upscaler_rows::dlss_preset(opts.dlss_model, env && *env ? env : nullptr)) { + told = true; + logf("vk: NVIDIA DLSS: %llu MB of video memory: the K row draws with preset E (RX_VK_DLSS_CNN_VRAM=1)", + (unsigned long long)(vram >> 20)); + } + return preset; } /// DLSS's feature for this output size and preset, made when either changes (cmd: its creation's work): the engine's @@ -295,10 +313,17 @@ unsigned wanted_preset() { int prepare(VkCommandBuffer cmd, uint32_t rw, uint32_t rh, uint32_t ow, uint32_t oh, bool hdr, uint32_t* in_w, uint32_t* in_h, const char** fail) { State& s = st(); - const unsigned preset = wanted_preset(); - if (s.feature && s.feature_ow == ow && s.feature_oh == oh && rw <= s.feature_rw && rh <= s.feature_rh && - rw >= s.min_rw && rh >= s.min_rh && s.feature_preset == preset && s.feature_hdr == hdr) + // ((Triforce, 11-21, the performance audit's PRESENT-37, after Source A's) the DLSS model row's preset once the row has + // held it for RX_VK_DLSS_MODEL_SETTLE frames (30; 0: at once, as before): no feature for a preset stepped past) + static const uint64_t settle_frames = dlss_checks::settle_frames(getenv("RX_VK_DLSS_MODEL_SETTLE")); + const unsigned preset = s.settle.pick(wanted_preset(), s.feature_preset, s.feature != nullptr, texture_frame_no(), settle_frames); + // ((Triforce, UPS-01) the feature there is takes this frame: its one render size, or any of its range (dlss_feature.h takes) + const bool same_feature = s.feature && s.feature_ow == ow && s.feature_oh == oh && s.feature_preset == preset && s.feature_hdr == hdr; + if (same_feature && dlss_feature::takes(s.feature_exact, s.feature_rw, s.feature_rh, s.min_rw, s.min_rh, rw, rh)) return *in_w = s.feature_rw, *in_h = s.feature_rh, 1; + // (a feature of one render size, and a frame of another under the same output: the game changes the render size, so + // from here on the features of this run are made for their mode's whole range) + const bool found_range = dlss_feature::found_range(same_feature, s.feature_exact, s.feature_rw, s.feature_rh, rw, rh); static const char* const kNoMemory = "no video memory for its feature: tried again in a while"; if (s.retry_frame && texture_frame_no() < s.retry_frame) return *fail = kNoMemory, 0; // (why it is made: in the log with its time, so a stall in play can be matched with one) @@ -311,50 +336,90 @@ int prepare(VkCommandBuffer cmd, uint32_t rw, uint32_t rh, uint32_t ow, uint32_t snprintf(why, sizeof why, "preset %c, was %c (the DLSS model row)", upscaler_rows::preset_letter(preset), upscaler_rows::preset_letter(s.feature_preset)); else if (s.feature_hdr != hdr) snprintf(why, sizeof why, "the colour is %s now", hdr ? "HDR" : "LDR"); + else if (found_range) + snprintf(why, sizeof why, "the render size went from %ux%u to %ux%u: for its mode's whole range from now on", s.feature_rw, + s.feature_rh, rw, rh); else snprintf(why, sizeof why, "the render size %ux%u left its range, %ux%u to %ux%u", rw, rh, s.min_rw, s.min_rh, s.feature_rw, s.feature_rh); const uint64_t key = (uint64_t)ow << 44 ^ (uint64_t)oh << 24 ^ (uint64_t)preset << 1 ^ (hdr ? 1 : 0) ^ 1ull << 63; if (s.refused == key) return *fail = "its feature was not made", 0; // (until a DLSS setting changes: upstream #2550) - if (s.feature) { - NVSDK_NGX_Handle* old = s.feature; - temporal_later([old] { NVSDK_NGX_VULKAN_ReleaseFeature(old); }); - s.feature = nullptr; - } + if (found_range) s.range_seen = true; // (the quality mode whose range of render sizes holds this one: the network's 3/4 is DLSS's MaxQuality range) // (DLAA's when the render size is the output's) static const NVSDK_NGX_PerfQuality_Value scaled[] = {NVSDK_NGX_PerfQuality_Value_MaxQuality, NVSDK_NGX_PerfQuality_Value_DLAA, NVSDK_NGX_PerfQuality_Value_Balanced, NVSDK_NGX_PerfQuality_Value_MaxPerf, NVSDK_NGX_PerfQuality_Value_UltraPerformance}; static const NVSDK_NGX_PerfQuality_Value same[] = {NVSDK_NGX_PerfQuality_Value_DLAA, NVSDK_NGX_PerfQuality_Value_MaxQuality}; - const bool aa = rw == ow && rh == oh; + // ((Triforce, 11-22, the performance audit's PRESENT-32, after Source A's) with RX_VK_DLSS_PREWARM=1 the one-size modes + // only for NVIDIA DLAA: DLSS at the output's size (the Render resolution row's Auto, before the game lowers it) takes + // the scaled mode whose range holds it, and the smaller sizes after it draw with the same feature instead of making a + // second one in play (dlss_checks.h dlaa_feature). Off by default: as before, one size whenever the two are equal + static const bool dlaa_rule = dlss_checks::prewarm_rule(getenv("RX_VK_DLSS_PREWARM")); + const bool aa = dlss_checks::dlaa_feature(dlaa_rule, temporal_mode() == upscaler_rows::DLAA, rw, rh, ow, oh); const NVSDK_NGX_PerfQuality_Value* modes = aa ? same : scaled; const size_t nmodes = aa ? 2 : 5; NVSDK_NGX_PerfQuality_Value mode = modes[0]; unsigned max_w = 0, max_h = 0, min_w = 0, min_h = 0; bool found = aa; if (aa) max_w = min_w = ow, max_h = min_h = oh; // (DLAA: the render size is the output's, one size) - for (size_t i = 0; i < nmodes && !found; i++) { - const NVSDK_NGX_PerfQuality_Value m = modes[i]; - unsigned w = 0, h = 0, xw = 0, xh = 0, nw = 0, nh = 0; - float sharp = 0; - if (NVSDK_NGX_FAILED(NGX_DLSS_GET_OPTIMAL_SETTINGS(s.caps, ow, oh, m, &w, &h, &xw, &xh, &nw, &nh, &sharp)) || !w) continue; - if (rw >= nw && rh >= nh && rw <= xw && rh <= xh) { - mode = m, max_w = xw, max_h = xh, min_w = nw, min_h = nh, found = true; - break; + // ((Triforce, 11-22, the performance audit's PRESENT-38, after Source A's) every mode's range first, then the one that + // takes this size (dlss_checks.h mode_for): the first that holds it, else the first it is outside of by a rounding. The + // Render resolution row's shares are rounded up and NGX's sizes down, so at 2560x1440 the row's 33% (854x480) was in + // no range (Ultra Performance is 853x480), DLSS was refused every frame and the game's own upscale ran with DLSS + // chosen. RX_VK_DLSS_MODE_SLACK=0, or the lab's "dlssstrict" phases: only a range that holds the size, as before + if (!found) { + dlss_checks::ModeRange ranges[5]; + for (size_t i = 0; i < nmodes; i++) { + unsigned w = 0, h = 0, xw = 0, xh = 0, nw = 0, nh = 0; + float sharp = 0; + if (NVSDK_NGX_FAILED(NGX_DLSS_GET_OPTIMAL_SETTINGS(s.caps, ow, oh, modes[i], &w, &h, &xw, &xh, &nw, &nh, &sharp)) || !w) continue; + ranges[i] = {nw, nh, xw, xh, true}; + } + static const bool slack_env = !(getenv("RX_VK_DLSS_MODE_SLACK") && atoi(getenv("RX_VK_DLSS_MODE_SLACK")) == 0); + bool widened = false; + const int at = dlss_checks::mode_for(ranges, nmodes, rw, rh, ow, oh, slack_env && !lab_ab::on("dlssstrict"), &widened); + if (at >= 0) { + const dlss_checks::ModeRange& r = ranges[at]; + // (a size taken by the slack is in the range from here on: the feature's reuse test takes it next frame) + mode = modes[at], found = true; + min_w = std::min(r.min_w, rw), min_h = std::min(r.min_h, rh); + max_w = std::max(r.max_w, rw), max_h = std::max(r.max_h, rh); + static bool told; + if (widened && !told) { + told = true; + logf("vk: NVIDIA DLSS: %ux%u is a rounding outside mode %d's sizes for %ux%u (%ux%u to %ux%u): taken as that " + "mode's (RX_VK_DLSS_MODE_SLACK=0: refused, the game's upscale runs)", rw, rh, (int)mode, ow, oh, r.min_w, + r.min_h, r.max_w, r.max_h); + } } } + // (a size no mode takes: the feature there is stays, so the next size one takes does not make it again) if (!found) return *fail = "no DLSS mode takes this render size", 0; + if (s.feature) { // (only now is the old one let go: a mode takes this size, and a feature is made for it) + NVSDK_NGX_Handle* old = s.feature; + temporal_later([old] { NVSDK_NGX_VULKAN_ReleaseFeature(old); }); + s.feature = nullptr; + } // (the DLSS model row's preset for every mode, K by default: NVIDIA's transformer model. 0: NVIDIA's own per mode) static_assert(NVSDK_NGX_DLSS_Hint_Render_Preset_K == 11 && NVSDK_NGX_DLSS_Hint_Render_Preset_L == 12 && - NVSDK_NGX_DLSS_Hint_Render_Preset_M == 13 && NVSDK_NGX_DLSS_Hint_Render_Preset_Default == 0); + NVSDK_NGX_DLSS_Hint_Render_Preset_M == 13 && NVSDK_NGX_DLSS_Hint_Render_Preset_Default == 0 && + NVSDK_NGX_DLSS_Hint_Render_Preset_E == 5); // ((Triforce, PRESENT-36) E: upscaler_rows.h kPresetE) for (const char* k : {NVSDK_NGX_Parameter_DLSS_Hint_Render_Preset_DLAA, NVSDK_NGX_Parameter_DLSS_Hint_Render_Preset_Quality, NVSDK_NGX_Parameter_DLSS_Hint_Render_Preset_Balanced, NVSDK_NGX_Parameter_DLSS_Hint_Render_Preset_Performance, NVSDK_NGX_Parameter_DLSS_Hint_Render_Preset_UltraPerformance, NVSDK_NGX_Parameter_DLSS_Hint_Render_Preset_UltraQuality}) NVSDK_NGX_Parameter_SetUI(s.params, k, preset); NVSDK_NGX_DLSS_Create_Params cp{}; - cp.Feature.InWidth = max_w, cp.Feature.InHeight = max_h, cp.Feature.InTargetWidth = ow, cp.Feature.InTargetHeight = oh; + // ((Triforce, 11-21, the upscaler audit's UPS-01) a fixed share's feature for its own render size, not the largest its + // mode allows (NGX answers the output's own size for that: 2.25 times the pixels at Quality, 9 at Ultra Performance, for + // a feature whose cost and memory follow the size it is made for; NVIDIA's guide has the render size passed to the + // creation). For the mode's whole range where the game changes the render size: the Render resolution row's Auto, or a + // size seen changing under one output in this run. RX_VK_DLSS_FEATURE_MAX=1: always the largest, as before + static const bool whole_range = dlss_feature::whole_range(getenv("RX_VK_DLSS_FEATURE_MAX")); + const dlss_feature::Size made = + dlss_feature::size(aa, whole_range || s.range_seen || opts.resolution == 0, rw, rh, max_w, max_h); + cp.Feature.InWidth = made.w, cp.Feature.InHeight = made.h, cp.Feature.InTargetWidth = ow, cp.Feature.InTargetHeight = oh; cp.Feature.InPerfQualityValue = mode; // (LDR colour, already tonemapped: no exposure; motion at the render size, not jittered; depth not reversed) cp.InFeatureCreateFlags = NVSDK_NGX_DLSS_Feature_Flags_MVLowRes; @@ -367,7 +432,7 @@ int prepare(VkCommandBuffer cmd, uint32_t rw, uint32_t rh, uint32_t ow, uint32_t const double made_ms = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); note_work(cmd); // (NGX's own commands, which vk_work.h does not see: the next full_barrier is not left out) if (NVSDK_NGX_FAILED(r) || !s.feature) { - logf("vk: NVIDIA DLSS: its feature was not made (%#x) for %ux%u -> %ux%u", (unsigned)r, max_w, max_h, ow, oh); + logf("vk: NVIDIA DLSS: its feature was not made (%#x) for %ux%u -> %ux%u", (unsigned)r, made.w, made.h, ow, oh); s.feature = nullptr; // (out of video memory passes, when the game lets an area's textures go: not the end of DLSS for the run, as any // failure was before. Anything else is) @@ -387,13 +452,15 @@ int prepare(VkCommandBuffer cmd, uint32_t rw, uint32_t rh, uint32_t ow, uint32_t return *fail = "its feature was not made", 0; } s.retries = 0, s.retry_frame = 0, s.refused = 0; - s.feature_rw = max_w, s.feature_rh = max_h, s.feature_ow = ow, s.feature_oh = oh, s.min_rw = min_w, s.min_rh = min_h; + s.feature_rw = made.w, s.feature_rh = made.h, s.feature_ow = ow, s.feature_oh = oh, s.min_rw = min_w, s.min_rh = min_h; + s.feature_exact = made.exact; s.feature_preset = preset, s.feature_hdr = hdr; s.features_made++; logf("vk: NVIDIA DLSS: feature %llu made (%s; frame %llu, %.1f ms of CPU): %ux%u (%ux%u to %ux%u) -> %ux%u, mode %d, " - "preset %c%s", (unsigned long long)s.features_made, why, (unsigned long long)texture_frame_no(), made_ms, rw, rh, - min_w, min_h, max_w, max_h, ow, oh, (int)mode, upscaler_rows::preset_letter(preset), - preset ? "" : " (NVIDIA's default)"); + "preset %c%s; made for %ux%u%s", (unsigned long long)s.features_made, why, (unsigned long long)texture_frame_no(), + made_ms, rw, rh, min_w, min_h, max_w, max_h, ow, oh, (int)mode, upscaler_rows::preset_letter(preset), + preset ? "" : " (NVIDIA's default)", made.w, made.h, + made.exact ? ", this render size alone (RX_VK_DLSS_FEATURE_MAX=1: its mode's largest)" : ", the largest of its range"); *in_w = s.feature_rw, *in_h = s.feature_rh; return 2; // (a new history) } @@ -428,6 +495,7 @@ void release() { temporal_later([old] { NVSDK_NGX_VULKAN_ReleaseFeature(old); }); s.feature = nullptr; s.feature_rw = s.feature_rh = s.feature_ow = s.feature_oh = s.min_rw = s.min_rh = 0; + s.feature_exact = false; } uint64_t memory() { @@ -485,4 +553,13 @@ void dlss_device_extensions(std::vector&, const std::vector& want, const std::vector< void dlss_device_extensions(std::vector& want, const std::vector& all); /// DLSS as the temporal upscaler's engine (DLSS and DLAA on the Upscaler row); nullptr in a build without it const TemporalEngine* dlss_engine(); +/// (Triforce, 11-22, the performance audit's PRESENT-36, after Source A's and Source B's) The card's video memory for the +/// DLSS model row's CNN rule (upscaler_rows.h cnn_for_vram: the K row draws with preset E under 7 GB), in bytes; 0 unless +/// RX_VK_DLSS_CNN_VRAM=1 (off by default: the picture is the owner's call). DLSS's preset and the menu's row both ask +/// this, so they agree. In every build +uint64_t dlss_cnn_vram(); } // namespace mkwvk diff --git a/gpu/vk/vk_dlssg.cpp b/gpu/vk/vk_dlssg.cpp index a1f5ae0..3d023c7 100644 --- a/gpu/vk/vk_dlssg.cpp +++ b/gpu/vk/vk_dlssg.cpp @@ -557,16 +557,28 @@ void frame(VkCommandBuffer cmd, uint32_t out_width, uint32_t out_height) { if ((t = token(n))) p_tag(*t, vp, none, 2, (sl::CommandBuffer*)cmd); } if (generate && have) { + // ((Triforce, 11-20) the matrices in the clip space of the depth Streamline is given (FSR's copy, 1 at the near + // plane), not the game's own, whose clip z runs -1 to 1 the other way: dlss_math.h depth_clip_projection says what + // that did while the camera moved. RX_VK_DLSSG_CLIP=0: the game's own matrices, as before, for comparing) + static const bool game_clip = getenv("RX_VK_DLSSG_CLIP") && atoi(getenv("RX_VK_DLSSG_CLIP")) == 0; + float projection[16], clip_to_prev[16]; + if (game_clip) { + std::memcpy(projection, in.projection, sizeof projection); + std::memcpy(clip_to_prev, in.clip_to_prev, sizeof clip_to_prev); + } else { + dlss_math::depth_clip_projection(in.projection, in.camera.flip_depth, projection); + dlss_math::depth_clip_reprojection(in.clip_to_prev, in.camera.flip_depth, clip_to_prev); + } float clip_to_view[16], prev_to_clip[16]; - if (!inverse(in.projection, clip_to_view) || !inverse(in.clip_to_prev, prev_to_clip)) { + if (!inverse(projection, clip_to_view) || !inverse(clip_to_prev, prev_to_clip)) { static int told; if (!told++) logf("vk: dlssg: a frame whose projection has no inverse: no generated frame after it"); return; } sl::Constants c{}; - matrix(c.cameraViewToClip, in.projection); + matrix(c.cameraViewToClip, projection); matrix(c.clipToCameraView, clip_to_view); - matrix(c.clipToPrevClip, in.clip_to_prev); + matrix(c.clipToPrevClip, clip_to_prev); matrix(c.prevClipToClip, prev_to_clip); c.jitterOffset = sl::float2(in.jitter[0], in.jitter[1]); // (the motion is in render pixels: this makes it fractions of the picture) diff --git a/gpu/vk/vk_draw.cpp b/gpu/vk/vk_draw.cpp index ae2502c..825121f 100644 --- a/gpu/vk/vk_draw.cpp +++ b/gpu/vk/vk_draw.cpp @@ -41,6 +41,7 @@ #include "shader_recompiler/backend/spirv/emit_spirv.h" #include "vk_paths.h" +#include "cpu_count.h" // ((Triforce, 11-22, OS-49) the threads to size by: HOS_SIM_CPU #include "vk_draw.h" #include "flat_map.h" // (flat open-addressing maps for the per-draw lookups) #include "aa_rows.h" @@ -57,18 +58,34 @@ #include "vk_shader.h" #include "vk_texture.h" #include "movie_upscale_policy.h" +#include "upscale_off.h" // ((Triforce, 11-22, GPUWORK-20) which network dispatches the plain stretch stands in for) +#include "stub_same_size.h" // ((Triforce, 11-22, GPUWORK-21) a gameplay network that cannot run, at its input's size: upscale_stub) +#include "lab_ab.h" // ((Triforce, 11-22, GPUWORK-20) the lab's "upoff" phases) +#include "occ_lean.h" // ((Triforce, 11-22, DESC-82) the occlusion proxies' census, RX_VK_OCC_LEAN=3) #include "prepare_threads.h" +#include "warm_computes.h" // ((Triforce) the compute pipelines the translation pass builds: only play's on Intel, prepare_run) +#include "index_rebase.h" // ((Triforce, 11-21, DESC-12) an indexed draw's buffer bound at 0, its offset the first index) +#include "predict_stream.h" // ((Triforce, 11-21, PIPE-83) the predicted builder's plan as record numbers) #include "pcache_bound.h" #include "pcache_carry.h" // ((Triforce) the file carried over a stamp change at an update's preparation, capped) #include "pcache_gate.h" #include "pcache_play.h" // ((Triforce) the pipeline cache's play file: cached_create, pcache_play_open, pipeline_cache_save) #include "warm_checkpoint.h" // ((Triforce) the preparation's checkpoint saves of the pipeline cache: PcacheCheckpoints) +#include "precompile_save.h" // ((Triforce, 11-22, PIPE-23) the cache saved before the title after a long precompile: precompile_end_save) #include "mem_parts.h" #include "pipe_release.h" #include "pipe_warm.h" // ((Triforce) a load's pipelines made behind its draws: pipelines_warm) #include "pipe_order.h" // ((Triforce, wp18) and in the order the last sessions' draws first needed them: order_met) #include "gpl_rules.h" #include "lib_check.h" // ((Triforce) whether the prepared stage libraries are still in the cache: lib_check_cold, prepare_run) +#include "prepare_again.h" // ((Triforce) a program cache made again for another GPU, the mark of the part without libraries: prepare_run) +#include "predict_check.h" // ((Triforce, 11-22, PIPE-68) whether the driver still has the predicted pipelines: predict_check_cold) +#include "built_check.h" // ((Triforce, 11-22, PIPE-21) whether the caches still have what .built lists: built_check_cold) +#include "predict_types.h" // ((Triforce, 11-22, PIPE-44) a predicted pipeline built once for its program: predict_first) +#include "driver_faults.h" // ((Triforce, 11-22, PIPE-57) programs the driver faulted on kept out of it: fault_quarantine) +#include "translate_wait.h" // ((Triforce, 11-22, PIPE-30) a bounded wait for a translating program: draw, translate_note) +#include "pcache_libs.h" // ((Triforce, 11-22, PIPE-78) a cache file of libraries alone on small-RAM NVIDIA PCs) +#include "pipe_ahead.h" // ((Triforce, 11-22, PIPE-81, its first step) where first-drawn pipelines stand: order_met) #include "spv_nan_select.h" // ((Triforce) whether make_module rewrites modules on this device, for the stamps: module_patches) #include "spv_fog.h" // ((Triforce) whether make_module patches the fog's module, for the light precompile's record: built_computes_key) #include "pipe_wait.h" // ((Triforce) what a draw does whose pipeline is not ready, NX2navi's rules: draw, graphics_pipeline) @@ -78,6 +95,8 @@ #include "barrier_rules.h" // ((Triforce) RX_VK_DISPATCH_BARRIER: a dispatch right after another) #include "desc_keep.h" // (Triforce: per-binding descriptor reuse, build_descriptors) #include "desc_sig.h" // (Triforce: the whole-draw skip by the signature of a draw's descriptor inputs, draw) +#include "desc_pool_grow.h" // ((Triforce, 11-22, DESC-33) a slot's descriptor pool at half, whole once a recording runs out) +#include "desc_counts.h" // ((Triforce, 11-22, DESC-22) a program's descriptor counts kept with it: build_descriptors) #include "view_keep.h" // (Triforce: texture looks kept through other textures' ends, texture_view_of) #include "mesh_ahead.h" // (Triforce: a run of indirect mesh draws' records copied at its first, RX_VK_MESH_AHEAD: mesh_indirect_convert) #include "checkpoint_kinds.h" // (Triforce: the markers' kinds and what lies after each, checkpoints_report) @@ -153,6 +172,7 @@ const char* const kFineT[] = {"banks", "c1", "user uniform buffers", "storage bu enum FineC { FC_DRAWS, FC_BUILDS, FC_DESCS, FC_WRITES, FC_BANK_CALLS, FC_BANK_GEN, FC_BANK_FILLS, FC_BANK_FILL_BYTES, FC_BANK_SAME, FC_BANK_NEW, FC_BANK_NEW_BYTES, FC_C1_CALLS, FC_UBO_CALLS, FC_UBO_OVERLAY, FC_UBO_STREAM, FC_PUSH_FULL, FC_PUSH_DELTA, FC_PUSH_NONE, FC_PUSH_WRITES, FC_PUSH_DESCS, FC_BIND_PIPE, FC_DYN_FAST, FC_DYN_FULL, FC_DYN_CALLS, FC_VBIND, FC_IBIND, + FC_IBIND_MADE, // ((Triforce, 11-21, DESC-12) of FC_IBIND's lookups, those that called vkCmdBindIndexBuffer) FC_PUSHC, FC_PIPE_FAST, FC_PIPE_MAP, FC_POOL_SETS, FC_POOL_DESCS, FC_SSBO_DESCS, FC_COUNT }; const char* const kFineC[] = {"draws", "descriptor builds", "descriptors", "descriptor writes", "bank calls", "bank kept by its stage's generation", "bank fills", "bank bytes filled", "bank refills equal", @@ -161,6 +181,7 @@ const char* const kFineC[] = {"draws", "descriptor builds", "descriptors", "desc "full pushes", "pushes of changed writes only", "pushes of nothing (all as pushed)", "writes pushed", "descriptors pushed", "pipeline binds", "dynamic state from the draw before", "dynamic state worked out", "dynamic state commands", "vertex stream binds", "index buffer binds", + "index buffer binds made (not on the command buffer already)", "push constants", "pipeline keys from the fast pair", "pipeline keys from the thread's map", "sets allocated (over the push limit)", "descriptors in them", "storage buffer descriptors built"}; static_assert(std::size(kFineT) == FT_COUNT && std::size(kFineC) == FC_COUNT, "every section and count is named"); @@ -218,6 +239,34 @@ void pin_subgroup(VkPipelineShaderStageCreateInfo& st, uint32_t workgroup_x = 0) if (workgroup_x && !(workgroup_x % 32) && dev.vk13.computeFullSubgroups) st.flags |= VK_PIPELINE_SHADER_STAGE_CREATE_REQUIRE_FULL_SUBGROUPS_BIT; } +/// ((Triforce, 11-22, the performance audit's PIPE-52, after Source A's and Source B's) A compute program's stage: held to 32 +/// lanes only where the program can tell how many run beside it (compute_pin.h: a vote, a lane mask, the lane's index, a +/// shuffle, FSWZADD, a matrix multiply, a barrier of Subgroup scope, or shared memory in a workgroup of 32 or fewer). The +/// others get the driver's choice: neither the required size nor the full-subgroups flag. Mesa's Intel driver compiles a +/// pinned program for 32 lanes only, minutes and gigabytes on the large ones. RX_VK_COMPUTE_PIN=all, the default on every +/// other driver: pin_subgroup as it was, nothing asked and nothing counted. All four places that make a compute program's +/// pipeline call this, so the caches see one pipeline; graphics stages and the mesh fallback's stay with pin_subgroup +std::atomic st_pin_pinned{0}, st_pin_free{0}; +void pin_compute(VkPipelineShaderStageCreateInfo& st, const Program& p) { + const compute_pin::Mode mode = compute_pin_mode(); + if (mode == compute_pin::Mode::All) return pin_subgroup(st, p.workgroup[0]); + compute_pin::Facts f; + if (!p.stages.empty()) { + const Shader::Info& i = p.stages[0].info; + f.has_stage = true; + f.vote = i.uses_subgroup_vote, f.mask = i.uses_subgroup_mask, f.lane_index = i.uses_subgroup_invocation_id; + f.shuffles = i.uses_subgroup_shuffles, f.fswzadd = i.uses_fswzadd, f.matrices = i.uses_cooperative_matrix; + f.subgroup_barrier = !compute_lanes_free(st.module); // (the stage's module, set by every caller before this) + f.shared_memory = p.shared_memory; + f.invocations = (uint64_t)p.workgroup[0] * p.workgroup[1] * p.workgroup[2]; + } + if (!compute_pin::needed(mode, f)) { + st_pin_free++; + return; + } + st_pin_pinned++; + pin_subgroup(st, p.workgroup[0]); +} inline uint64_t mix(uint64_t h, uint64_t v) { h ^= v + 0x9e3779b97f4a7c15ull + (h << 6) + (h >> 2); @@ -419,6 +468,11 @@ thread_local uint64_t rec_value = 0; struct RecordingRes { VkDescriptorPool pool = VK_NULL_HANDLE; std::vector extra; // (pools added when the first ran out) + // ((Triforce, 11-22, the performance audit's DESC-33, after Source A's) the pool's size as a shift (1: half, 0: whole: + // desc_pool_grow.h), and what this recording took of the slot's pools (pool_took), for the "recording streams" line + uint32_t shift = 0; + desc_pool_grow::Took took; + bool reset_ahead = false; // ((Triforce, 11-21, REPLAY-04) its reset was made at the recording's start already: draw_slot_reset) // (the stream in use: the first, or one a recording that outgrew it was moved to. Descriptors written before the move hold // the old buffer's range, so the old ones stay until the slot's next recording, which the GPU's use of this one is done by) VkBuffer stream = VK_NULL_HANDLE; @@ -445,23 +499,41 @@ std::atomic stream_peak{0}, stream_extra{0}, stream_grown{0}; std::deque res; // (a deque: a recording thread keeps its own in place while another is added) thread_local RecordingRes* cur = nullptr; -VkDescriptorPool make_pool() { +// ((Triforce, 11-22, the performance audit's DESC-33, after Source A's; desc_pool_grow.h) A slot's own pool is made at half +// size and made whole at the first reset after a recording ran out of it (slot_reset_locked). RX_VK_DESC_POOL_GROW=0: whole +// from the start, as before; read once. What the recordings took and how many pools that cost are counted for the "recording +// streams" line, under ParLock: the most one recording took, the extra pools made, the slots made whole) +bool pool_grow_on() { + static const bool on = desc_pool_grow::on(getenv("RX_VK_DESC_POOL_GROW")); + return on; +} +desc_pool_grow::Took pool_most; +uint64_t pools_added = 0, pools_whole = 0; + +VkDescriptorPool make_pool(uint32_t shift = 0) { // (each recording's pool: 8192 sets of up to 32768 descriptors a type, for the programs with more descriptors than the push limit, // 24 or 48 pools at once, and the driver may keep host memory for all of that. A pool that runs out is followed by another of the same // size (cur->extra), so a smaller one is safe, and costs a pool made and destroyed per recording in a scene that needs more. - // RX_VK_DESC_POOL_DIV= (1..32): a n-th of both) + // RX_VK_DESC_POOL_DIV= (1..32): a n-th of both. (Triforce, 11-22, DESC-33) shift 1: half of both, a slot's first pool; + // the extra pools and a slot's pool once it ran out are whole, shift 0) static const uint32_t div = getenv("RX_VK_DESC_POOL_DIV") ? (uint32_t)std::clamp(atoi(getenv("RX_VK_DESC_POOL_DIV")), 1, 32) : 1; + namespace grow = desc_pool_grow; + const auto of = [&](int kind) { return grow::count(grow::kWhole[kind], shift, div); }; const VkDescriptorPoolSize sizes[] = { - {VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 32768 / div}, {VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 16384 / div}, - {VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 32768 / div}, {VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 4096 / div}, - {VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER, 2048 / div}, {VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER, 2048 / div}}; + {VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, of(grow::kUniform)}, {VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, of(grow::kStorage)}, + {VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, of(grow::kSampled)}, {VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, of(grow::kStorageImage)}, + {VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER, of(grow::kUniformTexel)}, {VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER, of(grow::kStorageTexel)}}; VkDescriptorPoolCreateInfo ci{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; - ci.maxSets = 8192 / div; + ci.maxSets = grow::count(grow::kSets, shift, div); ci.poolSizeCount = (uint32_t)std::size(sizes); ci.pPoolSizes = sizes; VkDescriptorPool p = VK_NULL_HANDLE; return vkCreateDescriptorPool(dev.device, &ci, nullptr, &p) == VK_SUCCESS ? p : VK_NULL_HANDLE; } +/// ((Triforce, 11-22, DESC-33) One more set allocated in the recording's pools, with these writes: counted for the peaks) +void pool_took(const VkWriteDescriptorSet* writes, size_t n) { + if (cur) desc_pool_grow::took(cur->took, writes, n); +} constexpr VkBufferUsageFlags kStreamUse = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT | @@ -792,6 +864,16 @@ struct PcachePreparing { // (prepare_run, for its length) PcachePreparing() { pcache_preparing++; } ~PcachePreparing() { pcache_preparing--; } }; +// ((Triforce, 11-22, the performance audit's PIPE-23, after Source A's; precompile_save.h) RX_VK_PRECOMPILE_SAVE_S: the +// seconds of building from which a keep-all precompile's end saves the cache before the title (30; 0: never, and the +// preparation's compute pipelines are neither counted for the main file nor saved at its end, as before). Read once. +// pcache_play_only_now: that save, on its caller's thread (a forced save runs there): the play file alone while there is +// a play cache, pipeline_cache_save) +int precompile_save_s() { + static const int s = precompile_save::after_s(getenv("RX_VK_PRECOMPILE_SAVE_S")); + return s; +} +thread_local bool pcache_play_only_now = false; // ---- (Triforce) The carry's state and its rules in use: tests/pipeline_cache_save_test.cpp builds this part as it is ---- // ((Triforce) The carry over a stamp change (pcache_carry.h has why and the rules): the session's history of the file (its // base and carries, written beside it as .base at each save), whether this session read a file of another stamp @@ -959,6 +1041,60 @@ void pcache_carry_close() { file_writers::write_whole(pcache_path + ".base", out, sizeof out); } // ---- (Triforce) The carry's part ends ---- +/// (hookshot) Whether a pipeline cache's data (its header, the first 32 bytes, and its length) is this device's and driver's: +/// the header's size, version, vendor, device and pipelineCacheUUID. ((Triforce) asked of the file's header before the rest of +/// it is read, pcache_bound::read, and of the data read, as before) +bool pcache_head_ok(const uint8_t* head, uint64_t size) { + if (size < 32) return false; + uint32_t head_size, version, vendor, device; + std::memcpy(&head_size, head, 4); + std::memcpy(&version, head + 4, 4); + std::memcpy(&vendor, head + 8, 4); + std::memcpy(&device, head + 12, 4); + return head_size >= 32 && head_size <= size && version == VK_PIPELINE_CACHE_HEADER_VERSION_ONE && + vendor == dev.props.vendorID && device == dev.props.deviceID && + !std::memcmp(head + 16, dev.props.pipelineCacheUUID, VK_UUID_SIZE); +} +// ((Triforce, 11-22, the performance audit's PIPE-78, after Source A's and Source B's; pcache_libs.h) RX_VK_PCACHE_LIBS_ONLY=1: +// on NVIDIA's own driver with 20 GB of RAM or less and 512 MB or more in its own cache, a fresh pipeline cache file keeps the +// stage libraries and the compute pipelines alone: whole graphics pipelines are left to the driver's cache (build_graphics), +// and there is no play file. Decided when a fresh file begins, kept in .lean for that file's life. Off unless set) +bool pcache_libs_only = false; +bool gpl_wanted(); +bool gpl_libs_cached(); +/// pcache_init's: fresh, no file read for these shaders (stamp); ram, the PC's. True: lean for this session +bool pcache_libs_decide(bool fresh, const char* stamp, uint64_t ram) { + const int mode = pcache_libs::mode(getenv("RX_VK_PCACHE_LIBS_ONLY")); + if (!mode) return false; + const std::string lean = pcache_path + ".lean"; + if (!fresh || prepare_only_start()) { // (a file read stays as it began; a start that only prepares decides nothing) + const bool recorded = !fresh && pcache_libs::lean_read(lean, stamp); + if (recorded) logf("vk: pipeline cache: libraries only (this file began so, %s; RX_VK_PCACHE_LIBS_ONLY)", lean.c_str()); + return pcache_libs::lean_now(mode, fresh, false, recorded); + } + pcache_libs::Pc pc; + pc.nvidia_proprietary = dev.driver_id == VK_DRIVER_ID_NVIDIA_PROPRIETARY; + pc.ram = ram; + pc.gpl = gpl_wanted() && gpl_libs_cached(); + pc.prepare_only = false; + if (pc.nvidia_proprietary || mode >= 2) { // (the driver's folder walked only where it can matter) +#ifdef _WIN32 + pc.glcache = pcache_libs::folder_bytes(pcache_libs::nvidia_cache_dir(true, getenv("LOCALAPPDATA"), nullptr, nullptr, nullptr)); +#else + pc.glcache = pcache_libs::folder_bytes(pcache_libs::nvidia_cache_dir(false, nullptr, getenv("__GL_SHADER_DISK_CACHE_PATH"), + getenv("XDG_CACHE_HOME"), getenv("HOME"))); +#endif + } + const bool on = pcache_libs::choose(mode, pc); + if (on) pcache_libs::lean_write(lean, stamp); + else remove(lean.c_str()); + logf("vk: pipeline cache: %s (%s driver, %llu MB of RAM, the driver's own cache %llu MB%s; RX_VK_PCACHE_LIBS_ONLY=%d)", + on ? "libraries only: whole graphics pipelines are left to the driver's own cache, no play file" + : "every pipeline in the file, as before", + pc.nvidia_proprietary ? "NVIDIA's" : "not NVIDIA's", (unsigned long long)(ram >> 20), (unsigned long long)(pc.glcache >> 20), + pc.gpl ? "" : ", no stage libraries kept", mode); + return on; +} void pcache_init() { if (pcache) return; uint64_t ws0 = 0, priv0 = 0; // ((Triforce) what the cache costs the RAM once the driver has it: the line below) @@ -1069,8 +1205,12 @@ void pcache_init() { } else { // (whole or not at all: a file over the bound, or one whose read ends early, leaves data empty) uint64_t len = 0; - const pcache_bound::Read got = pcache_bound::read(pcache_path.c_str(), 32, pcache_bound_bytes, data, &len); - if (got == pcache_bound::Read::OverBound) + const pcache_bound::Read got = pcache_bound::read(pcache_path.c_str(), 32, pcache_bound_bytes, data, &len, + pcache_head_ok); + if (got == pcache_bound::Read::Foreign) + logf("vk: pipeline cache %s (%llu MB) is another GPU's or driver's: not read past its header; the cache starts " + "empty and the next save writes a new file over it", pcache_path.c_str(), (unsigned long long)(len >> 20)); + else if (got == pcache_bound::Read::OverBound) logf("vk: pipeline cache %s (%llu MB) is over the bound of %llu MB: not read; the cache starts empty and the next " "save writes a new file over it (RX_VK_PCACHE_MAX_MB= sets the bound, 0 lifts it)", pcache_path.c_str(), (unsigned long long)(len >> 20), (unsigned long long)(pcache_bound_bytes >> 20)); @@ -1080,17 +1220,7 @@ void pcache_init() { } } // (data of another device or driver is dropped before Vulkan sees it) - if (data.size() >= 32) { - uint32_t head_size, version, vendor, device; - std::memcpy(&head_size, data.data(), 4); - std::memcpy(&version, data.data() + 4, 4); - std::memcpy(&vendor, data.data() + 8, 4); - std::memcpy(&device, data.data() + 12, 4); - if (head_size < 32 || head_size > data.size() || version != VK_PIPELINE_CACHE_HEADER_VERSION_ONE || // (hookshot) - vendor != dev.props.vendorID || device != dev.props.deviceID || - std::memcmp(data.data() + 16, dev.props.pipelineCacheUUID, VK_UUID_SIZE)) - data.clear(); - } + if (data.size() >= 32 && !pcache_head_ok(data.data(), data.size())) data.clear(); VkPipelineCacheCreateInfo ci{VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO}; ci.initialDataSize = data.size(); ci.pInitialData = data.empty() ? nullptr : data.data(); @@ -1108,6 +1238,8 @@ void pcache_init() { pcache_side = pcache_carry::session(how, history); if (with_data && carried) pcache_side.carried++; // (pcache_bound.h carried: one carry more, the base as it was) } + // ((Triforce, 11-22, PIPE-78) a fresh file may begin lean; a file read stays as it began: pcache_libs_decide) + pcache_libs_only = pcache_libs_decide(!with_data, pcache_for, ram); pcache_from_file = pcache && ci.initialDataSize > 0; // ((Triforce) and whether that file holds only what this driver's part is made for: .own, pcache_own_mark) pcache_from_own = pcache_from_file && !pcache_carried && pcache_own_named(); @@ -1141,9 +1273,10 @@ void pcache_init() { void pcache_play_open(bool same) { pcache_play_path = pcache_path.substr(0, pcache_path.size() - 4) + "_play.bin"; static const bool on = !(getenv("RX_VK_PCACHE_PLAY_FILE") && atoi(getenv("RX_VK_PCACHE_PLAY_FILE")) == 0); - if (!on || !pcache || !dev.vk13.pipelineCreationCacheControl) { + if (!on || !pcache || !dev.vk13.pipelineCreationCacheControl || pcache_libs_only) { // ((Triforce, 11-22, PIPE-78) lean: none) logf("vk: pipeline cache: no play file (%s): what a run compiles goes into the cache above", - !on ? "RX_VK_PCACHE_PLAY_FILE=0" : "no pipelineCreationCacheControl on this device"); + !on ? "RX_VK_PCACHE_PLAY_FILE=0" : pcache_libs_only ? "libraries only, RX_VK_PCACHE_LIBS_ONLY" + : "no pipelineCreationCacheControl on this device"); return; } char stamp[20]; @@ -1157,21 +1290,14 @@ void pcache_play_open(bool same) { } std::vector data; uint64_t len = 0; - if (for_these && pcache_bound::read(pcache_play_path.c_str(), 32, pcache_bound_bytes, data, &len) != pcache_bound::Read::Ok) - data.clear(); - if (data.size() >= 32) { // (another device's or driver's data is dropped, as the main file's) - uint32_t head_size, version, vendor, device; - std::memcpy(&head_size, data.data(), 4); - std::memcpy(&version, data.data() + 4, 4); - std::memcpy(&vendor, data.data() + 8, 4); - std::memcpy(&device, data.data() + 12, 4); - if (head_size < 32 || head_size > data.size() || version != VK_PIPELINE_CACHE_HEADER_VERSION_ONE || - vendor != dev.props.vendorID || device != dev.props.deviceID || - std::memcmp(data.data() + 16, dev.props.pipelineCacheUUID, VK_UUID_SIZE)) - data.clear(); - } else { - data.clear(); - } + const pcache_bound::Read got = for_these + ? pcache_bound::read(pcache_play_path.c_str(), 32, pcache_bound_bytes, data, &len, pcache_head_ok) : pcache_bound::Read::None; + if (got == pcache_bound::Read::Foreign) + logf("vk: pipeline cache %s (%llu MB) is another GPU's or driver's: not read past its header", pcache_play_path.c_str(), + (unsigned long long)(len >> 20)); + if (got != pcache_bound::Read::Ok) data.clear(); + // (another device's or driver's data is dropped, as the main file's) + if (data.size() < 32 || !pcache_head_ok(data.data(), data.size())) data.clear(); VkPipelineCacheCreateInfo ci{VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO}; ci.initialDataSize = data.size(); ci.pInitialData = data.empty() ? nullptr : data.data(); @@ -1294,6 +1420,70 @@ Targets rate_targets() { return t; } +// ---- (Triforce, 11-22, the performance audit's PIPE-57, after Source A's; driver_faults.h) On AMD's Windows driver, a program +// the driver faulted on is kept out of it, this run and (by triforce_driver_faults.txt) the next ones with that driver, and one +// build of a program is inside the driver at a time. fault_key: the program whose build this thread has inside the driver now, +// where that is on (0 otherwise: the guards are then as before); driver_fault_hit: the guard caught a fault in it ---- +thread_local uint64_t fault_key = 0; +thread_local bool driver_fault_hit = false; +uint32_t fault_vendor = 0, fault_driver = 0; // (the file's lines are this vendor's and this driver version's) +#ifdef _WIN32 +wchar_t fault_file_w[MAX_PATH * 2] = L""; // (the file's path, made at its load: the fault's handler only writes it) +/// The fault's handler's: the program's line appended to the file with the system's own calls, which take no lock of the +/// C library's that another thread may hold, before driver_fault_restart can end the process +void driver_fault_file_note() { + if (!fault_key || !fault_file_w[0]) return; + char l[driver_faults::kLine + 1]; + const int n = driver_faults::line(l, fault_vendor, fault_driver, fault_key); + const HANDLE h = CreateFileW(fault_file_w, FILE_APPEND_DATA, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (h == INVALID_HANDLE_VALUE) return; + DWORD wrote = 0; + WriteFile(h, l, (DWORD)n, &wrote, nullptr); + FlushFileBuffers(h); + CloseHandle(h); +} +#endif +int driver_faults_mode() { // (RX_VK_DRIVER_FAULTS, read once: 0 off, 1 AMD's Windows driver, 2 every driver) + static const int mode = driver_faults::mode(getenv("RX_VK_DRIVER_FAULTS")); + return mode; +} +bool driver_faults_on() { +#ifdef _WIN32 + static const bool on = driver_faults::on(driver_faults_mode(), true, dev.driver_id == VK_DRIVER_ID_AMD_PROPRIETARY); +#else + static const bool on = driver_faults::on(driver_faults_mode(), false, dev.driver_id == VK_DRIVER_ID_AMD_PROPRIETARY); +#endif + return on; +} +/// The run's quarantine and gate, the file beside the executable read into it once, and said once +driver_faults::Quarantine& fault_quarantine() { + static driver_faults::Quarantine q; + static std::once_flag once; + std::call_once(once, [] { + char exe[MAX_PATH]; + const DWORD n = game_module_path(exe, MAX_PATH); + const std::string dir(exe, n); + const std::string path = dir.substr(0, dir.find_last_of("\\/") + 1) + "triforce_driver_faults.txt"; + fault_vendor = dev.props.vendorID, fault_driver = dev.props.driverVersion; + std::string text; + if (FILE* f = fopen(path.c_str(), "rb")) { + char buf[4096]; + for (size_t got; (got = fread(buf, 1, sizeof buf, f)) > 0;) text.append(buf, got); + fclose(f); + } + const std::vector keys = driver_faults::parse(text, fault_vendor, fault_driver); + q.load(keys); +#ifdef _WIN32 + if (!MultiByteToWideChar(CP_UTF8, 0, path.c_str(), -1, fault_file_w, MAX_PATH * 2)) fault_file_w[0] = 0; +#endif + logf("vk: driver faults: %zu programs the driver faulted on before are kept out of it (%s, lines of this driver; delete " + "it to try them again); one build of a program inside the driver at a time (RX_VK_DRIVER_FAULTS=0: neither)", + keys.size(), path.c_str()); + }); + return q; +} + // (hookshot) A driver's compiler that faults on a shader (amdvlk64.dll on a mesh program, Hookshot issue 26) costs that one draw, not the game #ifdef _WIN32 // (Triforce, 11-13) Where the driver faulted: the module and offset of the faulting instruction, what it touched, and the @@ -1404,6 +1594,9 @@ __declspec(noinline) VkResult guarded_graphics(VkPipelineCache cache, const VkGr __try { return vkCreateGraphicsPipelines(dev.device, cache, 1, ci, nullptr, pl); } __except (driver_fault_note(GetExceptionInformation(), "graphics")) { + // ((Triforce, 11-22, PIPE-57) where the quarantine is on: no handle the driver half wrote is kept, and the program's + // line is in the file before a restart can end the process) + if (fault_key) *pl = VK_NULL_HANDLE, driver_fault_hit = true, driver_fault_file_note(); driver_fault_restart(); return VK_ERROR_UNKNOWN; } @@ -1502,11 +1695,79 @@ VkResult cached_compute(VkPipelineCache cache, const VkComputePipelineCreateInfo } VkPipeline build_graphics(const Program& p, const NvnState& s, const Targets& at, VkPrimitiveTopology topo, int part = -1, bool* cold = nullptr); +/// (Triforce, 11-21) RX_VK_MESH_OPT: -1 unset, 0 for "0", 1 for anything else +int mesh_opt_switch() { + static const int v = [] { + const char* e = getenv("RX_VK_MESH_OPT"); + return !e ? -1 : !strcmp(e, "0") ? 0 : 1; + }(); + return v; +} +/// Whether task and mesh pipelines are built without the driver's optimizer (build_graphics): on AMD, but for RADV (unset), on +/// every AMD driver (=0), nowhere (=1) +bool mesh_unoptimized() { + const int v = mesh_opt_switch(); + return dev.props.vendorID == 0x1002 && (v == 0 || (v < 0 && dev.driver_id != VK_DRIVER_ID_MESA_RADV)); +} +/// Whether the RADV rule above builds native task and mesh pipelines with the optimizer that a build before it made without: +/// the driver's part's stamp word kMeshOptRecipe (warmed_stamp_of) +bool mesh_opt_recipe() { + return dev.props.vendorID == 0x1002 && dev.driver_id == VK_DRIVER_ID_MESA_RADV && mesh_opt_switch() < 0 && dev.mesh_shader && + !dev.task_mesh_fallback; +} bool gpl_libs_cached() { static const bool on = !getenv("RX_VK_GPL_NO_CACHE"); return on; } thread_local bool driver_cache_only = false; // (predict_run's: pipelines for the driver's own cache, not into ours) +// ((Triforce, 11-22, the performance audit's PIPE-68, after Source B's; predict_check.h) RX_VK_PREDICT_CHECK, read once: 0 off, +// 1 on, 2 every start. And the check's counter: the builds in play of programs the plan predicts that took over 20 ms, which +// a driver that had kept them would not have compiled. predict_plan_keys: the plan's programs, sorted, set once by the +// builder's thread when it has read the plan (predict_plan_keys_publish) and never freed; until then nothing is counted) +int predict_check_mode() { + static const int mode = predict_check::mode(getenv("RX_VK_PREDICT_CHECK")); + return mode; +} +std::atomic*> predict_plan_keys{nullptr}; +std::atomic st_predicted_slow{0}, st_predicted_slow_us{0}; +void predicted_play_build(uint64_t prog, int64_t us) { + if (!predict_check::slow(us)) return; + const std::vector* keys = predict_plan_keys.load(std::memory_order_acquire); + if (!keys || !predict_check::has(*keys, prog)) return; + const int64_t n = st_predicted_slow.fetch_add(1, std::memory_order_relaxed) + 1; + const int64_t all = st_predicted_slow_us.fetch_add(us, std::memory_order_relaxed) + us; + static std::atomic said_ms{0}; + const int64_t now = std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); + int64_t said = said_ms.load(std::memory_order_relaxed); + if (predict_check::say(now, said) && said_ms.compare_exchange_strong(said, now)) + logf("vk: predicted pipelines: %lld builds in play of predicted programs took over 20 ms so far, %.1f s in all: the driver " + "compiled them, it did not have them (RX_VK_PREDICT_CHECK)", (long long)n, all / 1e6); +} +// ((Triforce, 11-22, the performance audit's PIPE-21, after Source A's and Source B's; built_check.h) RX_VK_BUILT_CHECK, read +// once: 0 off, 1 on, 2 always cold. And its counter: builds in play of pipelines the light precompile's record (.built) +// says are in the caches that took 40 ms or more, and recorded compute programs the first dispatch's ask did not find. +// built_play_keys: the record's keys and its compute programs, sorted, set once at the end of a light precompile that went by +// the record (built_play_publish) and never freed; until then, and in a precompile that keeps everything, nothing is counted) +int built_check_mode() { + static const int mode = built_check::mode(getenv("RX_VK_BUILT_CHECK")); + return mode; +} +std::atomic*> built_play_keys{nullptr}; +std::atomic st_built_slow{0}, st_built_slow_us{0}, st_built_missed{0}; +void built_play_build(uint64_t key, int64_t us, bool compute) { + const std::vector* keys = built_play_keys.load(std::memory_order_acquire); + if (!keys || !built_check::has(*keys, key)) return; + if (compute) st_built_missed.fetch_add(1, std::memory_order_relaxed); + else st_built_slow.fetch_add(1, std::memory_order_relaxed), st_built_slow_us.fetch_add(us, std::memory_order_relaxed); + static std::atomic said_ms{0}; + const int64_t now = std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); + int64_t said = said_ms.load(std::memory_order_relaxed); + if (built_check::say(now, said) && said_ms.compare_exchange_strong(said, now)) + logf("vk: precompile's record: in play so far, %lld pipelines it says are in the caches took 40 ms or more to build " + "(%.1f s in all), and %lld compute programs it lists were not in them at their first dispatch: the caches lost " + "them since (RX_VK_BUILT_CHECK)", (long long)st_built_slow.load(std::memory_order_relaxed), + st_built_slow_us.load(std::memory_order_relaxed) / 1e6, (long long)st_built_missed.load(std::memory_order_relaxed)); +} // (lab, 11-13) RX_VK_TEST_DRIVER_FAULT=[,]: the n-th predicted pipeline is handed to the driver broken, so that the // driver itself faults inside vkCreateGraphicsPipelines as it did in the lab on 11-13: what a start does after that can then be // watched at will. Mode 1 (default): the stages' pointer names no memory; mode 2: the first stage's module is no module @@ -1537,6 +1798,51 @@ uint32_t target_write_mask(const Program& p, const NvnState& s, int i) { const uint32_t written = fs->info.frag_color_components[i]; return (s.chmask.mask >> (4 * i)) & 0xF & (all || !written ? 0xFu : (uint32_t)written); } +// ((Triforce, 11-22, the performance audit's DESC-82, after Source B's: its first step) RX_VK_OCC_LEAN=3: the census of the +// draws an occlusion query counts that write no colour, by what would refuse them a pipeline without the fragment stage +// (occ_lean.h). Only counted and logged: no pipeline, key or descriptor changes. Read once; unset, off +} // namespace (the switch is vk_draw.h's, for status_tick's line) +bool occ_lean_census_on() { + static const bool on = [] { + const char* e = getenv("RX_VK_OCC_LEAN"); + const bool v = occ_lean::mode_of(e) == 3; + if (v) logf("vk: the occlusion proxies' census (RX_VK_OCC_LEAN=3): the colour-off counted draws by class, a line every 600 presents"); + else if (e && *e && atoi(e) != 0) + logf("vk: RX_VK_OCC_LEAN=%s: only the census (3) is built; the lean pipelines and their check are not: off", e); + return v; + }(); + return on; +} +namespace { +/// (draw(), the census on) a colour-off draw an open query counts: its class and its fragment stage's descriptors +void occ_lean_note(const Program& p, const NvnState& s) { + const Stage* fs = nullptr; + for (const Stage& st : p.stages) + if (st.vk_stage == VK_SHADER_STAGE_FRAGMENT_BIT) fs = &st; + occ_lean::Facts f; + f.has_fragment = fs != nullptr; + uint32_t descriptors = 0; + if (fs) { + const Shader::Info& i = fs->info; + f.depth_out = i.stores_frag_depth, f.sample_mask = i.stores_sample_mask, f.demote = i.uses_demote_to_helper_invocation; + f.stores = i.stores_global_memory; + for (const auto& d : i.storage_buffers_descriptors) f.stores = f.stores || d.is_written, descriptors += d.count; + f.image_writes = i.uses_typeless_image_writes; + for (const auto& d : i.image_descriptors) f.image_writes = f.image_writes || d.is_written, descriptors += d.count; + for (const auto& d : i.image_buffer_descriptors) f.image_writes = f.image_writes || d.is_written, descriptors += d.count; + f.atomics = i.uses_global_increment || i.uses_global_decrement || i.uses_atomic_f32_add || i.uses_atomic_f16x2_add || + i.uses_atomic_f16x2_min || i.uses_atomic_f16x2_max || i.uses_atomic_f32x2_add || i.uses_atomic_f32x2_min || + i.uses_atomic_f32x2_max || i.uses_atomic_s32_min || i.uses_atomic_s32_max || i.uses_int64_bit_atomics || + i.uses_atomic_image_u32; + f.global = i.uses_global_memory; + for (const auto& d : i.constant_buffer_descriptors) descriptors += d.count; + for (const auto& d : i.texture_descriptors) descriptors += d.count; + for (const auto& d : i.texture_buffer_descriptors) descriptors += d.count; + } + f.a2c = s.ms.a2c != 0; + f.alpha_test = s.color.alpha_func != 0 && s.color.alpha_func != NVN_ALPHA_FUNC_ALWAYS; // (not emulated: refused anyway) + occ_lean::census.note(occ_lean::classify(f), descriptors); +} thread_local const Program* cur_program = nullptr; // (the draw's, for its dynamic write masks) /// (hookshot) What target_write_mask reads of a program, as one word: whether its fragment stage stores each output and which /// components it writes of it (6 bits each). Two programs with the same word make the same masks from the same state @@ -1873,10 +2179,13 @@ bool state_shipped_new() { /// waits; normal in a start that only prepares (HOOKSHOT_PREPARE_ONLY: Setup's build and update, --prepare-shaders) whose /// process was started below normal priority, where nothing of the game runs and a normal thread is still below every /// program of normal priority. Only where that is known: on Windows a process of the below normal or idle class (the Windows -/// builders start their build tools so: Runner.cs, builder/py core.py BELOW_NORMAL_PRIORITY_CLASS), elsewhere a process with +/// builders start their build tools so only with HOOKSHOT_LOW_PRIORITY=1 since the performance audit's OS-57: Runner.cs, +/// builder/py core.py build_priority; by default the tools are of the normal class, and these workers then stay below +/// normal in it, base 7 where they had base 6, still under every program of normal priority), elsewhere a process with /// a nice value above 0 (the Linux builder sets none, and there the posix SetThreadPriority leaves a normal thread at the /// process's nice). Any other process keeps them below normal: a process of normal priority (a --prepare-shaders from a -/// shell) would have them compete with the desktop for the whole preparation, and in one above normal a normal thread is +/// shell, a builder's default start) would have them compete with the desktop for the whole preparation, and in one +/// above normal a normal thread is /// above every other program (prepare_threads.h: "on all of them at High the driver's compiles froze the whole PC"). As the /// process is at the first call int prepare_priority() { @@ -1967,7 +2276,7 @@ struct PrepTimes { /// first well under 100 while the second is too, the workers waited (on one another, on a lock) rather than compiled std::string cpu_text(unsigned workers) const { const double wall = us_since(t0) / 1e6; - const unsigned cpus = std::max(1u, std::thread::hardware_concurrency()); + const unsigned cpus = std::max(1u, cpu_count()); const uint64_t p1 = process_cpu_us(), used = p1 > process0 ? p1 - process0 : 0, run = run_us.load(), cpu = cpu_us.load(); char s[320]; snprintf(s, sizeof s, "the %u workers on a processor %.0f%% of the time they ran (%.0f s of CPU in %.0f s), the process " @@ -1983,7 +2292,7 @@ struct PrepTimes { /// keep their own counts unsigned prepare_workers() { static const unsigned n = [] { - const unsigned cpus = std::max(1u, std::thread::hardware_concurrency()); + const unsigned cpus = std::max(1u, cpu_count()); const char* e = getenv("RX_VK_PREPARE_THREADS"); uint64_t ram_free = 0; bool above_normal = false; @@ -1993,7 +2302,11 @@ unsigned prepare_workers() { if (GlobalMemoryStatusEx(&ms)) ram_free = ms.ullAvailPhys; const DWORD cls = GetPriorityClass(GetCurrentProcess()); above_normal = cls == ABOVE_NORMAL_PRIORITY_CLASS || cls == HIGH_PRIORITY_CLASS || cls == REALTIME_PRIORITY_CLASS; + uint32_t heap_signature = 0; + if (const HANDLE heap = GetProcessHeap()) memcpy(&heap_signature, (const char*)heap + 16, sizeof heap_signature); + const char* heap = prepare_threads::heap_name(heap_signature); #else + const char* heap = nullptr; if (FILE* f = fopen("/proc/meminfo", "r")) { char line[128]; unsigned long long kb = 0; @@ -2003,9 +2316,10 @@ unsigned prepare_workers() { } #endif const unsigned v = prepare_threads::count(cpus, ram_free, above_normal, e ? atoi(e) : 0); - logf("vk: the shaders' preparation runs on %u threads (%u processors, %llu MB of RAM free%s%s; " + logf("vk: the shaders' preparation runs on %u threads (%u processors, %llu MB of RAM free%s%s%s%s; " "RX_VK_PREPARE_THREADS=: on n)", v, cpus, (unsigned long long)(ram_free >> 20), - above_normal ? ", the process above normal priority: half" : "", e && atoi(e) > 0 ? ", as RX_VK_PREPARE_THREADS says" : ""); + above_normal ? ", the process above normal priority: half" : "", e && atoi(e) > 0 ? ", as RX_VK_PREPARE_THREADS says" : "", + heap ? ", the process heap: " : "", heap ? heap : ""); return v; }(); return n; @@ -2380,6 +2694,13 @@ std::atomic st_warm_over_budget{0}, st_warm_not_again{0}; // the draws are in the last ones' are pipe_order_mu's (taken alone, or under warm_index_mu while the index is built; the // index's places are not changed after that). RX_VK_PIPELINE_ORDER=0: none, the warm from the notes as before) pipe_order::Held pipe_order_links; // (gpl_mu) +// ((Triforce, 11-21) RX_VK_ORDER_SOFT_BEHIND=1: links behind the draws kept until their room is needed (pipe_order.h Held::soft). +// OFF by default: in the lab (adult tour, then child tour, the same order files each side) it took the links let go as behind +// from 1269 to 3 and the links the draws took from 1536 to 1792, but the pipeline stalls moved 2.0 -> 1.8 s (child) and 2.1 -> +// 2.2 s (adult), the worst 192 -> 319 ms: single tours, inside the noise +const bool pipe_order_soft_set = (pipe_order_links.soft(getenv("RX_VK_ORDER_SOFT_BEHIND") && + atoi(getenv("RX_VK_ORDER_SOFT_BEHIND")) == 1), + true); std::atomic pipe_order_used{false}; std::mutex pipe_order_mu; pipe_order::Index pipe_order_index; // (by_variant: warm_index_mu; place: pipe_order_mu) @@ -2558,12 +2879,26 @@ bool order_wanted(int64_t place) { /// ((Triforce, wp18; the draw's, under the table's lock) A draw needs the pipeline of key for the first time: a new entry, or /// one the precompile made that no draw had. Into this session's order; and where the draws are in the last sessions' moves /// on, the links held far behind them going with the libraries only they held (pipe_order.h) -void order_met(uint64_t key) { +// ((Triforce, 11-22, the performance audit's PIPE-81, after Source B's: its first step; pipe_ahead.h) With the order used, each +// pipeline the draws first need counted by its place against where the draws are: within RX_VK_PIPE_AHEAD places ahead +// (1024), further, behind, not in the order; and those within that the warm had linked. Nothing is built: the count says +// whether a builder of whole pipelines just ahead would be worth it (draw_left_out_take's line). RX_VK_PIPE_AHEAD=0: not counted +int64_t pipe_ahead_window() { + static const int64_t w = pipe_ahead::window(getenv("RX_VK_PIPE_AHEAD")); + return w; +} +std::atomic st_ahead[4] = {}, st_ahead_warm{0}; // (pipe_ahead::Where: none, near, far, behind) +void order_met(uint64_t key, bool warm_linked = false) { if (!pipe_order_on()) return; int64_t from = 0; { std::lock_guard lk(pipe_order_mu); pipe_order_rec.add(key); + if (pipe_order_used.load(std::memory_order_relaxed) && pipe_ahead_window()) { // ((Triforce, 11-22, PIPE-81) before it moves) + const pipe_ahead::Where w = pipe_ahead::where(pipe_order_index.at(key), pipe_order_front.furthest, pipe_ahead_window()); + st_ahead[(int)w].fetch_add(1, std::memory_order_relaxed); + if (w == pipe_ahead::Where::Near && warm_linked) st_ahead_warm.fetch_add(1, std::memory_order_relaxed); + } if (!pipe_order_used.load(std::memory_order_relaxed) || !pipe_order_front.met(pipe_order_index.at(key))) return; from = pipe_order_front.from(); } @@ -2849,6 +3184,31 @@ pipe_wait::Pool& pipe_pool() { // (RX_VK_PIPE_FRAME_WAIT_MS: the full-screen a static pipe_wait::Pool p(pipe_env_ms("RX_VK_PIPE_FRAME_WAIT_MS", pipe_wait::kFrameWaitMs, 500) * pipe_wait::kMs); return p; } +// ((Triforce, 11-22, the performance audit's PIPE-30, after Source A's; translate_wait.h) RX_VK_TRANSLATE_WAIT_MS=, read +// once: a draw into targets that hold an earlier frame whose program is still translating, of a key known to write no memory, +// waits at most that much a frame in all (its own budget), then is left out. Off unless set: every such draw waits, as before) +int translate_wait_ms() { + static const int ms = [] { + const char* e = getenv("RX_VK_TRANSLATE_WAIT_MS"); + const int v = translate_wait::parse(e); + if (e) + logf(v ? "vk: a draw whose program is still translating, into targets that hold an earlier frame and of a program " + "known to write no memory, waits up to %d ms a frame in all, then is left out (RX_VK_TRANSLATE_WAIT_MS)" + : "vk: RX_VK_TRANSLATE_WAIT_MS=%d? 1 to 100 ms, digits only: off, such draws wait as before", v ? v : 0); + return v; + }(); + return ms; +} +pipe_wait::Pool& translate_pool() { + static pipe_wait::Pool p(translate_wait_ms() * pipe_wait::kMs); + return p; +} +translate_wait::Notes& translate_notes() { + static translate_wait::Notes n; + return n; +} +std::atomic st_translate_bounded{0}, st_translate_left{0}; +void translate_note(const Program& p); // (below draw(): whether a translated program writes memory, for the bound) pipe_wait::Pool& pipe_last() { // (a full-screen draw's in a frame the hold cannot take, once the pool is spent) static pipe_wait::Pool p(pipe_wait::kLastWaitMs * pipe_wait::kMs); return p; @@ -2877,6 +3237,30 @@ const pipe_wait::Rules& pipe_rules() { }(); return r; } +// ((Triforce, 11-22, the performance audit's PIPE-29, after Source B's; pipe_wait.h depth_new) RX_VK_PIPE_DEPTH_BUDGET=1: a +// depth-tested draw into a pass drawn once whose shaders write no memory waits from the frame's pool, then is left out; a +// pipeline that ran out of time is left out at once until built (pipe_depth_out, emptied every 600 presents). Off unless set, +// on every driver: the entry's default for Intel's Windows driver waits for the owner's word) +bool pipe_depth_budget() { + static const bool on = [] { + const bool v = pipe_wait::depth_budget(getenv("RX_VK_PIPE_DEPTH_BUDGET")); + if (v) logf("vk: a depth-tested draw into a pass drawn once waits for its pipeline from the frame's %lld ms, then is left out " + "(RX_VK_PIPE_DEPTH_BUDGET)", (long long)(pipe_pool().per_frame() / pipe_wait::kMs)); + return v; + }(); + return on; +} +std::mutex pipe_depth_mu; +std::unordered_set pipe_depth_out; // (pipe_depth_mu: entries whose wait ran out, left out at once) +std::atomic st_pipe_depth_waits{0}, st_pipe_depth_left{0}; +bool pipe_depth_timed_out(const void* pe) { + std::lock_guard lk(pipe_depth_mu); + return pipe_depth_out.count(pe) != 0; +} +void pipe_depth_time_out(const void* pe) { + std::lock_guard lk(pipe_depth_mu); + pipe_depth_out.insert(pe); +} bool pipe_skip_new() { // (RX_VK_PIPE_SKIP_NEW=0: a pass whose depth target alone was drawn before is kept, as before) static const bool on = !(getenv("RX_VK_PIPE_SKIP_NEW") && atoi(getenv("RX_VK_PIPE_SKIP_NEW")) == 0); return on; @@ -3029,6 +3413,7 @@ PipeEntry* graphics_pipeline(const Program& p, const NvnState& s, const Targets& e->looked.store(true, std::memory_order_relaxed); // ((Triforce) the draw's own, from the start: never given back) st_pipe_entries++, st_pipe_looked++; pipeline_note(key, p, s, at, topo, color_types); + translate_note(p); // ((Triforce, 11-22, PIPE-30) whether its key writes memory, for the bounded wait) static const bool sync = getenv("RX_VK_SYNC_PIPELINES") != nullptr; if (sync) { e->pipeline = build_graphics(p, s, at, topo); @@ -3039,10 +3424,12 @@ PipeEntry* graphics_pipeline(const Program& p, const NvnState& s, const Targets& // when a library is cold: the entry stays not ready, and the draw is left out or waits for the full build) uint64_t cold[2]{}; bool link_later = false; // ((Triforce) the frame's links took their budget (pipe_wait.h LinkBudget): a worker links it) + bool warm_linked = false; // ((Triforce, 11-22, PIPE-81) the warm had linked it: counted by order_met) if (gpl_wanted() && !p.fallback) { const auto t0 = std::chrono::steady_clock::now(); // ((Triforce) the one the warm linked for this key behind the draws, in place of one of its own: pipe_warm.h) VkPipeline linked = warm_link_take(key); + warm_linked = linked != VK_NULL_HANDLE; const uint64_t frame = texture_frame_no(); link_later = !linked && !gpl_skipped(p) && pipe_fast_workers() && !pipe_links().links(frame); if (!linked && !link_later) linked = gpl_link(p, s, at, topo, cold), st_links++; @@ -3052,7 +3439,7 @@ PipeEntry* graphics_pipeline(const Program& p, const NvnState& s, const Targets& gpl_rules::linked(*e, linked); // ((Triforce) ready with it, or, not linked, left as it is for the whole build) if (gpl_skipped(p)) cold[0] = cold[1] = 0; // (a start died with this variant's libraries in the driver: none is built) } - order_met(key); // ((Triforce, wp18) the draws' first need, after the link the warm made for it was taken: pipe_order.h) + order_met(key, warm_linked); // ((Triforce, wp18) the draws' first need, after the link the warm made for it was taken: pipe_order.h) struct Job { PipeEntry* e; const Program* p; @@ -3292,20 +3679,19 @@ VkPipeline build_graphics(const Program& p, const NvnState& s, const Targets& at // (from PR 430, YunaraDebugs: on AMD the task and mesh programs (OoT's foliage culling, following pointers through its // tables) hung the GPU when the driver's optimizer built them: a device lost one to four minutes into Kokiri, each time with // another of them first (RX 6600, issues 461 and 484); never in runs where they were built without it. They are built - // without it on AMD. RX_VK_MESH_OPT=1: with it, as the others) - static const bool amd = [] { - VkPhysicalDeviceProperties pr{}; - vkGetPhysicalDeviceProperties(dev.physical, &pr); - return pr.vendorID == 0x1002; - }(); - static const bool mesh_opt = getenv("RX_VK_MESH_OPT") != nullptr; - if (amd && !mesh_opt) { + // without it on AMD's own drivers. (Triforce, 11-21) Not on RADV, where it is the other way round: built without the + // optimizer, the same culling task programs ran long enough to time out the GPU's ring and reset it at the save load (Steam + // Deck OLED, Mesa 26.2; an RX 6800 XT on RADV the same), and with it they did not (mesh_unoptimized). + // RX_VK_MESH_OPT=1: with it everywhere; =0: without it on every AMD driver, RADV too, as before) + if (mesh_unoptimized()) { bool mesh = false; for (const Stage& st : p.stages) mesh |= (st.vk_stage & (VK_SHADER_STAGE_TASK_BIT_EXT | VK_SHADER_STAGE_MESH_BIT_EXT)) != 0; if (mesh) { ci.flags |= VK_PIPELINE_CREATE_DISABLE_OPTIMIZATION_BIT; static std::atomic told{0}; - if (told++ < 1) logf("vk: task and mesh programs built without the driver's optimizer (AMD; RX_VK_MESH_OPT=1: with it)"); + if (told++ < 1) + logf("vk: task and mesh programs built without the driver's optimizer (%s; RX_VK_MESH_OPT=1: with it)", + mesh_opt_switch() == 0 ? "AMD, RX_VK_MESH_OPT=0" : "AMD's own driver, not RADV"); } } } @@ -3323,8 +3709,24 @@ VkPipeline build_graphics(const Program& p, const NvnState& s, const Targets& at // (the two shader libraries go into our cache, as whole pipelines do: asked for under the flag, the driver answers from a cache // it is handed and not from its own on disk, so a library the warm pass built and let go was cold again at its first draw // (measured: 21 of 21). gpl_libs_cached: RX_VK_GPL_NO_CACHE=1 keeps them out, as before) + // ((Triforce, 11-22, the performance audit's PIPE-57, after Source A's; driver_faults.h) On AMD's Windows driver: a program + // the driver faulted on is not asked of it again, and one build of a program at a time: a second waits here, outside the + // driver, and a probe of the caches is told a compile is required. RX_VK_DRIVER_FAULTS=0, and other drivers: as before) + const bool fault_gated = driver_faults_on(); + if (fault_gated) { + const driver_faults::Quarantine::Begin b = fault_quarantine().begin(p.key, cold == nullptr); + if (b == driver_faults::Quarantine::Begin::Busy) return *cold = true, VK_NULL_HANDLE; + if (b == driver_faults::Quarantine::Begin::Faulted) { + static std::atomic told{0}; + if (told++ < 10) + logf("vk: %016llx is kept out of the driver: it faulted in it (RX_VK_DRIVER_FAULTS)", (unsigned long long)p.key); + return VK_NULL_HANDLE; + } + fault_key = p.key, driver_fault_hit = false; + } const bool lib_cached = (part == 1 || part == 2) && gpl_libs_cached(); - const VkPipelineCache wanted = lib_cached || (part < 0 && !driver_cache_only) ? pcache : VK_NULL_HANDLE; + // ((Triforce, 11-22, PIPE-78) a lean file: whole pipelines to the driver's own cache, libraries in ours: pcache_libs.h) + const VkPipelineCache wanted = lib_cached || (part < 0 && !driver_cache_only && !pcache_libs_only) ? pcache : VK_NULL_HANDLE; const PcacheUse use(pcache_door, wanted); // ((Triforce) held over the creation and its retry: none once it is let go) const VkPipelineCache cache = use.cache; test_driver_fault(ci); // (lab: RX_VK_TEST_DRIVER_FAULT) @@ -3332,13 +3734,20 @@ VkPipeline build_graphics(const Program& p, const NvnState& s, const Targets& at VkResult r = cached_graphics(cache, &ci, &pl); // The AMD Windows compiler can reject ordinary graphics pipelines with its optimizer on. if (dev.props.vendorID == 0x1002 && dev.task_mesh_fallback && !p.mesh && part < 0 && r == VK_ERROR_UNKNOWN && - !(ci.flags & VK_PIPELINE_CREATE_DISABLE_OPTIMIZATION_BIT)) { + !(ci.flags & VK_PIPELINE_CREATE_DISABLE_OPTIMIZATION_BIT) && !driver_fault_hit) { // ((Triforce, PIPE-57) not after a fault) ci.flags |= VK_PIPELINE_CREATE_DISABLE_OPTIMIZATION_BIT; pl = VK_NULL_HANDLE; r = cached_graphics(cache, &ci, &pl); if (r == VK_SUCCESS) logf("vk: graphics pipeline for %016llx recovered with optimization disabled", (unsigned long long)p.key); } + if (fault_gated) { // ((Triforce, 11-22, PIPE-57) out of the gate; a fault keeps the program out from now on) + fault_key = 0; + if (fault_quarantine().end(p.key, driver_fault_hit)) + logf("vk: the driver faulted on %016llx: kept out of it from now on, and in triforce_driver_faults.txt for the next " + "starts with this driver (RX_VK_DRIVER_FAULTS)", (unsigned long long)p.key); + driver_fault_hit = false; + } if (cold && r == VK_PIPELINE_COMPILE_REQUIRED) return *cold = true, VK_NULL_HANDLE; if (part >= 0) { if (r != VK_SUCCESS) { @@ -3356,7 +3765,15 @@ VkPipeline build_graphics(const Program& p, const NvnState& s, const Targets& at } st_pipelines++; pcache_built_since_save += !driver_cache_only; - st_pipeline_us += std::chrono::duration_cast(std::chrono::steady_clock::now() - t0).count(); + const int64_t build_us = std::chrono::duration_cast(std::chrono::steady_clock::now() - t0).count(); + st_pipeline_us += build_us; + // ((Triforce, 11-22, PIPE-68) a whole pipeline made for the game, not for the driver's cache: counted when it is a + // predicted program's and took a compile's time. RX_VK_PREDICT_CHECK=0: not) + if (!driver_cache_only && pl && predict_check_mode()) predicted_play_build(p.key, build_us); + // ((Triforce, 11-22, PIPE-21) and when the light precompile's record says it is in the caches: built_play_build. The key + // is made only for a slow build, which is rare) + if (!driver_cache_only && pl && built_check::slow(build_us) && built_play_keys.load(std::memory_order_relaxed)) + built_play_build(pipeline_key(p, s, at, topo), build_us, false); return pl; } @@ -4011,8 +4428,9 @@ VkDescriptorBufferInfo uniform_range(const NvnState& s, const Stage& st, uint32_ // (updates the stream made to this range in this submission or ones still in flight: a copy with them in) // (room first: the overlay copies into the stream's mapping, which a move to a bigger stream in the middle would change) if (stream_reserve(n, align)) - if (const uint64_t at = ub_overlay(u.addr, n, cur->mapped, [&](uint64_t bytes) { return stream_alloc(bytes, align); }); - at != UINT64_MAX) { + // ((Triforce, 11-22, DESC-06) stream_alloc and its alignment as they are: it was a std::function made of a lambda + // at every uniform buffer) + if (const uint64_t at = ub_overlay(u.addr, n, cur->mapped, stream_alloc, align); at != UINT64_MAX) { FINE_N(FC_UBO_OVERLAY, 1); return {cur->stream, at, n}; } @@ -4032,7 +4450,10 @@ VkDescriptorBufferInfo uniform_range(const NvnState& s, const Stage& st, uint32_ return (dev.props.vendorID == 0x8086 || dev.props.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU) && !pools_mirrored(); }(); BufferRef b = snapshot ? BufferRef{} : gpu_buffer_bind(u.addr, n, false); - if (b && b.offset % align == 0) return {b.buffer, b.offset, n}; + // ((Triforce, 11-22, the performance audit's DESC-06, after Source A's) align is a power of two (the device's + // minUniformBufferOffsetAlignment, or 16; stream_reserve asks the same of it): a mask, where it was a division a + // uniform buffer) + if (b && !(b.offset & (align - 1))) return {b.buffer, b.offset, n}; const uint64_t at = stream_alloc(n, align); // (unaligned or outside the pools: copied into the stream) if (at == UINT64_MAX) return bi; FINE_N(FC_UBO_STREAM, 1); @@ -4057,6 +4478,11 @@ struct DescWriter { writes.clear(), bufs.clear(), imgs.clear(), views.clear(); } }; +// ((Triforce, 11-22, the performance audit's DESC-20, after Source A's) dw() is a thread-local pointer read and a null test +// at each use (and a call for the address in a Linux library built with -fPIC): build_descriptors and bind_descriptors take +// the thread's writer once at their top (auto& dwr = dw();) and use that. Sound because a thread has one DescWriter for its +// whole life: made at the first dw(), reset in place (reset()), and nothing ever puts another object behind the pointer. +// draw() keeps its four dw() (tests/test_mesh_fallback.py reads them). tests/test_tls_once.py holds the count) TLS_OBJECT(dw, DescWriter) TLS_OBJECT(draw_sampled, std::vector) // (the textures the descriptors being built sample or store) TLS_OBJECT(gfx_sampled, std::vector) // (the last draw's: kept while its descriptors are reused) @@ -4325,6 +4751,14 @@ uint32_t sampled_handle(const NvnState& s, const Stage& st, const Shader::Textur : handle_of(s, st, d, k); } +/// ((Triforce, 11-22, the performance audit's DESC-22, after Source A's; desc_counts.h) Whether build_descriptors goes by +/// the counts kept with the program: on unless RX_VK_DESC_COUNTS=0 (read once), and not in the lab's "descwalk" phases. +/// (Here, before the part the descriptor reuse's tests cut out whole) +bool desc_counts_on() { + static const bool env = desc_counts::on(getenv("RX_VK_DESC_COUNTS")); + return env && !lab_ab::on("descwalk"); +} + // ---- (Triforce) Per-binding descriptor reuse (desc_keep.h), and what the descriptors' writes come to ---- // (the counts of the frame statistics' "a frame's descriptors" line, desc_report. Each added as a load and a store, not an atomic add: // a few a draw on the replay thread, and the helper threads' racing adds lose a count at worst, as the file's other counters do) @@ -4477,8 +4911,9 @@ VkSampler kept_texture(desc_keep::Prog* kp, size_t j, uint32_t handle, VkImageVi /// keep: a draw's build, through the per-binding reuse. scene: the draw renders the jittered scene (draw_in_scene), its /// textures sampled with the upscaler's LOD bias; a dispatch's never void build_descriptors(const NvnState& s, const Program& p, bool keep = false, bool scene = false) { - dw().reset(); - dw().prog = &p; + auto& dwr = dw(); // ((Triforce, 11-22, DESC-20) once a call: the thread's one DescWriter, reset in place, never swapped (TLS_OBJECT(dw)) + dwr.reset(); + dwr.prog = &p; desc_fp().clear(), desc_fp_dummy = false; // (RX_VK_DESC_SLOTGEN: the samplers' generation is read before the first handle, desc_samp_seen = sampler_registrations_generation(); // (so one registered during the build still forces the next rebuild) desc_destroy_seen = texture_destroy_generation(); // (an eviction retires views behind the slots' backs: it bumps only this) @@ -4487,15 +4922,31 @@ void build_descriptors(const NvnState& s, const Program& p, bool keep = false, b dump_written().clear(); dump_inputs().clear(); size_t nbuf = 0, nimg = 0, nview = 0; - for (const Stage& st : p.stages) { - for (const auto& d : st.info.constant_buffer_descriptors) nbuf += d.count; - for (const auto& d : st.info.storage_buffers_descriptors) nbuf += d.count; - for (const auto& d : st.info.texture_buffer_descriptors) nview += d.count; - for (const auto& d : st.info.image_buffer_descriptors) nview += d.count; - for (const auto& d : st.info.texture_descriptors) nimg += d.count; - for (const auto& d : st.info.image_descriptors) nimg += d.count; + // ((Triforce, 11-22, the performance audit's DESC-22, after Source A's; desc_counts.h) the three sums are the program's, + // fixed once it is translated: kept packed on it (Program::desc_n), stored at its first build once ready and read at + // every later one. The six lists a stage are walked only while the word holds no counts (not counted yet, more than + // 1023 of a kind, the switch off, or the guard at the end said never) + const bool counts_on = desc_counts_on(); + const uint32_t counted = counts_on ? p.desc_n.load(std::memory_order_relaxed) : desc_counts::kUnset; + const bool walked = !desc_counts::unpack(counted, &nbuf, &nimg, &nview); + if (walked) { + for (const Stage& st : p.stages) { + for (const auto& d : st.info.constant_buffer_descriptors) nbuf += d.count; + for (const auto& d : st.info.storage_buffers_descriptors) nbuf += d.count; + for (const auto& d : st.info.texture_buffer_descriptors) nview += d.count; + for (const auto& d : st.info.image_buffer_descriptors) nview += d.count; + for (const auto& d : st.info.texture_descriptors) nimg += d.count; + for (const auto& d : st.info.image_descriptors) nimg += d.count; + } + // (only once it is translated: its stages are filled before ready and never after) + if (desc_counts::stores(counts_on, p.ready.load(std::memory_order_acquire), p.desc_n.load(std::memory_order_relaxed))) + p.desc_n.store(desc_counts::pack(nbuf, nimg, nview), std::memory_order_relaxed); } - dw().bufs.reserve(nbuf), dw().imgs.reserve(nimg), dw().views.reserve(nview); // (stable pointers below) + dwr.bufs.reserve(nbuf), dwr.imgs.reserve(nimg), dwr.views.reserve(nview); // (stable pointers below) + // ((Triforce, 11-22, DESC-22) where the three are now: the guard at the end) + const void* const bufs_at = dwr.bufs.data(); + const void* const imgs_at = dwr.imgs.data(); + const void* const views_at = dwr.views.data(); FINE_N(FC_DESCS, (int64_t)(nbuf + nimg + nview)); // ((Triforce) a draw's descriptors through the per-binding reuse (desc_keep.h), each by its place in this order (jn): kp's // results are this program's. Not a dispatch's (keep: draw() only), a dump's, or the temporal upscaler's second recording) @@ -4516,28 +4967,28 @@ void build_descriptors(const NvnState& s, const Program& p, bool keep = false, b const int ns = st.nvn_stage; for (const auto& d : info.constant_buffer_descriptors) { VkWriteDescriptorSet w = write(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, d.count); - const size_t first = dw().bufs.size(); + const size_t first = dwr.bufs.size(); const size_t jb = jn; jn += d.count; for (uint32_t k = 0; k < d.count; k++) { const auto t0 = slow_desc_timing() ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; - dw().bufs.push_back(kept_uniform(kp, jb + k, s, st, d.index + k)); - if (slow_desc_timing()) slow_descriptor(t0, "uniform buffer", d.index + k, dw().bufs.back().range); + dwr.bufs.push_back(kept_uniform(kp, jb + k, s, st, d.index + k)); + if (slow_desc_timing()) slow_descriptor(t0, "uniform buffer", d.index + k, dwr.bufs.back().range); } - w.pBufferInfo = &dw().bufs[first]; - dw().writes.push_back(w); + w.pBufferInfo = &dwr.bufs[first]; + dwr.writes.push_back(w); } for (const auto& d : info.storage_buffers_descriptors) { FINE_SCOPE(FT_STORAGE); FINE_N(FC_SSBO_DESCS, d.count); VkWriteDescriptorSet w = write(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, d.count); - const size_t first = dw().bufs.size(); + const size_t first = dwr.bufs.size(); const size_t jb = jn; jn += d.count; for (uint32_t k = 0; k < d.count; k++) { const uint32_t off = d.cbuf_offset + k * 16; if (d.count == 1 && d.cbuf_index == 0 && d.cbuf_offset == 0xff0) { // (EvilOOT, RX_VK_MISS_LOG) - dw().bufs.push_back(miss_binding(p.key)); + dwr.bufs.push_back(miss_binding(p.key)); continue; } // (AyyBeeDee20738, PR 138; EvilOOT the same before it, for OoT's culling tables in c4) through the stage's banks as @@ -4553,7 +5004,7 @@ void build_descriptors(const NvnState& s, const Program& p, bool keep = false, b const bool kept_took = ke && desc_keep::take(*ke, desc_keep::kSsbo, addr, size, 0, kn); if (kept_took && desc_keep_mode() != 2) { count_add(st_dk_taken, 1); - dw().bufs.push_back({(VkBuffer)(uintptr_t)ke->out[0], ke->out[1], ke->out[2]}); + dwr.bufs.push_back({(VkBuffer)(uintptr_t)ke->out[0], ke->out[1], ke->out[2]}); continue; } // (EvilOOT) a table entry that is a bare pointer (its size word 0, as OoT's culling setup keeps them): bound up to @@ -4788,10 +5239,10 @@ void build_descriptors(const NvnState& s, const Program& p, bool keep = false, b "%#llx%s", (unsigned long long)p.key, ns, (unsigned)d.cbuf_index, off, (unsigned long long)addr, (unsigned long long)size, d.is_written ? " (written)" : ""); } - dw().bufs.push_back(b ? VkDescriptorBufferInfo{b.buffer, b.offset, n} + dwr.bufs.push_back(b ? VkDescriptorBufferInfo{b.buffer, b.offset, n} : VkDescriptorBufferInfo{dev.null_descriptor ? VK_NULL_HANDLE : zero_buffer, 0, dev.null_descriptor ? VK_WHOLE_SIZE : 16}); - if (ke) buf_keep(*kp, jb + k, *ke, desc_keep::kSsbo, addr, size, kn, kept_took, (bool)b, dw().bufs.back()); + if (ke) buf_keep(*kp, jb + k, *ke, desc_keep::kSsbo, addr, size, kn, kept_took, (bool)b, dwr.bufs.back()); { // (RX_VK_LOG_WRITTEN=1: every storage buffer range a program writes, once, 300 of them) static const bool on = getenv("RX_VK_LOG_WRITTEN") != nullptr; if (on && d.is_written) { @@ -4810,13 +5261,13 @@ void build_descriptors(const NvnState& s, const Program& p, bool keep = false, b if (slow_desc_timing()) slow_descriptor(t1, "stamp", base, n); } } - w.pBufferInfo = &dw().bufs[first]; - dw().writes.push_back(w); + w.pBufferInfo = &dwr.bufs[first]; + dwr.writes.push_back(w); } auto texel_views = [&](const auto& list, VkDescriptorType type, bool storage) { for (const auto& d : list) { VkWriteDescriptorSet w = write(type, d.count); - const size_t first = dw().views.size(); + const size_t first = dwr.views.size(); jn += d.count; // ((Triforce) not kept: desc_keep.h) for (uint32_t k = 0; k < d.count; k++) { const uint32_t handle = handle_of(s, st, d, k); @@ -4829,10 +5280,10 @@ void build_descriptors(const NvnState& s, const Program& p, bool keep = false, b ParLock lk; bv = texel_buffer_view(&t, has_view ? &v : nullptr, storage); } - dw().views.push_back(bv); + dwr.views.push_back(bv); } - w.pTexelBufferView = &dw().views[first]; - dw().writes.push_back(w); + w.pTexelBufferView = &dwr.views[first]; + dwr.writes.push_back(w); } }; texel_views(info.texture_buffer_descriptors, VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER, false); @@ -4840,7 +5291,7 @@ void build_descriptors(const NvnState& s, const Program& p, bool keep = false, b for (const auto& d : info.texture_descriptors) { FINE_SCOPE(FT_TEX); VkWriteDescriptorSet w = write(VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, d.count); - const size_t first = dw().imgs.size(); + const size_t first = dwr.imgs.size(); const VkImageViewType vt = view_type(d.type); const size_t jb = jn; jn += d.count; @@ -4858,7 +5309,7 @@ void build_descriptors(const NvnState& s, const Program& p, bool keep = false, b // position or asks the size of, is made whole: vk_texture.cpp texture_want_whole) if (tex && tex->detail_skip && !tex->whole_asked && shader_reads_exact(stage_exact_key(st), w.dstBinding)) texture_want_whole(tex); - if (dump_on()) dump_input('t', (uint32_t)(dw().imgs.size() - 0), handle, view ? tex : nullptr); + if (dump_on()) dump_input('t', (uint32_t)(dwr.imgs.size() - 0), handle, view ? tex : nullptr); if (view && tex && trace_tex_wanted(tex)) trace_tex_hit = tex; if (log_img_wanted(p.key)) { static std::atomic told{0}; @@ -4896,14 +5347,14 @@ void build_descriptors(const NvnState& s, const Program& p, bool keep = false, b view = dummies.view[(int)vt], st_dummy_textures += handle != 0, st_dummy_known += handle != 0 && view_slot_known; desc_dummies += handle != 0, desc_dummies_known += handle != 0 && view_slot_known; // ((Triforce) desc_sig.h) } - dw().imgs.push_back({smp, view, VK_IMAGE_LAYOUT_GENERAL}); + dwr.imgs.push_back({smp, view, VK_IMAGE_LAYOUT_GENERAL}); } - w.pImageInfo = &dw().imgs[first]; - dw().writes.push_back(w); + w.pImageInfo = &dwr.imgs[first]; + dwr.writes.push_back(w); } for (const auto& d : info.image_descriptors) { VkWriteDescriptorSet w = write(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, d.count); - const size_t first = dw().imgs.size(); + const size_t first = dwr.imgs.size(); const VkImageViewType vt = view_type(d.type); jn += d.count; // ((Triforce) not kept: desc_keep.h) for (uint32_t k = 0; k < d.count; k++) { @@ -4931,14 +5382,27 @@ void build_descriptors(const NvnState& s, const Program& p, bool keep = false, b view ? "view made" : "NO view"); } if (view && tex && d.is_written && (dump_on() || probe_on())) dump_written().push_back({tex, level}); - if (dump_on()) dump_input(d.is_written ? 'W' : 'R', (uint32_t)dw().imgs.size(), handle, view ? tex : nullptr); + if (dump_on()) dump_input(d.is_written ? 'W' : 'R', (uint32_t)dwr.imgs.size(), handle, view ? tex : nullptr); if (!view && !dev.null_descriptor) view = VK_NULL_HANDLE; - dw().imgs.push_back({VK_NULL_HANDLE, view, VK_IMAGE_LAYOUT_GENERAL}); + dwr.imgs.push_back({VK_NULL_HANDLE, view, VK_IMAGE_LAYOUT_GENERAL}); } - w.pImageInfo = &dw().imgs[first]; - dw().writes.push_back(w); + w.pImageInfo = &dwr.imgs[first]; + dwr.writes.push_back(w); } } + // ((Triforce, 11-22, the performance audit's DESC-22; desc_counts.h) the guard: a build that went by kept counts and + // had one of the three move was short of room, and the writes above may point into storage that is gone. That program + // walks from here on (kNever), and this build is made again, walked, before anything reads the writes. It cannot + // happen while the counts are the walk's own; it is said once if it does) + if (desc_counts::moved(walked, bufs_at, dwr.bufs.data(), imgs_at, dwr.imgs.data(), views_at, dwr.views.data())) { + p.desc_n.store(desc_counts::kNever, std::memory_order_relaxed); + static std::atomic told{false}; + if (!told.exchange(true)) + logf("vk: program %016llx: the descriptor counts kept with it (%zu buffers, %zu images, %zu texel views) were short of " + "a build: its builds walk its stages again from here on, and this one is made again (RX_VK_DESC_COUNTS=0: every " + "program's)", (unsigned long long)p.key, nbuf, nimg, nview); + build_descriptors(s, p, keep, scene); + } } /// Binds the descriptors built: pushed, or a set from the recording's pool @@ -4992,8 +5456,25 @@ void desc_report(double frames) { /// (hookshot) The pipeline cache the game's pipelines are kept in across runs (mkw_vk_pipelines.bin), for the back end's own /// pipelines made elsewhere (vk_fsr3.cpp, vk_aa.cpp); none before it is loaded. (Outside the unnamed namespace: vk_fsr3.cpp links /// to it.) (Triforce) The compute pipeline is made here, through the cache's gate: it may be let go in play (pcache_let_go) +/// ((Triforce, 11-22, the performance audit's PIPE-95, after Source A's) And through the play cache, as a dispatch's pipeline +/// is (cached_compute, pcache_play.h): the main cache asked without compiling, a miss compiled into the play cache, a driver +/// fault caught (the pass is then not made, and its feature is off for the session). Made bare into the main cache, FSR 3.1's +/// dozen passes, the anti-aliasing's and the texture converters' grew that cache by a little, which is written in play only +/// after 64 MB of growth, so they waited for the exit's save of the whole main file (650 MB fresh, 2 to 3 GB with the stage +/// libraries); an exit that cut that write short left the old file, and they were compiled again at the next start. +/// RX_VK_PCACHE_OWN_PASSES=0: the bare creation into the main cache, as before (read once). Without a play cache (no +/// pipelineCreationCacheControl, RX_VK_PCACHE_PLAY_FILE=0) only the guard is new VkResult shared_cache_compute(const VkComputePipelineCreateInfo* ci, VkPipeline* out) { - return vkCreateComputePipelines(dev.device, pcache_use().cache, 1, ci, nullptr, out); + static const bool own_passes = pcache_play::own_passes(getenv("RX_VK_PCACHE_OWN_PASSES")); + const auto use = pcache_use(); // (the gate held over the creation, as dispatch's) + if (!own_passes) return vkCreateComputePipelines(dev.device, use.cache, 1, ci, nullptr, out); + return cached_compute(use.cache, ci, out); +} +/// ((Triforce) RX_VK_TEX_PIPES_AHEAD) A start whose shader preparation was skipped at the build runs no precompile_run: the +/// pipeline cache read here and the texture converters' pipelines made through it before the game all the same +void texture_pipelines_unprepared() { + pcache_init(); + texture_pipelines_warm(); } void draw_skip_report(double frames) { skip_report_impl(frames), fine_report(frames), desc_report(frames); } /// (Triforce) how many texture bindings have sampled the dummy so far (a texture with no image yet: left for a later frame, not @@ -5275,12 +5756,13 @@ bool push_write_template(VkCommandBuffer cmd, VkPipelineBindPoint bp, const Prog } bool bind_descriptors(VkCommandBuffer cmd, VkPipelineBindPoint bp, const Program& p) { + auto& dwr = dw(); // ((Triforce, 11-22, DESC-20) once a call: the thread's one DescWriter, reset in place, never swapped (TLS_OBJECT(dw)) if (p.renodx) renodx_bind(cmd, bp, p.layout, p.renodx); if (renodx_tracing.load(std::memory_order_relaxed) && bp == VK_PIPELINE_BIND_POINT_GRAPHICS) renodx_trace_draw(p.frag_crc); // (local, RenoDX port) set 1: its settings and state - if (dw().writes.empty()) return true; + if (dwr.writes.empty()) return true; // A failed sampler is retryable, but cannot be published to either descriptor path. // Check the entire array before updating the pushed-state cache or allocating a set. - for (const auto& w : dw().writes) + for (const auto& w : dwr.writes) if (w.descriptorType == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER) for (uint32_t k = 0; k < w.descriptorCount; k++) if (!w.pImageInfo[k].sampler) return false; @@ -5288,24 +5770,24 @@ bool bind_descriptors(VkCommandBuffer cmd, VkPipelineBindPoint bp, const Program if (p.push) { static const bool whole = getenv("RX_VK_PUSH_ALL") != nullptr; // (A/B: every binding every time) static const bool old_state = getenv("RX_VK_PUSH_STATE_OLD") != nullptr; // (A/B: a vector of words a write, as before) - if (whole || ps.cmd != cmd || ps.layout != p.layout || ps.n != dw().writes.size()) { + if (whole || ps.cmd != cmd || ps.layout != p.layout || ps.n != dwr.writes.size()) { if (g_fine) { - fine_n[FC_PUSH_FULL]++, fine_n[FC_PUSH_WRITES] += (int64_t)dw().writes.size(); - for (const VkWriteDescriptorSet& w : dw().writes) fine_n[FC_PUSH_DESCS] += w.descriptorCount; + fine_n[FC_PUSH_FULL]++, fine_n[FC_PUSH_WRITES] += (int64_t)dwr.writes.size(); + for (const VkWriteDescriptorSet& w : dwr.writes) fine_n[FC_PUSH_DESCS] += w.descriptorCount; } { FINE_SCOPE(FT_PUSHCALL); if (!push_write_template(cmd, bp, p)) // ((Triforce) through the program's push template where it has one) - vkCmdPushDescriptorSetKHR(cmd, bp, p.layout, 0, (uint32_t)dw().writes.size(), dw().writes.data()); + vkCmdPushDescriptorSetKHR(cmd, bp, p.layout, 0, (uint32_t)dwr.writes.size(), dwr.writes.data()); } - count_add(st_push_full, 1), count_add(st_push_full_n, descs_of(dw().writes.data(), dw().writes.size())); // ((Triforce) desc_report) - ps.cmd = cmd, ps.layout = p.layout, ps.n = dw().writes.size(); + count_add(st_push_full, 1), count_add(st_push_full_n, descs_of(dwr.writes.data(), dwr.writes.size())); // ((Triforce) desc_report) + ps.cmd = cmd, ps.layout = p.layout, ps.n = dwr.writes.size(); if (old_state) { - ps.writes.resize(dw().writes.size()); - for (size_t i = 0; i < dw().writes.size(); i++) write_words(dw().writes[i], ps.writes[i]); + ps.writes.resize(dwr.writes.size()); + for (size_t i = 0; i < dwr.writes.size(); i++) write_words(dwr.writes[i], ps.writes[i]); } else { ps.words.clear(), ps.at.clear(); - for (const VkWriteDescriptorSet& w : dw().writes) ps.at.push_back((uint32_t)ps.words.size()), words_append(w, ps.words); + for (const VkWriteDescriptorSet& w : dwr.writes) ps.at.push_back((uint32_t)ps.words.size()), words_append(w, ps.words); } return true; } @@ -5313,19 +5795,19 @@ bool bind_descriptors(VkCommandBuffer cmd, VkPipelineBindPoint bp, const Program static thread_local std::vector words; changed.clear(); if (old_state) { - for (size_t i = 0; i < dw().writes.size(); i++) { - write_words(dw().writes[i], words); - if (words != ps.writes[i]) changed.push_back(dw().writes[i]), ps.writes[i].swap(words); + for (size_t i = 0; i < dwr.writes.size(); i++) { + write_words(dwr.writes[i], words); + if (words != ps.writes[i]) changed.push_back(dwr.writes[i]), ps.writes[i].swap(words); } } else { // (the same layout, so the same count of words in each write: compared and kept in place) - for (size_t i = 0; i < dw().writes.size(); i++) { + for (size_t i = 0; i < dwr.writes.size(); i++) { const uint64_t* end = ps.words.data() + (i + 1 < ps.at.size() ? ps.at[i + 1] : ps.words.size()); - const int same = words_same_store(dw().writes[i], ps.words.data() + ps.at[i], end); + const int same = words_same_store(dwr.writes[i], ps.words.data() + ps.at[i], end); if (same < 0) { // (not so: nothing is known of what is pushed; the whole set again) ps.cmd = VK_NULL_HANDLE; return bind_descriptors(cmd, bp, p); } - if (!same) changed.push_back(dw().writes[i]); + if (!same) changed.push_back(dwr.writes[i]); } } if (!changed.empty()) { @@ -5345,7 +5827,7 @@ bool bind_descriptors(VkCommandBuffer cmd, VkPipelineBindPoint bp, const Program ps.cmd = VK_NULL_HANDLE; // (a set bound: what was pushed is not known to stay) if (g_fine) { fine_n[FC_POOL_SETS]++; - for (const VkWriteDescriptorSet& w : dw().writes) fine_n[FC_POOL_DESCS] += w.descriptorCount; + for (const VkWriteDescriptorSet& w : dwr.writes) fine_n[FC_POOL_DESCS] += w.descriptorCount; } // ((Triforce) the last set written alike for this layout here: bound again, not allocated and written (set_kept)) if (const VkDescriptorSet again = set_kept(cmd, bp, p.layout)) { @@ -5374,11 +5856,12 @@ bool bind_descriptors(VkCommandBuffer cmd, VkPipelineBindPoint bp, const Program return false; } } + pool_took(dwr.writes.data(), dwr.writes.size()); // ((Triforce, 11-22, DESC-33) for the "recording streams" line's peaks) if (!set_write_template(set, p)) { // ((Triforce) through the program's update template where it has one) - for (auto& w : dw().writes) w.dstSet = set; - vkUpdateDescriptorSets(dev.device, (uint32_t)dw().writes.size(), dw().writes.data(), 0, nullptr); + for (auto& w : dwr.writes) w.dstSet = set; + vkUpdateDescriptorSets(dev.device, (uint32_t)dwr.writes.size(), dwr.writes.data(), 0, nullptr); } - count_add(st_set_new, 1), count_add(st_set_new_n, descs_of(dw().writes.data(), dw().writes.size())); + count_add(st_set_new, 1), count_add(st_set_new_n, descs_of(dwr.writes.data(), dwr.writes.size())); vkCmdBindDescriptorSets(cmd, bp, p.layout, 0, 1, &set, 0, nullptr); set_written(cmd, bp, p.layout, set); return true; @@ -5864,6 +6347,11 @@ struct Pass { uint16_t sampled_targets = 0; // (colour i bit i, depth bit 8: targets earlier draws sampled) uint16_t written = 0; // (its targets written so far: colour i bit i, depth bit 8) }; +// ((Triforce, 11-22, the performance audit's DESC-20, after Source A's) pass() likewise: begin_pass, dynamic_state, +// draw_hazards, end_pass and draw take the thread's Pass once at their top (Pass& ps = pass();), where end_pass alone asked +// 57 times. A thread has one Pass for its whole life: a new recording assigns into it (pass() = Pass{}, draw_begin_recording), +// begin_pass and end_pass change its members, and nothing swaps the object behind the pointer, so the reference stays +// good across the calls these functions make (end_pass from begin_pass, the carry's resume). tests/test_tls_once.py) TLS_OBJECT(pass, Pass) // ((Triforce) RX_VK_SPLIT_BARRIER, barrier_rules.h: the targets the open pass was begun with; and the pass a part's split ended, with // what its draws sampled and wrote, and its targets, carried into the next part, which begins it again: pass_carry_take, _resume) @@ -6017,9 +6505,10 @@ bool full_screen_draw(const NvnState& s, const Targets& at, const DrawArgs& d) { } void begin_pass(VkCommandBuffer cmd, const Targets& at) { - bool same = pass().open && pass().depth == at.depth_view && pass().n == at.n && pass().width == at.width && - pass().height == at.height; - for (int i = 0; i < at.n && same; i++) same = pass().color[i] == at.color_view[i]; + Pass& ps = pass(); // ((Triforce, 11-22, DESC-20) once a call: the thread's one Pass, reset in place, never swapped (TLS_OBJECT(pass)) + bool same = ps.open && ps.depth == at.depth_view && ps.n == at.n && ps.width == at.width && + ps.height == at.height; + for (int i = 0; i < at.n && same; i++) same = ps.color[i] == at.color_view[i]; if (same) return; end_pass(cmd); if (pass_resuming) pass_resume_barrier(cmd); // ((Triforce) the pass a part's split ended, again: pass_carry_resume) @@ -6036,12 +6525,12 @@ void begin_pass(VkCommandBuffer cmd, const Targets& at) { ca[i].imageLayout = VK_IMAGE_LAYOUT_GENERAL; ca[i].loadOp = VK_ATTACHMENT_LOAD_OP_LOAD; ca[i].storeOp = VK_ATTACHMENT_STORE_OP_STORE; - pass().color[i] = at.color_view[i]; - pass().tex[i] = at.color[i], pass().level[i] = at.color_level[i], pass().layer[i] = at.color_layer[i]; + ps.color[i] = at.color_view[i]; + ps.tex[i] = at.color[i], ps.level[i] = at.color_level[i], ps.layer[i] = at.color_layer[i]; texture_written(at.color[i], "a pass's colour target"); texture_rendered(at.color[i]); } - for (int i = at.n; i < 8; i++) pass().color[i] = VK_NULL_HANDLE; + for (int i = at.n; i < 8; i++) ps.color[i] = VK_NULL_HANDLE; VkRenderingAttachmentInfo da{VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO}; if (at.depth) { da.imageView = at.depth_view; @@ -6051,9 +6540,9 @@ void begin_pass(VkCommandBuffer cmd, const Targets& at) { texture_written(at.depth, "a pass's depth target"); texture_rendered(at.depth); } - pass().depth = at.depth_view; - pass().depth_tex = at.depth, pass().depth_level = at.depth_level, pass().depth_layer = at.depth_layer; - pass().n = at.n, pass().width = at.width, pass().height = at.height; + ps.depth = at.depth_view; + ps.depth_tex = at.depth, ps.depth_level = at.depth_level, ps.depth_layer = at.depth_layer; + ps.n = at.n, ps.width = at.width, ps.height = at.height; VkRenderingInfo ri{VK_STRUCTURE_TYPE_RENDERING_INFO}; ri.renderArea = {{0, 0}, {at.width, at.height}}; ri.layerCount = 1; @@ -6062,12 +6551,12 @@ void begin_pass(VkCommandBuffer cmd, const Targets& at) { if (at.depth && depth_aspect(at.depth_format)) ri.pDepthAttachment = &da; if (at.depth && has_stencil(at.depth_format)) ri.pStencilAttachment = &da; gprof_begin(cmd); - pass().draws = 0, pass().format0 = at.n ? at.color_format[0] : at.depth_format; - pass().wm_or = 0, pass().wm_n = 0; - pass().clears = 0, pass().first_prog = 0; - pass().sampled.clear(), pass().sampled_targets = 0, pass().written = 0; + ps.draws = 0, ps.format0 = at.n ? at.color_format[0] : at.depth_format; + ps.wm_or = 0, ps.wm_n = 0; + ps.clears = 0, ps.first_prog = 0; + ps.sampled.clear(), ps.sampled_targets = 0, ps.written = 0; vkCmdBeginRendering(cmd, &ri); - pass().open = true; + ps.open = true; st_passes++; } @@ -6101,13 +6590,12 @@ thread_local bool last_dyn_valid = false; bool dyn_same(const float* fp, int nf, const uint32_t* up, int nu, const uint8_t* raw, int nb, const float* cfp, const uint32_t* cup, const uint8_t* craw) { if (nf <= 0 || nu <= 0 || nb <= 0) return false; + // ((Triforce, 11-22, the performance audit's DESC-06, after Source A's) the words and the bytes by one memcmp each: they + // were two loops of compares at every draw, 35 words and up to 160 bytes. The floats still one by one and by value, so + // that -0 is the same as 0 and a NaN the same as nothing, as before) for (int i = 0; i < nf; i++) if (fp[i] != cfp[i]) return false; - for (int i = 0; i < nu; i++) - if (up[i] != cup[i]) return false; - for (int i = 0; i < nb; i++) - if (raw[i] != craw[i]) return false; - return true; + return !std::memcmp(up, cup, (size_t)nu * sizeof *up) && !std::memcmp(raw, craw, (size_t)nb); } /// (Triforce) The face a draw culls, NVN's: the game's, or with RX_VK_FOLIAGE_SHADOW_CULL (on from layer 1 unless it is off) a mesh /// program's draw into the sun's far shadow cascades with one face culled (foliage_shadow.h). dyn_pack keeps it, so a draw into @@ -6204,13 +6692,14 @@ int dyn_pack(const NvnState& s, const Targets& at, VkPrimitiveTopology topo, boo } void dynamic_state(VkCommandBuffer cmd, const NvnState& s, const Targets& at, VkPrimitiveTopology topo, bool mesh) { + Pass& ps = pass(); // ((Triforce, 11-22, DESC-20) once a call: the thread's one Pass, reset in place, never swapped (TLS_OBJECT(pass)) float df[kDynF]; uint32_t du[kDynU]; uint8_t db[160]; const int nb = dyn_pack(s, at, topo, mesh, df, du, db); if (!dump_on() && last_dyn_valid && dyn_fast.on && dyn_fast.nb == nb && dyn_same(df, kDynF, du, kDynU, db, nb, dyn_fast.fp, dyn_fast.up, dyn_fast.raw)) { - pass().wm_or |= dyn_fast.wm0, pass().wm_n = dyn_fast.wm_n; // (the pass trace still sees the masks) + ps.wm_or |= dyn_fast.wm0, ps.wm_n = dyn_fast.wm_n; // (the pass trace still sees the masks) FINE_N(FC_DYN_FAST, 1); return; } @@ -6390,7 +6879,7 @@ void dynamic_state(VkCommandBuffer cmd, const NvnState& s, const Targets& at, Vk std::memcpy(dyn_fast.up, du, sizeof du); std::memcpy(dyn_fast.raw, db, (size_t)nb); dyn_fast.wm0 = d.n ? (uint32_t)d.wm[0] : 0, dyn_fast.wm_n = (uint32_t)d.n; - pass().wm_or |= dyn_fast.wm0, pass().wm_n = dyn_fast.wm_n; + ps.wm_or |= dyn_fast.wm0, ps.wm_n = dyn_fast.wm_n; } #undef DYN_CHANGED @@ -6528,6 +7017,7 @@ bool writes_sampled_target(uint16_t writes, uint16_t sampled_targets) { /// the targets' writes with each other only, not with the draws' texture reads of them. (A draw that samples a target /// it writes itself is left as it is.) Returns the targets the draw writes. RX_VK_NO_PASS_HAZARDS=1: not checked. uint16_t draw_hazards(VkCommandBuffer cmd, const Program& p, const NvnState& s, const Targets& at) { + Pass& ps = pass(); // ((Triforce, 11-22, DESC-20) once a call: the thread's one Pass, reset in place, never swapped (TLS_OBJECT(pass)) uint16_t writes = 0; for (int i = 0; i < at.n; i++) if (at.color[i] && target_write_mask(p, s, i)) writes |= (uint16_t)(1u << i); @@ -6548,9 +7038,9 @@ uint16_t draw_hazards(VkCommandBuffer cmd, const Program& p, const NvnState& s, } } static const bool off = getenv("RX_VK_NO_PASS_HAZARDS") != nullptr; - if (off || !pass().open) return writes; - bool same = pass().depth == at.depth_view && pass().n == at.n && pass().width == at.width && pass().height == at.height; - for (int i = 0; i < at.n && same; i++) same = pass().color[i] == at.color_view[i]; + if (off || !ps.open) return writes; + bool same = ps.depth == at.depth_view && ps.n == at.n && ps.width == at.width && ps.height == at.height; + for (int i = 0; i < at.n && same; i++) same = ps.color[i] == at.color_view[i]; if (!same) return writes; // (other targets: begin_pass ends this pass with a barrier anyway) auto slot_of = [&](const Texture* t) { for (int i = 0; i < at.n; i++) @@ -6560,15 +7050,15 @@ uint16_t draw_hazards(VkCommandBuffer cmd, const Program& p, const NvnState& s, const char* kind = nullptr; for (const Texture* t : gfx_sampled()) { const int k = slot_of(t); - if (k >= 0 && (pass().written >> k & 1) && !(writes >> k & 1)) { kind = "samples a target the pass wrote"; break; } + if (k >= 0 && (ps.written >> k & 1) && !(writes >> k & 1)) { kind = "samples a target the pass wrote"; break; } } - if (!kind && writes_sampled_target(writes, pass().sampled_targets)) + if (!kind && writes_sampled_target(writes, ps.sampled_targets)) kind = "writes a target an earlier draw sampled"; if (!kind) return writes; static int logged; if (logged++ < 8) logf("vk: a draw (program %016llx) %s, after %d draws of the pass (%ux%u): a pass of its own", (unsigned long long)p.key, - kind, pass().draws, pass().width, pass().height); + kind, ps.draws, ps.width, ps.height); end_pass(cmd); note_pass_split(cmd); // ((Triforce) a full barrier before the draw's pass: it may read the targets in its vertex or mesh stage) return writes; @@ -6658,6 +7148,60 @@ void draw_flush_recording(int slot) { } } +/// (ParLock held) A slot's reset for its next recording: its descriptor pool, the extra pools, and the streams the last +/// recording grew into +void slot_reset_locked(RecordingRes& r) { + // ((Triforce, 11-22, the performance audit's DESC-33, after Source A's; desc_pool_grow.h) a slot whose last recording ran + // out of its half pool (it has extra pools) gets a whole one here, where the extra pools go anyway: the half one destroyed, + // not reset. Only once the whole one is made, so the slot is never left without a pool + bool whole = false; + if (desc_pool_grow::remake(r.shift, r.extra.size())) { + if (const VkDescriptorPool made = make_pool(0)) { + vkDestroyDescriptorPool(dev.device, r.pool, nullptr); + r.pool = made, r.shift = 0, whole = true; + pools_whole++; + static int told; + if (told++ < 8) + logf("vk: a recording ran out of its slot's half descriptor pool: the slot has a whole one from here on (%llu slots " + "so far; RX_VK_DESC_POOL_GROW=0: whole from the start)", (unsigned long long)pools_whole); + } + } + if (!whole) vkResetDescriptorPool(dev.device, r.pool, 0); + pools_added += r.extra.size(); + for (VkDescriptorPool p : r.extra) vkDestroyDescriptorPool(dev.device, p, nullptr); + r.extra.clear(); + // (the streams the last recording of the slot grew into: given back after 120 recordings that stayed in the first, the + // slot's last one being done (recording_wait). RX_VK_STREAM_KEEP=1: kept) + static const bool keep = getenv("RX_VK_STREAM_KEEP") != nullptr; + if (r.streams.size() > 1 && !keep) { + if (r.si > 0) r.idle = 0; + else if (++r.idle >= 120) { + for (size_t i = 1; i < r.streams.size(); i++) { + stream_extra -= r.streams[i].size; + destroy_buffer(r.streams[i].buffer, r.streams[i].alloc); + } + r.streams.resize(1); + } + } +} +/// ((Triforce, 11-21, the performance audit's REPLAY-04, after Source A's) A recording slot's reset made at the recording's +/// start (vk_replay.cpp begin_recording), right after the slot's command pools are reset and before its first packet. With +/// the recorder thread (RX_VK_RECORDER=thread) a descriptor pool's reset, a pool's destruction and a buffer's are boundaries: +/// the replay waits there until the recorder has run every packet before them. Made in draw_begin_recording they came after +/// the recording's first packets (the occlusion block's reset, the GPU timers, the checkpoint), so every recording began with +/// a wait for the recorder. Right after the command pools' resets, themselves boundaries, there is nothing to wait for. +/// A slot with no pool yet is left to draw_begin_recording, and a helper thread's segment, which makes its calls itself, +/// never comes here. RX_VK_RECORDER_OWN=0: reset in draw_begin_recording, as before +void draw_slot_reset(int slot) { + static const bool on = !(getenv("RX_VK_RECORDER_OWN") && atoi(getenv("RX_VK_RECORDER_OWN")) == 0); + if (!on) return; + ParLock lk; + if ((size_t)slot >= res.size()) return; + RecordingRes& r = res[slot]; + if (!r.pool) return; + slot_reset_locked(r); + r.reset_ahead = true; +} void draw_begin_recording(int slot, VkCommandBuffer cmd, uint64_t value) { rec_cmd = cmd; rec_value = value; @@ -6665,24 +7209,12 @@ void draw_begin_recording(int slot, VkCommandBuffer cmd, uint64_t value) { if ((size_t)slot >= res.size()) res.resize((size_t)slot + 1); RecordingRes& r = res[slot]; if (!r.pool) { - r.pool = make_pool(); + r.shift = desc_pool_grow::first_shift(pool_grow_on()); // ((Triforce, 11-22, DESC-33) half, whole once a recording ran out) + r.pool = make_pool(r.shift); + } else if (r.reset_ahead) { + r.reset_ahead = false; // ((Triforce, 11-21, REPLAY-04) made at the recording's start already: draw_slot_reset) } else { - vkResetDescriptorPool(dev.device, r.pool, 0); - for (VkDescriptorPool p : r.extra) vkDestroyDescriptorPool(dev.device, p, nullptr); - r.extra.clear(); - // (the streams the last recording of the slot grew into: given back after 120 recordings that stayed in the first, the - // slot's last one being done (recording_wait). RX_VK_STREAM_KEEP=1: kept) - static const bool keep = getenv("RX_VK_STREAM_KEEP") != nullptr; - if (r.streams.size() > 1 && !keep) { - if (r.si > 0) r.idle = 0; - else if (++r.idle >= 120) { - for (size_t i = 1; i < r.streams.size(); i++) { - stream_extra -= r.streams[i].size; - destroy_buffer(r.streams[i].buffer, r.streams[i].alloc); - } - r.streams.resize(1); - } - } + slot_reset_locked(r); } if (r.streams.empty()) stream_insert(r, 0, kStream); if (!r.streams.empty()) { @@ -6692,11 +7224,23 @@ void draw_begin_recording(int slot, VkCommandBuffer cmd, uint64_t value) { r.si = 0, r.stream = VK_NULL_HANDLE, r.stream_alloc = nullptr, r.mapped = nullptr, r.size = 0; } if (r.before + r.used > stream_peak.load(std::memory_order_relaxed)) stream_peak.store(r.before + r.used, std::memory_order_relaxed); + desc_pool_grow::peak(pool_most, r.took); // ((Triforce, 11-22, DESC-33) what the slot's last recording took of its pools) + r.took = {}; static int64_t begun; - if (++begun % 3000 == 0) + if (++begun % 3000 == 0) { logf("vk: the recording streams: %zu of %llu MB, the most one held %llu KB; %llu made on demand (%llu MB kept now)", res.size(), (unsigned long long)(kStream >> 20), (unsigned long long)(stream_peak.load(std::memory_order_relaxed) >> 10), (unsigned long long)stream_grown.load(), (unsigned long long)(stream_extra.load() >> 20)); + // ((Triforce, 11-22, DESC-33) and their descriptor pools: what a size is judged by) + size_t half = 0; + for (const RecordingRes& s : res) half += s.pool && s.shift ? 1 : 0; + logf("vk: the recording streams' descriptor pools: %zu of %zu slots at half size, %llu made whole, %llu extra pools made; the " + "most one recording took: %u sets, %u uniform and %u storage buffers, %u sampled and %u storage images, %u and %u texel views", + half, res.size(), (unsigned long long)pools_whole, (unsigned long long)pools_added, pool_most.sets, + pool_most.n[desc_pool_grow::kUniform], pool_most.n[desc_pool_grow::kStorage], pool_most.n[desc_pool_grow::kSampled], + pool_most.n[desc_pool_grow::kStorageImage], pool_most.n[desc_pool_grow::kUniformTexel], + pool_most.n[desc_pool_grow::kStorageTexel]); + } r.used = 0, r.before = 0; r.c1_at.reset(); // (its array kept: emptied at every recording's start, it fills again to about the same size) cur = &r; @@ -6951,10 +7495,11 @@ namespace { TLS_OBJECT(dump_mesh_indirect, std::vector>) // (program key, indirect address) } // namespace void end_pass(VkCommandBuffer cmd) { - if (!pass().open) return; + Pass& ps = pass(); // ((Triforce, 11-22, DESC-20) once a call: the thread's one Pass, reset in place, never swapped (TLS_OBJECT(pass)) + if (!ps.open) return; if (occ_live) occ_pass_end(cmd); // (Triforce, RX_VK_OCC_QUERY: a query begun in the pass ends in it) vkCmdEndRendering(cmd); - pass().open = false; + ps.open = false; note_pass_end(cmd); // (before the dumps and probes below: they record work, and the next pass's barrier is then a full one) if (dump_on() && !dump_mesh_indirect().empty()) { for (const auto& [key, addr] : dump_mesh_indirect()) { @@ -6966,52 +7511,52 @@ void end_pass(VkCommandBuffer cmd) { dump_mesh_indirect().clear(); } if (probe_on()) - for (int i = 0; i < pass().n; i++) - if (pass().color[i] && pass().tex[i]) probe_image(cmd, pass().tex[i], pass().level[i], pass().layer[i], pass().prog, 'p', pass().draws); + for (int i = 0; i < ps.n; i++) + if (ps.color[i] && ps.tex[i]) probe_image(cmd, ps.tex[i], ps.level[i], ps.layer[i], ps.prog, 'p', ps.draws); if (pass_trace_on()) { char sampled[160] = ""; int k = 0; - for (size_t i = 0; i < pass().sampled.size() && i < 6 && k < 140; i++) - k += snprintf(sampled + k, sizeof sampled - k, " %#llx", (unsigned long long)pass().sampled[i]->gpu); + for (size_t i = 0; i < ps.sampled.size() && i < 6 && k < 140; i++) + k += snprintf(sampled + k, sizeof sampled - k, " %#llx", (unsigned long long)ps.sampled[i]->gpu); trace_line("pass %ux%u f%d %dc%s wm%x/%u: %d draws %d clears, c0 %#llx/%u, depth %#llx/%u, progs %016llx..%016llx, " - "sampled%s", pass().width, pass().height, (int)pass().format0, pass().n, pass().depth ? "+d" : "", pass().wm_or, pass().wm_n, pass().draws, - pass().clears, (unsigned long long)(pass().n && pass().tex[0] ? pass().tex[0]->gpu : 0), pass().layer[0], - (unsigned long long)(pass().depth_tex ? pass().depth_tex->gpu : 0), pass().depth_layer, - (unsigned long long)pass().first_prog, (unsigned long long)pass().prog, sampled); + "sampled%s", ps.width, ps.height, (int)ps.format0, ps.n, ps.depth ? "+d" : "", ps.wm_or, ps.wm_n, ps.draws, + ps.clears, (unsigned long long)(ps.n && ps.tex[0] ? ps.tex[0]->gpu : 0), ps.layer[0], + (unsigned long long)(ps.depth_tex ? ps.depth_tex->gpu : 0), ps.depth_layer, + (unsigned long long)ps.first_prog, (unsigned long long)ps.prog, sampled); } if (gprof_on()) { char label[160]; - snprintf(label, sizeof label, "pass %ux%u f%d %dc%s, %s draws, first prog %016llx, last prog %016llx", pass().width, pass().height, - (int)pass().format0, pass().n, pass().depth ? "+d" : "", pass().draws > 256 ? ">256" : pass().draws > 32 ? "33-256" : pass().draws > 4 ? "5-32" : "0-4", - (unsigned long long)pass().first_prog, (unsigned long long)pass().prog); - gprof_end(cmd, label, pass().draws); + snprintf(label, sizeof label, "pass %ux%u f%d %dc%s, %s draws, first prog %016llx, last prog %016llx", ps.width, ps.height, + (int)ps.format0, ps.n, ps.depth ? "+d" : "", ps.draws > 256 ? ">256" : ps.draws > 32 ? "33-256" : ps.draws > 4 ? "5-32" : "0-4", + (unsigned long long)ps.first_prog, (unsigned long long)ps.prog); + gprof_end(cmd, label, ps.draws); } if (dump_on()) - for (int i = 0; i < pass().n; i++) - if (pass().color[i] && pass().tex[i]) { + for (int i = 0; i < ps.n; i++) + if (ps.color[i] && ps.tex[i]) { char what[64]; - snprintf(what, sizeof what, "pass c%d prog %016llx", i, (unsigned long long)pass().prog); - dump_image(cmd, pass().tex[i], pass().level[i], pass().layer[i], what); + snprintf(what, sizeof what, "pass c%d prog %016llx", i, (unsigned long long)ps.prog); + dump_image(cmd, ps.tex[i], ps.level[i], ps.layer[i], what); } - if (dump_on() && pass().depth && pass().depth_tex) { + if (dump_on() && ps.depth && ps.depth_tex) { char what[64]; - snprintf(what, sizeof what, "pass depth prog %016llx", (unsigned long long)pass().prog); - dump_image(cmd, pass().depth_tex, pass().depth_level, pass().depth_layer, what); + snprintf(what, sizeof what, "pass depth prog %016llx", (unsigned long long)ps.prog); + dump_image(cmd, ps.depth_tex, ps.depth_level, ps.depth_layer, what); } if (dump_on()) for (Texture* t : dump_sampled()) { char what[64]; - snprintf(what, sizeof what, "sampled by %016llx", (unsigned long long)pass().prog); + snprintf(what, sizeof what, "sampled by %016llx", (unsigned long long)ps.prog); static const bool all_layers = getenv("RX_VK_DUMP_ALL_LAYERS") != nullptr; // (Triforce: every layer of an array) for (uint32_t l = 0; l < std::min(t->layers, all_layers ? 256 : 2); l++) dump_image(cmd, t, 0, l, what); if (t->guest_size >= (8u << 20)) texture_log_guest(t); // (and what the game's memory holds of it) for (uint32_t lv = 1; t->layers == 1 && lv < std::min(t->levels, 8); lv++) { // (every level) - snprintf(what, sizeof what, "sampled by %016llx level %u", (unsigned long long)pass().prog, lv); + snprintf(what, sizeof what, "sampled by %016llx level %u", (unsigned long long)ps.prog, lv); dump_image(cmd, t, lv, 0, what); } } dump_sampled().clear(); - pass().prog = 0; + ps.prog = 0; } bool pass_carry_take() { @@ -7085,7 +7630,7 @@ void log_dump_draw(const NvnState& s, const Program& p, const DrawArgs& d) { const nvn_p_bind_buf& u = s.ubo[0][i]; if (!u.addr) continue; float f[16] = {}; - const bool upd = ub_overlay(u.addr, sizeof f, (uint8_t*)f, [](uint64_t) { return (uint64_t)0; }) != UINT64_MAX; + const bool upd = ub_overlay(u.addr, sizeof f, (uint8_t*)f, [](uint64_t, uint64_t) { return (uint64_t)0; }, 0) != UINT64_MAX; if (!upd) host->gpu_read(u.addr, f, sizeof f); n += snprintf(line + n, sizeof line - n, " C%d%s %#llx+%#llx [", i + 3, upd ? "*" : "", (unsigned long long)u.addr, (unsigned long long)u.size); @@ -7113,7 +7658,7 @@ void log_dump_draw(const NvnState& s, const Program& p, const DrawArgs& d) { if (!u.addr) continue; float f[16] = {}; // (as the draw reads it: guest memory with the stream's uniform updates over it; '*': updated) - const bool upd = ub_overlay(u.addr, sizeof f, (uint8_t*)f, [](uint64_t) { return (uint64_t)0; }) != UINT64_MAX; + const bool upd = ub_overlay(u.addr, sizeof f, (uint8_t*)f, [](uint64_t, uint64_t) { return (uint64_t)0; }, 0) != UINT64_MAX; if (!upd) host->gpu_read(u.addr, f, sizeof f); n += snprintf(line + n, sizeof line - n, " c%d%s %#llx+%#llx [", i + 3, upd ? "*" : "", (unsigned long long)u.addr, (unsigned long long)u.size); @@ -8008,27 +8553,49 @@ bool fallback_buffers_of(const Program& p, FallbackBuffers*& out) { void* mapped = nullptr; const VkBufferUsageFlags usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT; - // (the records' header is the vertex stage's indirect draw and the mesh stage's indirect dispatch) - if (!create_buffer(records, usage | VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT, false, f.records, f.records_mem, &mapped)) return false; - f.records_address = fallback_buffer_address(f.records); - f.records_bytes = records; - if (!f.records_address) { - destroy_buffer(f.records, f.records_mem); - return false; + // ((Triforce, 11-21, the performance audit's MEM-154, after Source A's) A payload that fits no pair made a whole new + // pair, 64 MB of records with it, and the old pairs stay for the session. The records are the same size for every + // program, so the new pair takes the newest pair's and only the payload is made: shared for the reason all programs + // share one pair's already (a thread records its draws in order, each clears the slots it runs, and a barrier orders + // each dispatch after the draw before). RX_VK_FALLBACK_SHARED_RECORDS=0: a pair of its own, as before) + static const bool share = !(getenv("RX_VK_FALLBACK_SHARED_RECORDS") && atoi(getenv("RX_VK_FALLBACK_SHARED_RECORDS")) == 0); + const bool shared = share && !sets.empty() && sets.back().records && sets.back().records_bytes >= records; + if (shared) { + const FallbackBuffers& last = sets.back(); + f.records = last.records, f.records_mem = last.records_mem; + f.records_address = last.records_address, f.records_bytes = last.records_bytes; + } else { + // (the records' header is the vertex stage's indirect draw and the mesh stage's indirect dispatch) + if (!create_buffer(records, usage | VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT, false, f.records, f.records_mem, &mapped)) return false; + f.records_address = fallback_buffer_address(f.records); + f.records_bytes = records; + if (!f.records_address) { + destroy_buffer(f.records, f.records_mem); + return false; + } } + // (records another pair holds too are not this one's to free) + const auto drop_records = [&] { + if (!shared) destroy_buffer(f.records, f.records_mem); + }; if (payload) { if (!create_buffer(payload, usage, false, f.payload, f.payload_mem, &mapped)) { - destroy_buffer(f.records, f.records_mem); + drop_records(); return false; } f.payload_address = fallback_buffer_address(f.payload); f.payload_bytes = payload; if (!f.payload_address) { destroy_buffer(f.payload, f.payload_mem); - destroy_buffer(f.records, f.records_mem); + drop_records(); return false; } } + static std::atomic told{0}; + if (told++ < 16) + logf("vk: mesh fallback: a buffer pair for a recording thread: payload %llu MB, records %llu MB %s " + "(RX_VK_FALLBACK_SHARED_RECORDS=0: its own records each time)", (unsigned long long)(payload >> 20), + (unsigned long long)(f.records_bytes >> 20), shared ? "shared with the pair before" : "made"); sets.push_back(f); out = &sets.back(); return true; @@ -8049,6 +8616,43 @@ bool fallback_compute_pipelines(Program& p) { if (!p.fallback_compute.empty()) return true; if (failed.count({p.key, p.variant})) return false; } + // ((Triforce, 11-22, the performance audit's PIPE-56, after Source A's) On RADV for GCN 1 to 4 and on Mesa's Intel driver, + // a program with a compute stage over the driver's size limit (fallback_size_policy.h: 50,000 phis on GCN; 20,000 phis or + // 200 calls on Mesa's Intel driver) is not handed to the driver at all: those builds took tens of GB and minutes and ended + // killed. It goes into the failed set as a refused build does, before any stage is built, and its draws are left out. + // RX_VK_FALLBACK_PHI_LIMIT / RX_VK_FALLBACK_CALL_LIMIT: the limits there, 0 none. Nothing is asked on any other driver + if (const fallback_size::Driver size_driver = fallback_size_driver(); size_driver != fallback_size::Driver::Other) { + static const fallback_size::Limits limits = + fallback_size::limits(size_driver, getenv("RX_VK_FALLBACK_PHI_LIMIT"), getenv("RX_VK_FALLBACK_CALL_LIMIT")); + for (size_t i = 0; i < p.stages.size(); i++) { + const Stage& st = p.stages[i]; + if (st.vk_stage != VK_SHADER_STAGE_COMPUTE_BIT) continue; + const fallback_size::Size size = stage_size(st); + { // (the largest stage seen, to measure the limits again: a line when it grows by a tenth, 16 at most) + static std::mutex seen_mu; + static fallback_size::Size largest; + static int seen_told = 0; + std::lock_guard lk(seen_mu); + const bool more = size.phis > largest.phis + largest.phis / 10 + 100 || size.calls > largest.calls + largest.calls / 10 + 10; + if (size.phis > largest.phis) largest.phis = size.phis; + if (size.calls > largest.calls) largest.calls = size.calls; + if (more && seen_told++ < 16) + logf("vk: fallback: the largest compute stage so far: %u phis, %u calls (%016llx stage %zu; this driver's limits %u " + "and %u, 0 none)", size.phis, size.calls, (unsigned long long)p.key, i, limits.phis, limits.calls); + } + if (!fallback_size::refused(limits, size)) continue; + { + std::lock_guard lk(mu); + failed.insert({p.key, p.variant}); + } + static std::atomic size_told{0}; + if (size_told++ < 8) + logf("vk: fallback: stage %zu of %016llx has %u phis and %u calls, over this driver's limit of %u phis or %u calls " + "(0: none; RX_VK_FALLBACK_PHI_LIMIT, RX_VK_FALLBACK_CALL_LIMIT): not built, its draws are left out", + i, (unsigned long long)p.key, size.phis, size.calls, limits.phis, limits.calls); + return false; + } + } std::vector made(p.stages.size(), VK_NULL_HANDLE); bool any = false; for (size_t i = 0; i < p.stages.size(); i++) { @@ -8261,6 +8865,7 @@ void bind_index(VkCommandBuffer cmd, VkBuffer buffer, VkDeviceSize offset, VkInd const uint64_t b = (uint64_t)(uintptr_t)buffer; if (index_bind_same(b, (uint64_t)offset, (uint32_t)type, last_ib.buffer, last_ib.offset, last_ib.type, last_ib.on)) return; + FINE_N(FC_IBIND_MADE, 1); vkCmdBindIndexBuffer(cmd, buffer, offset, type); last_ib = {b, (uint64_t)offset, (uint32_t)type, true}; } @@ -8457,9 +9062,16 @@ static bool fallback_list_held(uint64_t list) { static bool program_writes_memory(const Program& p); // ((Triforce) writes_memory, kept per program: below draw()) void draw(VkCommandBuffer cmd, NvnState& s, const DrawArgs& d) { + Pass& ps = pass(); // ((Triforce, 11-22, DESC-20) once a call: the thread's one Pass, reset in place, never swapped (TLS_OBJECT(pass)) + lab_ab::each(lab_ab::E_DRAW); // ((Triforce, 11-22, REPLAY-51) the lab's e_draw: a wait at each draw, before the profile's clock PROF_HOT(PROF_DRAW); FINE_N(FC_DRAWS, 1); stats_tick(); + { // ((Triforce, 11-21, a lab measure only) RX_VK_LAB_NODRAW=1: every draw left out before its pass, pipeline, descriptors + // and commands: the picture is wrong, the frame rate says what the per-draw work costs the frame at most) + static const bool nodraw = getenv("RX_VK_LAB_NODRAW") && atoi(getenv("RX_VK_LAB_NODRAW")) == 1; + if (nodraw) SKIP_HERE(); + } if (occ_live) occ_draw_enter(); // (Triforce, RX_VK_OCC_QUERY: a draw a samples-passed count wants; occ_draw_issued below) mesh_indirect_convert(cmd, d); // (EvilOOT: before the pass opens; sets mesh_conv_buf for the mesh draw below) VkPrimitiveTopology topo = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; @@ -8635,7 +9247,26 @@ void draw(VkCommandBuffer cmd, NvnState& s, const DrawArgs& d) { }; if (!cp->ready.load(std::memory_order_acquire)) { if (legacy_translation_skip && targets_persistent(at)) return left_out_pending(), void(st_pending_skips++); - wait_for(cp->ready, st_shader_wait_us); + // ((Triforce, 11-22, the performance audit's PIPE-30, after Source A's; translate_wait.h) RX_VK_TRANSLATE_WAIT_MS: into + // targets that hold an earlier frame, a program whose key is known to write no memory is waited for at most what the + // frame's budget has left, the job hurried but not run here, then left out; a spent budget leaves it out at once) + if (translate_wait_ms() && !always_wait) { + const uint64_t frame = texture_frame_no(); + const translate_wait::Act a = translate_wait::choose(translate_wait_ms(), pass_kept(at, pipe_skip_new()), + translate_notes().clean(cp->key), translate_pool().left(frame)); + if (a != translate_wait::Act::AsBefore) { + bool got = false; + if (a == translate_wait::Act::Bounded) { + const auto t0 = std::chrono::steady_clock::now(); + got = jobs_wait_for(cp->ready, (int)std::max(1, translate_pool().left(frame) / pipe_wait::kMs)); + note_wait(t0, st_shader_wait_us); + translate_pool().charge(frame, std::chrono::duration_cast(std::chrono::steady_clock::now() - t0).count()); + st_translate_bounded.fetch_add(1, std::memory_order_relaxed); + } + if (!got) return st_translate_left.fetch_add(1, std::memory_order_relaxed), left_out_pending(), void(st_pending_skips++); + } + } + if (!cp->ready.load(std::memory_order_acquire)) wait_for(cp->ready, st_shader_wait_us); } if (exposure_log_on() && cp) exposure_note(s, *cp); if (cp->failed) SKIP_HERE(); @@ -8675,9 +9306,15 @@ void draw(VkCommandBuffer cmd, NvnState& s, const DrawArgs& d) { const bool full = kept && !writes && full_screen_draw(s, at, d); const bool held = full && hold_takes_frame(); const uint64_t frame = texture_frame_no(); + // ((Triforce, 11-22, PIPE-29) RX_VK_PIPE_DEPTH_BUDGET: a depth-tested draw into a pass drawn once, its shaders writing + // nothing, no samples-passed count open, not the icon capture's kept targets; one whose wait ran out: left out at once) + const bool depth_new = !always_wait && !kept && pipe_depth_budget() && at.depth && s.ds.dt && !targets_persistent(at) && + !writes_memory(p) && !occ_counting_now(); + const bool depth_out = depth_new && pipe_depth_timed_out(pe); const pipe_wait::Choice c = always_wait ? pipe_wait::Choice{pipe_wait::Act::WaitAll} + : depth_out ? pipe_wait::Choice{pipe_wait::Act::Skip} : pipe_wait::choose(rules, kept, writes, full, held, pipe_pool().left(frame), - pipe_last().left(frame)); + pipe_last().left(frame), depth_new); if (!always_wait && !kept && targets_persistent(at)) st_pipe_new_waits++; // (by the new colour target alone: RX_VK_PIPE_SKIP_NEW) if (c.act == pipe_wait::Act::WaitAll) { wait_for(pe->ready, st_pipe_wait_us); // ((Triforce) a pipeline's wait, as pipe_wait_for's: the overlay's Pipeline waits) @@ -8688,10 +9325,18 @@ void draw(VkCommandBuffer cmd, NvnState& s, const DrawArgs& d) { got = pipe_wait_for(pe->ready, c.ns); const int64_t spent = std::chrono::duration_cast(std::chrono::steady_clock::now() - t0).count(); (c.kind == pipe_wait::K_LAST ? pipe_last() : pipe_pool()).charge(frame, spent); - if (c.kind != pipe_wait::K_ONESHOT) st_pipe_full_waits++; + if (c.kind == pipe_wait::K_DEPTH) st_pipe_depth_waits.fetch_add(1, std::memory_order_relaxed); + else if (c.kind != pipe_wait::K_ONESHOT) st_pipe_full_waits++; } if (!got) { if (full) (held ? st_pipe_full_held : st_pipe_full_left)++; + if (depth_new) { // ((Triforce, 11-22, PIPE-29) ran out of time (or the pool was spent): left out at once from now on) + pipe_depth_time_out(pe); + const int64_t n = st_pipe_depth_left.fetch_add(1, std::memory_order_relaxed); + if (n < 16) + logf("vk: a depth-tested draw of %016llx into a pass drawn once is left out: its pipeline is not built " + "within the frame's wait (RX_VK_PIPE_DEPTH_BUDGET)", (unsigned long long)p.key); + } return left_out_pending(), void(st_pending_skips++); } } @@ -8944,25 +9589,34 @@ void draw(VkCommandBuffer cmd, NvnState& s, const DrawArgs& d) { FINE_SCOPE(FT_HAZARD); begin_pass(cmd, at); } - probe_draw(s, p, pass().draws); // (hookshot: RX_VK_PROBE) + probe_draw(s, p, ps.draws); // (hookshot: RX_VK_PROBE) gprof_run(cmd, p.key); // (RX_VK_PASS_TIMING=2) - pass().prog = p.key; - if (!pass().first_prog) pass().first_prog = p.key; + ps.prog = p.key; + if (!ps.first_prog) ps.first_prog = p.key; for (const Texture* t : gfx_sampled()) { - if (std::find(pass().sampled.begin(), pass().sampled.end(), t) != pass().sampled.end()) continue; - pass().sampled.push_back(t); + if (std::find(ps.sampled.begin(), ps.sampled.end(), t) != ps.sampled.end()) continue; + ps.sampled.push_back(t); int slot = -1; for (int i = 0; i < at.n; i++) if (at.color[i] == t) { slot = i; break; } if (slot < 0 && at.depth == t) slot = 8; - if (slot >= 0) pass().sampled_targets |= (uint16_t)(1u << slot); + if (slot >= 0) ps.sampled_targets |= (uint16_t)(1u << slot); } - pass().written |= writes; + ps.written |= writes; const bool bound = pl != last_pipeline; if (bound) { FINE_SCOPE(FT_BINDPL); FINE_N(FC_BIND_PIPE, 1); vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, pl), last_pipeline = pl; + // ((Triforce, 11-22) a mesh pipeline has the vertex strides static (only the other pipelines have them dynamic, + // build_graphics' dyn): binding it leaves the strides undefined for the next pipeline that has them dynamic, as it does + // the topology (dyn_pack sets that again after a mesh draw). The next vertex draw binds all 16 streams again, strides + // too: the set already bound (vertex_set_same) and the streams that did not change (vertex_bind_range, 0.6.0) were left + // with whatever strides the driver kept. Off by default: in the lab it cost the Market about 3% (the replay's submit 8.5 -> + // 9.1 ms a frame, the mesh draws' binds coming between the vertex draws' all frame), and the tickets it may answer are not + // reproduced; RX_VK_MESH_VB_REBIND=1: on, for a player's test) + static const bool mesh_vb_rebind = getenv("RX_VK_MESH_VB_REBIND") && atoi(getenv("RX_VK_MESH_VB_REBIND")) != 0; + if (p.mesh && mesh_vb_rebind) last_vb_valid = false, vertex_set.on = false; } // ((Triforce) the pipeline has the shading rate dynamic: set right after its bind, before anything can leave the draw out, so // that a later draw that finds it bound finds the rate set too; draw_rate) @@ -9056,7 +9710,13 @@ void draw(VkCommandBuffer cmd, NvnState& s, const DrawArgs& d) { } } StatsQuery stats_query(cmd, p.key, d); // (EvilOOT, RX_VK_DRAW_STATS) - if (occ_live && pass().open) occ_draw_begin(cmd); // (Triforce, RX_VK_OCC_QUERY: a count's first draw of the pass begins its query) + // ((Triforce, 11-22, the performance audit's GPUWORK-01, after Source A's) a draw that writes a colour target (its write + // mask, as draw_hazards made it: bit 8 is the depth's) is told to the count first: counted for the occlusion line, and + // with RX_VK_OCC_DEPTH_ONLY=1 the count is given up, its open query ended before this draw) + if (occ_live && ps.open && (writes & 0xFF)) occ_draw_colour(cmd); + if (occ_live && ps.open) occ_draw_begin(cmd); // (Triforce, RX_VK_OCC_QUERY: a count's first draw of the pass begins its query) + // ((Triforce, 11-22, DESC-82) RX_VK_OCC_LEAN=3: a draw the open query counts that writes no colour, in the census) + if (occ_live && ps.open && !(writes & 0xFF) && occ_lean_census_on() && occ_counting()) occ_lean_note(p, s); if (stats_frame_now() && stats_prog_wanted(p.key)) { // (and the wanted programs' buffers: a mesh draw's stages, fragment) for (const int stage : {6, 7, 1, 0}) stats_dump_stage(p.key, s, stage); // (and as the GPU holds them, after it: RX_VK_GPU_DUMP=1 only; its bindings can sync a pool's mirror mid-frame, and @@ -9068,7 +9728,7 @@ void draw(VkCommandBuffer cmd, NvnState& s, const DrawArgs& d) { if (p.fallback) { // (hookshot: the generated vertex stage reads the records; one vertex a corner) vkCmdDrawIndirect(cmd, fallback_draw_records, 0, 1, sizeof(VkDrawIndirectCommand)); // (its count: the records taken) st_mesh++; - st_draws++, pass().draws++; + st_draws++, ps.draws++; if (occ_live) occ_draw_issued(); return; } @@ -9155,7 +9815,7 @@ void draw(VkCommandBuffer cmd, NvnState& s, const DrawArgs& d) { vkCmdDrawMeshTasksIndirectEXT(cmd, mesh_conv_buf, mesh_conv_off, conv_n, 12); mesh_conv_buf = VK_NULL_HANDLE; st_mesh++; - st_draws++, pass().draws++; + st_draws++, ps.draws++; if (occ_live) occ_draw_issued(); return; } @@ -9187,7 +9847,7 @@ void draw(VkCommandBuffer cmd, NvnState& s, const DrawArgs& d) { (uint32_t)std::max(d.groups[2], 1)); } st_mesh++; - st_draws++, pass().draws++; + st_draws++, ps.draws++; if (occ_live) occ_draw_issued(); return; } @@ -9285,7 +9945,7 @@ void draw(VkCommandBuffer cmd, NvnState& s, const DrawArgs& d) { push_capture(cmd, p, s, d, at, topo, false); vkCmdDrawIndexed(cmd, quads * 6, (uint32_t)std::max(1, d.instances), 0, d.indexed ? d.base_vertex : 0, (uint32_t)d.base_instance); - st_draws++, pass().draws++; + st_draws++, ps.draws++; if (occ_live) occ_draw_issued(); return; } @@ -9311,28 +9971,42 @@ void draw(VkCommandBuffer cmd, NvnState& s, const DrawArgs& d) { } else if (d.indexed) { const uint32_t unit = d.itype == 0 ? 1 : d.itype == 1 ? 2 : 4; BufferRef ib; + // ((Triforce, 11-21, the performance audit's DESC-12) the buffer bound at 0 and the draw's offset as its first index, + // so that draws at different offsets of one buffer bind it once: index_rebase.h. RX_VK_INDEX_REBASE=0: at the draw's + // own offset, first index 0, as before) + static const bool rebase_on = index_rebase::on(getenv("RX_VK_INDEX_REBASE")); + uint32_t first_index = 0; { FINE_SCOPE(FT_VERTEX); ib = gpu_buffer_bind(d.ibuf, (uint64_t)std::max(d.count, 1) * unit, false); if (!ib) SKIP_HERE(); FINE_N(FC_IBIND, 1); - bind_index(cmd, ib.buffer, ib.offset, itype); + const bool rebased = rebase_on && index_rebase::rebase((uint64_t)ib.offset, unit, &first_index); + bind_index(cmd, ib.buffer, rebased ? 0 : ib.offset, itype); } push_capture(cmd, p, s, d, at, topo, true); FINE_SCOPE(FT_DRAWCALL); - vkCmdDrawIndexed(cmd, (uint32_t)d.count, (uint32_t)std::max(1, d.instances), 0, d.base_vertex, (uint32_t)d.base_instance); + vkCmdDrawIndexed(cmd, (uint32_t)d.count, (uint32_t)std::max(1, d.instances), first_index, d.base_vertex, (uint32_t)d.base_instance); } else { push_capture(cmd, p, s, d, at, topo, true); FINE_SCOPE(FT_DRAWCALL); vkCmdDraw(cmd, (uint32_t)d.count, (uint32_t)std::max(1, d.instances), (uint32_t)d.first, (uint32_t)d.base_instance); } - st_draws++, pass().draws++; + st_draws++, ps.draws++; if (occ_live) occ_draw_issued(); } // ((Triforce, redrawn.h) asked at every draw: a walk of the stages' few descriptor lists. Not kept in the Program: a field // there changes sizeof(Program), which the program cache's stamp holds (vk_shader.cpp disk_stamp), and every player's // program and pipeline caches would be thrown away at the update (272 -> 280 bytes did that, 11-18)) static bool program_writes_memory(const Program& p) { return writes_memory(p); } +/// ((Triforce, 11-22, PIPE-30; translate_wait.h) A translated graphics program's key noted as writing memory or not, where +/// RX_VK_TRANSLATE_WAIT_MS is set: graphics_pipeline's new entry and the precompile's loads. (In the unnamed namespace its +/// declaration above graphics_pipeline is in) +namespace { +void translate_note(const Program& p) { + if (translate_wait_ms() && !p.failed && !p.compute) translate_notes().note(p.key, writes_memory(p)); +} +} // namespace /// A dispatch over 2 ms is logged, 60 times, and only while RX_VK_SLOW_DISPATCH is set. Off, the clock is not read. bool slow_dispatch_timing() { @@ -9499,7 +10173,14 @@ bool upscale_stub(VkCommandBuffer cmd, const NvnState& s, uint64_t key) { if (a > in_area) in = t, in_area = a, in_h = s.tex[5][i]; } if (!in_area) return declined("it samples no texture of its output's kind that it does not also write"); - if (out_area <= in_area) return declined("its output is no larger than its input: not an upscale's last pass"); + // ((Triforce, 11-22, the performance audit's GPUWORK-21, after Source B's and Source A's; stub_same_size.h) a gameplay + // network's output as large as its input is taken too: 9dc98871eb7fc101, the network of Render resolution 100%, where it + // was left out or did not translate. Declined, nothing wrote the picture: a black 3D under the HUD. Any other program of + // that shape is still declined. RX_VK_STUB_SAME_SIZE=0: declined, as before (read once) + static const bool same_size = stub_same_size::on(getenv("RX_VK_STUB_SAME_SIZE")); + const bool gameplay = gameplay_network(key); + if (!stub_same_size::sized(out_area, in_area, gameplay, same_size)) + return declined("its output is no larger than its input: not an upscale's last pass"); // (Triforce, the parallel replay: a segment's worker records this dispatch too, and the texture cache is the recording // lock's (ParLock), as for every other caller: texture_get may make a texture or upload one, which changes the cache's // tables and the staging arenas under any other worker's hands. Not held across the temporal upscaler below, which takes @@ -9552,8 +10233,11 @@ bool upscale_stub(VkCommandBuffer cmd, const NvnState& s, uint64_t key) { } // Failed upscales can receive RGB9E5 bits in a shared R11 image, including the 75% gameplay TAAU. // Match the observed base-level 2D case only; other fallbacks keep their existing blit. + // ((Triforce, 11-22, GPUWORK-21) and the other gameplay network too: the typed upscale reads the scene as what it is and + // undoes the network's square-root encoding, where the blit below copies the encoded bits. RX_VK_STUB_SAME_SIZE=0: the + // 3/4 network alone, as before) const bool gameplay_taau = key == 0xfd029a317a9a4b94ull; - if ((!gameplay_network(key) || gameplay_taau) && in.d.format == NVN_FORMAT_RGB9E5F && out.d.format == NVN_FORMAT_R11G11B10F && + if (stub_same_size::typed(gameplay, gameplay_taau, same_size) && in.d.format == NVN_FORMAT_RGB9E5F && out.d.format == NVN_FORMAT_R11G11B10F && src->format == VK_FORMAT_B10G11R11_UFLOAT_PACK32 && dst->format == VK_FORMAT_B10G11R11_UFLOAT_PACK32 && (fd.optimalTilingFeatures & VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT) && src->type == VK_IMAGE_TYPE_2D && dst->type == VK_IMAGE_TYPE_2D && src->depth == 1 && dst->depth == 1 && @@ -9592,7 +10276,7 @@ bool upscale_stub(VkCommandBuffer cmd, const NvnState& s, uint64_t key) { } } } - if (gameplay_taau && in.d.format == NVN_FORMAT_RGB9E5F && out.d.format == NVN_FORMAT_R11G11B10F && + if (stub_same_size::warns(gameplay, gameplay_taau, same_size) && in.d.format == NVN_FORMAT_RGB9E5F && out.d.format == NVN_FORMAT_R11G11B10F && src->format == VK_FORMAT_B10G11R11_UFLOAT_PACK32 && dst->format == VK_FORMAT_B10G11R11_UFLOAT_PACK32) { static std::atomic warned_gameplay_upscale{false}; if (!warned_gameplay_upscale.exchange(true, std::memory_order_relaxed)) @@ -9621,6 +10305,13 @@ bool upscale_stub(VkCommandBuffer cmd, const NvnState& s, uint64_t key) { return true; } +/// ((Triforce, 11-22, the performance audit's DESC-30) The lab's "diagold" phases (RX_VK_LAB_AB=diagold): a dispatch's +/// diagnostics as they were before DESC-30, for measuring it inside one run: the keys written out as text and compared as +/// text (taau_key, RX_VK_LOG_PUSH), a clock pair around every program pick, the first-pipeline clock read at every dispatch. +/// Nothing reads it without that variable +bool dispatch_diag_old() { + return lab_ab::on("diagold"); +} /// (hookshot) OoT's gameplay TAAU network (fd029a317a9a4b94) stood in for: its input upscaled into its output /// (shaders/taau_upscale.comp). The network's translation loses the blue channel, the yellow cast over every /// scene; with this the picture is right, without the network's temporal anti-aliasing. @@ -9631,6 +10322,42 @@ bool taau_key(uint64_t key) { static const char* keys = getenv("RX_VK_TAAU_KEYS") ? getenv("RX_VK_TAAU_KEYS") : Shader::Backend::SPIRV::mma_software || getenv("RX_VK_NO_F16") ? "fd029a317a9a4b94" : "-"; // (RX_VK_NO_F16: vk_shader.cpp leaves the network out) + // ((Triforce, 11-22, the performance audit's DESC-30, after Source A's and Source B's) The list parsed once into (value, + // mask) prefixes, as RX_VK_SKIP_PROG's is in draw(): this is asked several times at every compute dispatch, and each time + // the key was written out as text and the list walked. The same answers: an entry is up to 16 lower-case hex digits and + // matches the keys that begin with them; an empty one, one that begins with '-', one with any other character or more + // than 16 never matches (the text compare ran into the key's end, or met a character a key's text never has) + struct Prefix { + uint64_t value, mask; + }; + static const std::vector prefixes = [] { + std::vector list; + for (const char* q = keys; *q;) { + const char* e = q; + while (*e && *e != ',') e++; + const size_t n = (size_t)(e - q); + uint64_t v = 0; + bool hex_only = n > 0 && n <= 16; + for (size_t i = 0; hex_only && i < n; i++) { + const char c = q[i]; + if (c >= '0' && c <= '9') v = v << 4 | (uint64_t)(c - '0'); + else if (c >= 'a' && c <= 'f') v = v << 4 | (uint64_t)(c - 'a' + 10); + else hex_only = false; + } + if (hex_only) { + const unsigned shift = 4 * (16 - (unsigned)n); + list.push_back({v << shift, ~0ull << shift}); + } + q = *e ? e + 1 : e; + } + return list; + }(); + if (!dispatch_diag_old()) { + for (const Prefix& p : prefixes) + if ((key & p.mask) == p.value) return true; + return false; + } + // (the lab's "diagold" phases: the text compare, as it was) char hex[24]; snprintf(hex, sizeof hex, "%016llx", (unsigned long long)key); bool match = false; @@ -9890,10 +10617,29 @@ static bool temporal_rejected_network(VkCommandBuffer cmd, uint64_t key, Texture (unsigned long long)key, src->width, src->height, dst->width, dst->height); return true; } +/// ((Triforce, 11-22, the performance audit's GPUWORK-20, after Source A's) The upscaler Off: RX_VK_UPSCALE_OFF=1, or the +/// lab's "upoff" phases. Every gameplay upscale network is left out and its input stretched into its output +/// (upscale_off.h). It is reached only while the game's own upscaler draws: DLSS, FSR 3.1 and FSR 4 take the dispatch +/// first (compute_dispatch's order). The compile skips keep asking taau_key alone, so the networks are made as before. +static bool upscale_off_on() { + static const bool env = getenv("RX_VK_UPSCALE_OFF") && atoi(getenv("RX_VK_UPSCALE_OFF")) > 0; + return env || lab_ab::on("upoff"); +} bool taau_stand_in(VkCommandBuffer cmd, const NvnState& s, const Program& p) { - if (!taau_key(p.key) || !dev.push_descriptor || !vkCmdPushDescriptorSetKHR || p.stages.empty()) return false; + if (!dev.push_descriptor || !vkCmdPushDescriptorSetKHR || p.stages.empty()) return false; const Stage& st = p.stages[0]; + const bool off = upscale_off_on(); + const bool shaped = off && st.info.uses_cooperative_matrix && st.info.texture_descriptors.size() == 1 && + st.info.image_descriptors.size() == 1; + const upscale_off::Choice ch = upscale_off::choose(taau_key(p.key), off, gameplay_network(p.key), shaped); + if (!ch.stand_in) return false; if (st.info.texture_descriptors.empty() || st.info.image_descriptors.empty()) return false; + if (ch.by_shape) { // (a network of that shape that is not a gameplay upscale, a movie's: its input is not the scene's format) + Texture* in = nullptr; + const uint32_t in_handle = handle_of(s, st, st.info.texture_descriptors[0], 0); + if (in_handle) texture_view_of(in_handle, VK_IMAGE_VIEW_TYPE_2D, ViewUse::Sample, VK_FORMAT_UNDEFINED, &in); + if (!in || in->format != VK_FORMAT_E5B9G9R9_UFLOAT_PACK32) return false; + } size_t ntex = 0; // (build_descriptors' image infos: the textures', then the images') for (const auto& d : st.info.texture_descriptors) ntex += d.count; if (dw().imgs.size() <= ntex || !dw().imgs[0].imageView || !dw().imgs[ntex].imageView) return false; @@ -9904,11 +10650,11 @@ bool taau_stand_in(VkCommandBuffer cmd, const NvnState& s, const Program& p) { if (!handle) return false; texture_view_of(handle, VK_IMAGE_VIEW_TYPE_2D, ViewUse::Storage, image_format(od.format), &out, &level); if (!out || out->format != VK_FORMAT_B10G11R11_UFLOAT_PACK32) return false; - const char* keys = getenv("RX_VK_TAAU_KEYS") ? getenv("RX_VK_TAAU_KEYS") : "fd029a317a9a4b94"; + const char* keys = !ch.temporal ? "the upscaler Off" : getenv("RX_VK_TAAU_KEYS") ? getenv("RX_VK_TAAU_KEYS") : "fd029a317a9a4b94"; bool done; { ParLock lk; // (Triforce: typed_upscale's pipeline and history images are shared by every recording thread: upscale_stub) - done = typed_upscale(cmd, dw().imgs[0].imageView, dw().imgs[ntex].imageView, out, level, p.key, keys, true); + done = typed_upscale(cmd, dw().imgs[0].imageView, dw().imgs[ntex].imageView, out, level, p.key, keys, ch.temporal); } if (done) hudless_after_dispatch(cmd, p.key); // ((Triforce, FG-2) the picture it wrote in the network's place, before the HUD) return done; @@ -10152,12 +10898,16 @@ constexpr uint32_t kWarmRecipe = 2; // (2: the libraries are in our pipeline c // selects of AMD's older compiler, spv_nan_select.h: those modules are new to every cache of the driver's), kRateRecipe in the // driver's part where warm_libraries also makes the shading rate's libraries (pipeline_rate). Counted up as kWarmRecipe is) constexpr uint32_t kModulePatches = 1, kRateRecipe = 1; +// ((Triforce, 11-21) and kMeshOptRecipe where the task and mesh pipelines are built with the optimizer by the RADV rule +// (mesh_opt_recipe): RADV with native mesh shaders, RX_VK_MESH_OPT unset. Those pipelines are not what the driver's caches hold +// from a build without it, so the driver's part is made again before the title, not met cold in play) +constexpr uint32_t kMeshOptRecipe = 1; uint32_t module_patches() { static const bool nan_selects = spv_nan_select::on(getenv("RX_VK_NAN_SELECTS")); // (as make_module reads it) // ((Triforce, 11-19) and 2: the water's simplification, v0.5.2's RX 470 .. 590 port, which came without a new stamp: shader_water_mode) return (dev.fixes.nan_bits && nan_selects ? 1u : 0u) | (shader_water_mode() ? 2u : 0u); } -uint64_t warmed_stamp_of(bool libraries, uint32_t patches, bool rate) { +uint64_t warmed_stamp_of(bool libraries, uint32_t patches, bool rate, bool mesh_opt) { uint64_t h = program_cache_stamp(); const auto add = [&h](const void* d, size_t n) { for (size_t i = 0; i < n; i++) h = (h ^ ((const uint8_t*)d)[i]) * 1099511628211ull; @@ -10171,28 +10921,29 @@ uint64_t warmed_stamp_of(bool libraries, uint32_t patches, bool rate) { add(p, sizeof p); } if (rate) add(&kRateRecipe, sizeof kRateRecipe); + if (mesh_opt) add(&kMeshOptRecipe, sizeof kMeshOptRecipe); return h; } /// What the pipelines built for the driver's caches are good for (the light precompile's record, the predicted pipelines') uint64_t warmed_stamp(bool libraries) { - return warmed_stamp_of(libraries, module_patches(), false); + return warmed_stamp_of(libraries, module_patches(), false, mesh_opt_recipe()); } /// What the driver's part is good for (prepare_run, triforce_prepared.bin's second stamp): warmed_stamp's, and with the stage /// libraries on a device with dev.foliage_rate, the shading rate's libraries of the native mesh programs in it too (a folder /// warmed by a build that made none of them is warmed once more: the libraries it has come back from the cache) uint64_t libraries_stamp(bool libraries) { - return warmed_stamp_of(libraries, module_patches(), libraries && dev.foliage_rate); + return warmed_stamp_of(libraries, module_patches(), libraries && dev.foliage_rate, mesh_opt_recipe()); } /// ((Triforce) What a pipeline cache file of this session's own holds pipelines for (.own, pcache_own_mark): prepare_run's /// driver part's stamp uint64_t pcache_own_stamp() { return libraries_stamp(gpl_wanted() && !getenv("RX_VK_NO_WARM")); } -/// ((Triforce) The skip list of the driver's part is its stamp's (gpl_rules.h skip_read). Where the stamp changed by the two +/// ((Triforce) The skip list of the driver's part is its stamp's (gpl_rules.h skip_read). Where the stamp changed by the /// words above alone, it is the same driver with the same programs, and the variants it crashed on are left out still: carried /// over from the stamp it was, rather than met again in the driver's hands (one start lost to each) void warm_skip_carry(const std::string& skip, bool libraries, uint64_t now) { if (!gpl_rules::skip_read(skip, now).empty()) return; - for (int k = 0; k < 4; k++) { - const uint64_t was = warmed_stamp_of(libraries, k & 1 ? module_patches() : 0u, libraries && (k & 2)); + for (int k = 0; k < 8; k++) { + const uint64_t was = warmed_stamp_of(libraries, k & 1 ? module_patches() : 0u, libraries && (k & 2), (k & 4) != 0); if (was == now) continue; const std::vector ids = gpl_rules::skip_read(skip, was); if (ids.empty()) continue; @@ -10679,7 +11430,7 @@ bool prepare_compute_pipeline(Program& p) { ci.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT; ci.stage.module = p.stages[0].module; ci.stage.pName = "main"; - pin_subgroup(ci.stage, p.workgroup[0]); + pin_compute(ci.stage, p); ci.layout = p.layout; if (getenv("RX_VK_NOOPT")) ci.flags |= VK_PIPELINE_CREATE_DISABLE_OPTIMIZATION_BIT; VkPipeline pl = VK_NULL_HANDLE; @@ -10697,6 +11448,9 @@ bool prepare_compute_pipeline(Program& p) { if (built) { vkDestroyPipeline(dev.device, pl, nullptr); pcache_built_since_save++; + // ((Triforce, 11-22, PIPE-23) and for the main file's save: built straight into the main cache, it was counted nowhere + // a save looks, and a save with a play cache left the main file out (pcache_play.h save_what)) + if (precompile_save::on(precompile_save_s())) pcache_main_built++; } if (p.layout) vkDestroyPipelineLayout(dev.device, p.layout, nullptr), p.layout = VK_NULL_HANDLE; if (p.set_layout) vkDestroyDescriptorSetLayout(dev.device, p.set_layout, nullptr), p.set_layout = VK_NULL_HANDLE; @@ -10758,7 +11512,7 @@ bool prepare_run(void (*progress)(size_t done, size_t total), bool background) { const std::string path = prepared_path(); PreparedFor prepared_for; double hash_ms = 0; - const PrepareWhy why = prepare_check(path, seed_path(), program_cache_stamp(), &prepared_for, &hash_ms); + PrepareWhy why = prepare_check(path, seed_path(), program_cache_stamp(), &prepared_for, &hash_ms); const bool libraries = gpl_wanted() && !getenv("RX_VK_NO_WARM"); const uint64_t warmed = libraries_stamp(libraries); const std::string warming = path + ".warming"; @@ -10774,6 +11528,20 @@ bool prepare_run(void (*progress)(size_t done, size_t total), bool background) { // the compute pipelines the carried run built, again, from the program cache. The record says the shaders are translated // and their driver part is not made, so the stage libraries follow below, and nothing is translated again) if (!background && prepare_cs_again.exchange(false) && !warm_kept_computes(progress)) return false; + // ((Triforce, 11-22, after Casual Gamer's fix) recorded as prepared, but the program cache may have been made again since for + // another GPU or other settings: one of another stamp, or of its head alone, holds none of these shaders (prepare_again.h). + // Prepared again, the stale record removed first, so that a start closed in it goes on as from a first start) + static const bool recheck = prepare_again::on(getenv("RX_VK_PREPARE_RECHECK")); + if (why == PrepareWhy::None && !background && recheck && !getenv("RX_VK_NO_PROGRAM_CACHE")) { + const std::string programs = path.substr(0, path.find_last_of("\\/") + 1) + "triforce_programs.bin"; + uint64_t size = 0; + if (prepare_again::programs_none_at(programs, program_cache_stamp(), &size)) { + logf("vk: the shaders are recorded as prepared, but the program cache (%llu KB) holds none of them (made again for another " + "GPU or other settings since): preparing again (RX_VK_PREPARE_RECHECK=0: not)", (unsigned long long)(size >> 10)); + remove(path.c_str()); + why = PrepareWhy::First; + } + } if (why == PrepareWhy::None && !getenv("RX_VK_PREPARE_AGAIN")) { // (perf push 11-11: a shipped state list the notes were not made from, in a folder prepared already: its pipelines are // noted now, for the precompile that follows, as a prepare would have. See state_shipped_id) @@ -10847,12 +11615,20 @@ bool prepare_run(void (*progress)(size_t done, size_t total), bool background) { // ((Triforce) with the stage libraries the part has a journal, and a start goes on with what the last one did not // end: warm_begin, gpl_rules.h. Without them it is as it was, below) if (!warm_begin(warming, warmed)) warm = warm_cs = false; - } else if (FILE* f = fopen(warming.c_str(), "rb")) { - fclose(f); - warm = warm_cs = false; - logf("vk: the last start did not finish the driver's part of the shaders: left out"); - } else if (warm || warm_cs) { - if (FILE* w = fopen(warming.c_str(), "wb")) fclose(w); + } else { + // ((Triforce, 11-22, after Casual Gamer's fix) the mark holds what the part is made for (warmed: the GPU, the driver, the + // programs), as the libraries' journal does: another stamp's mark is another driver's, and the part goes on here. An empty + // one, an older build's, is this driver's as before (prepare_again.h; RX_VK_PREPARE_RECHECK=0: any mark, as before) + const prepare_again::Mark left = prepare_again::mark_left(warming, warmed, recheck); + if (left == prepare_again::Mark::Ours) { + warm = warm_cs = false; + logf("vk: the last start did not finish the driver's part of the shaders: left out"); + } else { + if (left == prepare_again::Mark::Other) + logf("vk: the last start did not finish the driver's part of the shaders for another GPU, driver or programs: made " + "for these now"); + if (warm || warm_cs) prepare_again::mark_write(warming, warmed, recheck); + } } pcache_init(); // (once, here: the workers below build compute pipelines into it) std::atomic next{0}, done{0}, made{0}, libs{0}, variants{0}, asking{0}, capped{0}; @@ -10871,8 +11647,22 @@ bool prepare_run(void (*progress)(size_t done, size_t total), bool background) { } // (hookshot) compute pipelines too, for the driver's own cache: built in play they stopped the game while the driver compiled // them, seconds each for the culling programs (53caac5f: 6.5 s, 41 s inlined), which looked like a hang + // ((Triforce, the Intel logs of 11-21) on Intel only the programs play dispatched (triforce_computes.bin) once there is a + // list: an Arc A750 spent 5175 s of this pass's 5858 s on compute pipelines, a tenth of which play asks for. warm_computes.h; + // RX_VK_WARM_COMPUTES=listed or all) + const bool cs_listed_only = warm_cs && warm_computes::listed_only(getenv("RX_VK_WARM_COMPUTES"), dev.props.vendorID); + const std::unordered_set cs_listed = cs_listed_only ? computes_read() : std::unordered_set{}; + std::atomic cs_left{0}; + if (cs_listed_only) + logf("vk: the preparation's compute pipelines: %s (RX_VK_WARM_COMPUTES=all: every one, =listed: only those)", + cs_listed.empty() ? "every one, as no list of the programs play dispatched is there yet" + : "only the programs play dispatched (triforce_computes.bin), the others when play first does"); const auto warm_compute = [&](Program& p) { if (!warm_cs || taau_key(p.key)) return; + if (!warm_computes::build(cs_listed_only, cs_listed.empty(), cs_listed.count(p.key) != 0)) { + cs_left++; + return; + } // ((Triforce) named for the carry's rebuild, which builds it again into a fresh cache (warm_kept_computes); and not // built into a carried cache past what it may hold: that one is let go, pcache_carry.h) if (pcache_carried) { @@ -10923,7 +11713,7 @@ bool prepare_run(void (*progress)(size_t done, size_t total), bool background) { } }; std::vector threads; - const unsigned cpus = std::max(1u, std::thread::hardware_concurrency()); + const unsigned cpus = std::max(1u, cpu_count()); // (in play: a quarter of the CPUs, two to four, the game's own threads before them. Before the title nothing of the game // runs: prepare_workers, every processor where the PC's memory and the process's priority allow it; (Triforce) fewer on // AMD's driver for RDNA 1 and 2, prepare_translate_workers) @@ -10958,6 +11748,9 @@ bool prepare_run(void (*progress)(size_t done, size_t total), bool background) { logf("vk: prepared %zu shader programs%s (%zu new, %lld failed, %zu compute pipelines; %zu compute variants, %zu programs with " "texture variants, %zu at the limit) in %.1f s", programs, background ? " in play" : "", made.load(), (long long)failed, libs.load(), variants.load(), asking.load(), capped.load(), std::chrono::duration(std::chrono::steady_clock::now() - t0).count()); + if (cs_left.load()) + logf("vk: the preparation left %zu compute variants of programs play has not dispatched to their first dispatch " + "(%zu programs listed; RX_VK_WARM_COMPUTES=all builds them)", cs_left.load(), cs_listed.size()); if (warm) { // (translated, whatever becomes of the libraries: a start that is closed in them goes on with them alone) prepared_for.warmed = 0; @@ -10968,6 +11761,10 @@ bool prepare_run(void (*progress)(size_t done, size_t total), bool background) { } } if (!background) warm_end(warming); + // ((Triforce, 11-22, the performance audit's PIPE-23; precompile_save.h) a preparation without stage libraries built its + // compute pipelines into the main cache and ended without a save, where warm_libraries saves after its own: on the disk + // now, before the record says the part is made. The workers are joined. RX_VK_PRECOMPILE_SAVE_S=0: not, as before) + if (precompile_save::prepare_saves(precompile_save_s(), warm, background, libs.load())) pcache_saved_now(); // (the driver's part left out, by a start that ended in it or in play, is not tried again for this driver: as before. // (Triforce) With the stage libraries that is after gpl_rules::kTries such starts in a row: warm_begin) prepared_for.warmed = warmed; @@ -11106,6 +11903,20 @@ bool precompile_keeps_all() { uint64_t ws = 0, priv = 0; memory_process(&ws, &priv); // ((Triforce, 11-18) less what the pipeline cache's read added, but on AMD's driver: mem_parts.h own_for_keep) + // ((Triforce, 11-22, after Casual Gamer's fix) On a discrete GPU in a PC of under 20 GB, every pipeline is kept while the + // game is told its mapped pages are used (host/hos_mem.c query_used, HOS_MEM_QUERY_USED): it lets old areas go since, + // and the RAM the rule below spared costs less than the pipelines left to their first draw. A 16 GB laptop: ~0.6 GB more + // private memory, frames of 1 s or more 15 and 11 in two tours -> 3 and 6. Integrated GPUs keep the RAM rule (the + // pipelines' memory is the RAM the GPU has too); RX_VK_PRECOMPILE_KEEP=auto: the RAM rule, as before) + const bool discrete = dev.props.deviceType == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU; + const bool query_on = + mem_parts::query_used_on(getenv("HOS_MEM_QUERY_USED"), getenv("HOS_MEM_CENSUS"), getenv("HOS_MEM_GIVE_BACK")); + if (!e && mem_parts::keep_all_small_pc(discrete, ms.ullTotalPhys, query_on)) { + logf("vk: precompile: the pipelines it builds are kept for the session: a discrete GPU in a PC of %llu MB (under 20 GB), " + "the game told its mapped pages are used (RX_VK_PRECOMPILE_KEEP=auto: by the RAM available, =0: never)", + (unsigned long long)(ms.ullTotalPhys >> 20)); + return true; + } const bool amd = dev.props.vendorID == 0x1002; const uint64_t cache_read = pcache_read_ws.load(), own = mem_parts::own_for_keep(ws, cache_read, amd); const bool keep = mem_parts::keep_all_pipelines(ms.ullTotalPhys, ms.ullAvailPhys, own); @@ -11136,7 +11947,157 @@ void built_write(uint64_t stamp, const std::unordered_set& keys) { all.insert(all.end(), keys.begin(), keys.end()); if (FILE* f = fopen((pipe_notes_path() + ".built").c_str(), "wb")) fwrite(all.data(), 8, all.size(), f), fclose(f); } +// ---- (Triforce, 11-22, the performance audit's PIPE-21, after Source A's and Source B's) Whether the caches still have what +// .built says the light precompile built: built_check.h has why, the rules and what a start does when they do not ---- +int built_check_next = 0; // (the strikes recorded once everything is built again: built_check::decide, built_check_made) +VkComputePipelineCreateInfo compute_create_info(const Program& q); +/// A light precompile's, at a start whose record (built, for stamp) is not empty, before it is trusted: up to +/// built_check::kGraphics of the recorded notes asked of our caches as a draw asks, and with the record's compute key +/// (computes_built) up to built_check::kComputes of the listed compute programs as a dispatch asks; without the flag that +/// asks without compiling, the notes built twice and timed. True: cold, and everything is built again now, before the title +/// (the caller forgets the record for this start). RX_VK_BUILT_CHECK=0: never looked at +bool built_check_cold(const std::vector>& notes, const std::unordered_set& built, + uint64_t (*key_of)(const PipeNote&), bool computes_built, const std::vector& computes_listed, + uint64_t stamp) { + const int mode = built_check_mode(); + if (!mode || built.empty()) return false; + const auto t0 = std::chrono::steady_clock::now(); + const auto ms = [](std::chrono::steady_clock::duration d) { return std::chrono::duration(d).count(); }; + const bool ask = dev.vk13.pipelineCreationCacheControl; + std::vector recorded; + for (const auto& n : notes) + if (built.count(key_of(*n))) recorded.push_back(n.get()); + std::vector probes; + const uint64_t seed = (uint64_t)t0.time_since_epoch().count() / 1000; + int graphics = 0, computes = 0; + for (const size_t i : built_check::spots(recorded.size(), (size_t)built_check::kLook, seed)) { + if (graphics >= built_check::kGraphics || prepare_stopped()) break; + const PipeNote& n = *recorded[i]; + bool tried = false; + // (a copy of the program read for it and let go: the precompile has read none yet, and those in memory are not ours) + program_cache_borrow(n.prog, [&](Program& p) { + if (tried || p.variant != n.variant || p.failed || p.compute || p.stages.empty()) return; + tried = true; + ensure_layout(p); + built_check::Probe pr; + pr.asked = ask, pr.made = true; + for (int again = 0; again < (ask ? 1 : 2); again++) { + bool cold = false; + const auto b0 = std::chrono::steady_clock::now(); + const VkPipeline pl = build_graphics(p, n.s, note_targets(n), (VkPrimitiveTopology)n.topo, -1, ask ? &cold : nullptr); + (again ? pr.again_ms : pr.first_ms) += ms(std::chrono::steady_clock::now() - b0); + if (pl) vkDestroyPipeline(dev.device, pl, nullptr); + else { // (cold, or refused: neither says the caches have it) + pr.answered_cold = cold, pr.made = false; + break; + } + } + probes.push_back(pr); + }); + graphics += tried; + } + // (the record's compute programs, asked as a dispatch asks: only where the caches can be asked without compiling) + if (computes_built && ask) + for (const size_t i : built_check::spots(computes_listed.size(), (size_t)built_check::kComputes, seed)) { + if (prepare_stopped()) break; + bool tried = false; + program_cache_borrow(computes_listed[i], [&](Program& p) { + if (tried || p.failed || !p.compute || p.stages.empty()) return; + tried = true; + ensure_layout(p); + VkComputePipelineCreateInfo ci = compute_create_info(p); + ci.flags |= VK_PIPELINE_CREATE_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT; + VkPipeline pl = VK_NULL_HANDLE; + built_check::Probe pr; + pr.asked = true; + const auto b0 = std::chrono::steady_clock::now(); + const VkResult r = cached_compute(pcache_use().cache, &ci, &pl); + pr.first_ms = ms(std::chrono::steady_clock::now() - b0); + if (r == VK_SUCCESS && pl) pr.made = true, vkDestroyPipeline(dev.device, pl, nullptr); + else pr.answered_cold = r == VK_PIPELINE_COMPILE_REQUIRED; + probes.push_back(pr); + }); + computes += tried; + } + const built_check::Tally t = built_check::tally(probes); + const built_check::Verdict v = built_check::verdict(t); + const std::string state = pipe_notes_path() + ".built.check"; + const built_check::State had = built_check::state_read(state); + const built_check::Do d = built_check::decide(v, had, stamp, mode == 2, &built_check_next); + const double took = ms(std::chrono::steady_clock::now() - t0); + if (d == built_check::Do::Again) { + logf("vk: precompile's record: %d of %d pipelines asked of the caches were not there (%d graphics, %d compute asked; %s) " + "in %.0f ms, though the record lists them as built: the caches lost them (the pipeline cache's files gone or " + "refused, the driver's own cache cleared or full). Everything is built again now, before the title%s " + "(RX_VK_BUILT_CHECK=0: not looked at)", t.cold, t.got, graphics, computes, + ask ? "asked without compiling" : "each built twice and timed", took, mode == 2 ? " (RX_VK_BUILT_CHECK=2)" : ""); + return true; + } + if (v == built_check::Verdict::Unknown) { + logf("vk: precompile's record: %d answers of the caches, too few to tell whether they still have what it lists (%.0f ms)", + t.got, took); + } else if (v == built_check::Verdict::Warm) { + if (had.found) remove(state.c_str()); // (kept this time: a strike of an earlier loss is forgotten) + logf("vk: precompile's record: the caches have what it lists (%d of %d asked were not there; %d graphics, %d compute " + "asked) in %.0f ms (RX_VK_BUILT_CHECK=0: not looked at)", t.cold, t.got, graphics, computes, took); + } else { + logf("vk: precompile's record: the caches lost what it lists again (%d of %d asked) right after it was built again: " + "they do not keep it on this PC, and those pipelines are left to their first draws, as before. The driver's own " + "shader cache may be too small for them (on NVIDIA: the control panel's Shader Cache Size). Remove %s to try " + "again", t.cold, t.got, state.c_str()); + } + return false; +} +/// precompile_run's, after built_check_cold said cold and everything was built and recorded again: so that a start that +/// finds them cold again right after leaves them to the draws (built_check::decide) +void built_check_made(uint64_t stamp) { + const std::string state = pipe_notes_path() + ".built.check"; + if (!built_check::state_write(state, stamp, built_check_next)) logf("vk: %s cannot be written", state.c_str()); +} +/// The end of a light precompile that went by its record: the record's keys and, when it lists them, the compute programs, +/// for the counter of play builds (built_play_build). Once +void built_play_publish(const std::unordered_set& built, uint64_t computes_key, const std::vector& computes_listed) { + if (!built_check_mode() || built.empty() || built_play_keys.load(std::memory_order_acquire)) return; + std::vector all(built.begin(), built.end()); + if (built.count(computes_key)) all.insert(all.end(), computes_listed.begin(), computes_listed.end()); + const std::vector* keys = new std::vector(built_check::sorted(std::move(all))); + const std::vector* none = nullptr; + if (!built_play_keys.compare_exchange_strong(none, keys)) delete keys; +} +/// ((Triforce, 11-22, the performance audit's PIPE-23, after Source A's; precompile_save.h) The end of a precompile that +/// keeps its pipelines, after build_s seconds of building: from RX_VK_PRECOMPILE_SAVE_S seconds (30) the pipeline cache is +/// saved now, before the title, and the log says what that wrote or why nothing was. It was first saved in play, at +/// present 300 after 32 more builds, on Intel only at a clean close: a crash or a kill before that cost the whole build +/// again. The play file alone while there is a play cache (pcache_play_only_now); never while workers still build +/// (gone_on), after a stop, or in a start that only prepares, which save by themselves +void precompile_end_save(double build_s, bool gone_on) { + const int save_s = precompile_save_s(); + if (!precompile_save::long_build(build_s, save_s)) return; + const bool stopped = prepare_stopped(), prepare_only = prepare_only_start(); + if (!pcache || !precompile_save::due(build_s, save_s, gone_on, stopped, prepare_only)) { + logf("vk: precompile: %.0f s of building: %s (RX_VK_PRECOMPILE_SAVE_S)", build_s, + precompile_save::said(precompile_save::why_not(gone_on, stopped, prepare_only, pcache != VK_NULL_HANDLE))); + return; + } + const size_t main_was = pcache_saved_size.load(), play_was = pcache_play_saved.load(); + const auto from = std::chrono::steady_clock::now(); + { + struct Now { // (pipeline_cache_save's play-only save on this thread: pcache_play_only_now) + Now() { pcache_play_only_now = true; } + ~Now() { pcache_play_only_now = false; } + } now; + pipeline_cache_save(true); + } + const size_t main_now = pcache_saved_size.load(), play_now = pcache_play_saved.load(); + const size_t wrote = (main_now != main_was ? main_now : 0) + (play_now != play_was ? play_now : 0); + const double ms = std::chrono::duration(std::chrono::steady_clock::now() - from).count(); + if (wrote) + logf("vk: precompile: %.0f s of building: %s (%zu KB in %.0f ms; RX_VK_PRECOMPILE_SAVE_S=0: not)", build_s, + precompile_save::said(precompile_save::Outcome::Saved), wrote >> 10, ms); + else + logf("vk: precompile: %.0f s of building: %s", build_s, precompile_save::said(precompile_save::Outcome::Nothing)); +} void precompile_run(void (*progress)(size_t done, size_t total)) { const auto t0 = std::chrono::steady_clock::now(); // (a start that dies in here would die in here every time, the notes being what it builds from: a player saw the console reach @@ -11159,6 +12120,9 @@ void precompile_run(void (*progress)(size_t done, size_t total)) { if (FILE* f = fopen(running.c_str(), "wb")) fprintf(f, "%d\n", ended_inside + 1), fclose(f); struct Done { const std::string& path; ~Done() { remove(path.c_str()); } } done_marker{running}; pcache_init(); + // ((Triforce) the texture converters' pipelines first, inside the marker: a driver that crashes making them counts as a start + // that did not get through, and the second such start plays without them, as before. RX_VK_TEX_PIPES_AHEAD) + texture_pipelines_warm(); uint64_t ws0 = 0, priv0 = 0; // ((Triforce) what its programs and pipelines cost the RAM: the memory line after it) memory_process(&ws0, &priv0); std::vector note_at; // ((Triforce) each note's record in the file, for the warm: warm_index_set) @@ -11200,10 +12164,15 @@ void precompile_run(void (*progress)(size_t done, size_t total)) { const auto built_key = [](const PipeNote& n) { return dev.foliage_rate && n.n == 0 && depth_aspect((VkFormat)n.depth_format) ? mix(n.key, kRateLibrary) : n.key; }; + bool rebuilt = false; // ((Triforce, 11-22, PIPE-21) the record not trusted this start: built_check_cold) if (light) { built = built_read(built_stamp); computes_built = built.count(computes_key) != 0; - std::erase_if(notes, [&](const std::unique_ptr& n) { return built.count(built_key(*n)) != 0; }); + // ((Triforce, 11-22, the performance audit's PIPE-21) unless the caches lost it since, which a sample says + // (built_check.h). Cold, nothing is left out below: everything noted and listed is built again, and recorded anew) + if (!built.empty() && built_check_cold(notes, built, built_key, computes_built, computes_listed, built_stamp)) + built.clear(), computes_built = false, rebuilt = true; + std::erase_if(notes,[&](const std::unique_ptr& n) { return built.count(built_key(*n)) != 0; }); logf("vk: precompile: RAM spared (RX_VK_PRECOMPILE_KEEP=1: as before): %zu of %zu noted pipelines were built by " "an earlier start and are left to their first draw%s", noted_n - notes.size(), noted_n, computes_built ? ", the compute programs' too" : ""); @@ -11224,6 +12193,7 @@ void precompile_run(void (*progress)(size_t done, size_t total)) { std::vector computes, fallbacks; const size_t programs_n = program_cache_load_some(wanted, [&](Program& p) { ensure_layout(p); + translate_note(p); // ((Triforce, 11-22, PIPE-30) whether its key writes memory, for the bounded wait) if (p.compute) computes.push_back(&p); else if (p.fallback) fallbacks.push_back(&p); // (hookshot: the mesh fallback's compute pipelines too, before play) }, all_computes, true); @@ -11281,7 +12251,7 @@ void precompile_run(void (*progress)(size_t done, size_t total)) { ci.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT; ci.stage.module = p.stages[0].module; ci.stage.pName = "main"; - pin_subgroup(ci.stage, p.workgroup[0]); + pin_compute(ci.stage, p); ci.layout = p.layout; VkPipeline made = VK_NULL_HANDLE; const auto built_from = std::chrono::steady_clock::now(); @@ -11344,7 +12314,8 @@ void precompile_run(void (*progress)(size_t done, size_t total)) { } }; std::vector threads; - const unsigned cpus = std::max(1u, std::thread::hardware_concurrency()); + const unsigned cpus = std::max(1u, cpu_count()); + const auto build_from = std::chrono::steady_clock::now(); // ((Triforce, 11-22, PIPE-23) how long it builds: precompile_end_save) for (unsigned i = 0; i < std::min(cpus, 64u); i++) threads.emplace_back(worker); static const int stall_s = getenv("RX_VK_PRECOMPILE_STALL_S") ? atoi(getenv("RX_VK_PRECOMPILE_STALL_S")) : 30; size_t last_done = 0; @@ -11391,11 +12362,21 @@ void precompile_run(void (*progress)(size_t done, size_t total)) { built.insert(computes_key); pipeline_cache_save(true); built_write(built_stamp, built); + if (rebuilt) built_check_made(built_stamp); // ((Triforce, 11-22, PIPE-21) built again: recorded for the strikes) } + // ((Triforce, 11-22, PIPE-21) what the record says is in the caches, for the counter of play builds) + if (light && !prepare_stopped()) built_play_publish(built, computes_key, computes_listed); } + // ((Triforce, 11-22, the performance audit's PIPE-23) a precompile that keeps its pipelines saved nothing here: after a + // long build the cache is saved before the title (precompile_end_save). The light one saved above, as before) + if (!light) precompile_end_save(std::chrono::duration(std::chrono::steady_clock::now() - build_from).count(), gone_on); logf("vk: precompile: %zu programs, %zu graphics and %zu compute pipelines, %zu mesh fallback programs in %.0f ms (%.0f ms " "of it the caches opened, the notes and programs read)", programs_n, graphics_n, computes_n, fallbacks_n, std::chrono::duration(std::chrono::steady_clock::now() - t0).count(), read_ms); + // ((Triforce, 11-22, PIPE-52) how the rule fell so far, where it is asked at all (not with RX_VK_COMPUTE_PIN "all") + if (compute_pin_mode() != compute_pin::Mode::All) + logf("vk: precompile: compute pipelines held to 32 lanes: %lld, left to the driver's choice: %lld (RX_VK_COMPUTE_PIN=%s)", + (long long)st_pin_pinned.load(), (long long)st_pin_free.load(), compute_pin::name(compute_pin_mode())); { uint64_t ws1 = 0, priv1 = 0; int64_t modules = 0, module_bytes = 0, packed = 0, packed_bytes = 0; @@ -11485,13 +12466,27 @@ void predicted_done(uint64_t stamp, uint64_t key) { } } // namespace +// ((Triforce, 11-22, the performance audit's PIPE-44, after Source A's; predict_types.h) A program's predictions can give +// the same pipeline twice: two layouts that differ only in what is dynamic (a stride). Each is built once now, by its key as +// play keys it (pipeline_key): the same pipelines, fewer builds. And counted for the lines: the ones built once already, and +// the programs the programs' file did not lend (a load took their records off its list: their predictions are not built). +// RX_VK_PREDICT_TYPES=0: every one built, as before +std::atomic st_predict_twice{0}, st_predict_unlent{0}; +/// Whether predicted pipeline n of program variant p is still to build: seen, this program's keys so far +bool predict_first(std::unordered_set& seen, const Program& p, const PipeNote& n) { + static const bool once = predict_types::once(predict_types::mode(getenv("RX_VK_PREDICT_TYPES"))); + if (!once || seen.insert(pipeline_key(p, n.s, note_targets(n), (VkPrimitiveTopology)n.topo)).second) return true; + st_predict_twice.fetch_add(1, std::memory_order_relaxed); + return false; +} void predict_run(void (*progress)(size_t done, size_t total)) { const auto t0 = std::chrono::steady_clock::now(); const std::vector> notes = pipe_notes_read(); if (notes.empty() || getenv("RX_VK_NO_PREDICT")) return; size_t programs = 0; std::vector plan; - const std::vector> predicted = predict_pipelines(notes, &programs, &plan); + PredictCounts left; // ((Triforce, 11-22, PIPE-44) what the prediction left out, for its line) + const std::vector> predicted = predict_pipelines(notes, &programs, &plan, &left); plan_write(plan); // (for predict_background: after the precompile the programs' file cannot be gone through whole) std::unordered_map> by_program; for (const auto& n : predicted) by_program[n->prog].push_back(n.get()); @@ -11519,19 +12514,23 @@ void predict_run(void (*progress)(size_t done, size_t total)) { const size_t i = next++; if (i >= keys.size()) return; const auto& list = by_program[keys[i]]; + std::unordered_set seen; // ((Triforce, 11-22, PIPE-44) each pipeline once: predict_first) + bool lent = false; program_cache_borrow(keys[i], [&](Program& p) { + lent = true; ensure_layout(p); for (const PipeNote* n : list) - if (n->variant == p.variant) + if (n->variant == p.variant && predict_first(seen, p, *n)) if (const VkPipeline pl = build_graphics(p, n->s, note_targets(*n), (VkPrimitiveTopology)n->topo)) vkDestroyPipeline(dev.device, pl, nullptr), made++; }); + if (!lent) st_predict_unlent.fetch_add(1, std::memory_order_relaxed); predicted_done(pstamp, keys[i]); done += list.size(); } }; std::vector threads; - const unsigned cpus = std::max(1u, std::thread::hardware_concurrency()); + const unsigned cpus = std::max(1u, cpu_count()); for (unsigned i = 0; i < std::clamp(cpus / 2, 1u, 32u); i++) threads.emplace_back(worker); while (next.load() < keys.size() + threads.size() && !prepare_stopped() && !out_of_time.load()) { if (progress) progress(done.load(), predicted.size()); @@ -11541,6 +12540,10 @@ void predict_run(void (*progress)(size_t done, size_t total)) { logf("vk: predicted %zu pipelines from %zu notes for %zu program variants no note names: %zu made in %.1f s%s", predicted.size(), notes.size(), programs, made.load(), std::chrono::duration(std::chrono::steady_clock::now() - t0).count(), out_of_time.load() ? " (the time budget, RX_VK_PREDICT_S, ran out: the rest are built behind the game)" : ""); + logf("vk: predicted pipelines: %lld left out as built already for their program, %lld programs the programs' file did not " + "lend, %zu notes of variants it has no shaders for, %zu vertex layouts that do not fit their program's inputs " + "(RX_VK_PREDICT_TYPES: unset, built once; 1, and the type check; 0, as before)", (long long)st_predict_twice.load(), + (long long)st_predict_unlent.load(), left.no_face, left.misfits); if (!out_of_time.load() && !prepare_stopped()) predicted_done(pstamp, kPredictedAll); } @@ -11590,6 +12593,105 @@ void frame_shown_note(int draws) { frame_shown_ms.store(bg_now_ms(), std::memory_order_relaxed); } +/// ((Triforce, 11-22, the performance audit's PIPE-68) The plan's programs for the counter of predicted_play_build: set once +void predict_plan_keys_publish(const std::vector& plan) { + if (!predict_check_mode() || predict_plan_keys.load(std::memory_order_acquire)) return; + const std::vector* keys = new std::vector(predict_check::programs(plan)); + const std::vector* none = nullptr; + if (!predict_plan_keys.compare_exchange_strong(none, keys)) delete keys; +} +/// ((Triforce, 11-22, the performance audit's PIPE-68, after Source B's and Source A's; predict_check.h has why and the rules) +/// At a start whose record says every predicted program is built for this driver: whether the driver still has them. Up to +/// predict_check::kSample of the plan's pipelines, from another place of the plan each start, each built for the driver's +/// cache alone, timed, and built once more as the warm reference; as the builder, only while the frame shown is a light one. +/// True: cold, and the record is removed: the caller builds them all again. On predict_background's idle thread +bool predict_check_cold(uint64_t pstamp) { + const int mode = predict_check_mode(); + if (!mode) return false; + std::vector plan; + if (!plan_read(&plan) || plan.empty()) return false; + predict_plan_keys_publish(plan); // (the counter of play builds, whether or not this start checks) + const std::string state_path = predicted_path() + ".check"; + const predict_check::State had = predict_check::state_read(state_path); + if (!predict_check::runs(mode, had, pstamp)) return false; + { // (a start whose predict_run built them has them in the driver's memory, which says nothing of what its disk kept) + std::lock_guard lk(predicted_mu); + if (predicted_file) return false; + } + const std::vector at = predict_check::spots(plan.size(), (size_t)predict_check::kLook, (uint64_t)bg_now_ms()); + std::vector want; + for (const size_t i : at) want.push_back(plan[i].vertex_note), want.push_back(plan[i].output_note); + FILE* f = fopen(pipe_notes_path().c_str(), "rb"); + if (!f) return false; + struct Close { + FILE* f; + ~Close() { fclose(f); } + } close{f}; + const std::unordered_map rec = predict_check::find(f, program_cache_stamp(), sizeof(PipeNote), want); + const auto v = std::make_unique(), o = std::make_unique(), n = std::make_unique(); + std::vector probes; + const auto ms = [](std::chrono::steady_clock::duration d) { return std::chrono::duration(d).count(); }; + for (const size_t i : at) { + if (predict_check::decided(predict_check::tally(probes))) break; + // (the builder's rule: not while the game is in play. A build is tens of ms inside the driver when it compiles) + while (!prepare_stopped() && bg_game_in_play()) Sleep(50); + if (prepare_stopped()) return false; + const PredictPlan& e = plan[i]; + const auto rv = rec.find(e.vertex_note), ro = rec.find(e.output_note); + if (rv == rec.end() || ro == rec.end() || !pipe_note_at(f, rv->second, *v) || !pipe_note_at(f, ro->second, *o)) continue; + predicted_into(*n, *v, *o, e.prog, e.variant); + predict_check::Probe probe; + bool tried = false; + program_cache_borrow(e.prog, [&](Program& p) { + if (p.variant != e.variant || tried) return; + tried = true; + ensure_layout(p); + driver_cache_only = true; // (as the builder's workers: for the driver's own cache, not into ours) + const auto b0 = std::chrono::steady_clock::now(); + const VkPipeline first = build_graphics(p, n->s, note_targets(*n), (VkPrimitiveTopology)n->topo); + const auto b1 = std::chrono::steady_clock::now(); + const VkPipeline again = first ? build_graphics(p, n->s, note_targets(*n), (VkPrimitiveTopology)n->topo) : VK_NULL_HANDLE; + const auto b2 = std::chrono::steady_clock::now(); + driver_cache_only = false; + if (first) vkDestroyPipeline(dev.device, first, nullptr); + if (again) vkDestroyPipeline(dev.device, again, nullptr); + probe.made = first && again, probe.first_ms = ms(b1 - b0), probe.again_ms = ms(b2 - b1); + }); + if (tried) probes.push_back(probe); + } + const predict_check::Tally t = predict_check::tally(probes); + const predict_check::Verdict verdict = predict_check::verdict(t); + int next = 0; + const predict_check::Do what = predict_check::decide(verdict, had, pstamp, &next); + if (verdict == predict_check::Verdict::Unknown) { + logf("vk: predicted pipelines: the check of the driver's cache says nothing: %d of %zu sampled could be built " + "(RX_VK_PREDICT_CHECK)", t.got, at.size()); + return false; + } + if (verdict == predict_check::Verdict::Warm) { + if (had.found) remove(state_path.c_str()); // (kept: a strike of an earlier loss is forgotten) + logf("vk: predicted pipelines: the driver still has them: %d of %d sampled were compiled again (a first build %.1f ms, " + "a second %.1f ms at the median) (RX_VK_PREDICT_CHECK=0: not checked)", t.cold, t.got, t.first_median, t.again_median); + return false; + } + predict_check::state_write(state_path, pstamp, next); + if (what == predict_check::Do::Stop) { + logf("vk: predicted pipelines: the driver has lost them again right after they were built again (%d of %d sampled " + "compiled, a first build %.1f ms against %.1f ms): this driver does not keep them, and they are left to their first " + "draw (not checked again for these shaders and this driver; RX_VK_PREDICT_CHECK=2: at every start)", t.cold, t.got, + t.first_median, t.again_median); + return false; + } + { // (the record of what is built: gone, so the builder below and the next starts' build every predicted program again) + std::lock_guard lk(predicted_mu); + if (predicted_file) fclose(predicted_file), predicted_file = nullptr; + remove(predicted_path().c_str()); + } + logf("vk: predicted pipelines: the driver no longer has them (%d of %d sampled were compiled again: a first build %.1f ms, a " + "second %.1f ms at the median), though the record said all were built: the record is removed and they are built again " + "behind the game (RX_VK_PREDICT_CHECK=0: not checked)", t.cold, t.got, t.first_median, t.again_median); + return true; +} /// (Triforce) The predicted pipelines predict_run did not get to, built behind the game: see above void predict_background() { static std::atomic started{false}; @@ -11602,26 +12704,56 @@ void predict_background() { if (prepare_stopped()) return; const auto t0 = std::chrono::steady_clock::now(); const uint64_t pstamp = warmed_stamp(gpl_wanted()); - const std::unordered_set had = predicted_read(pstamp); - if (had.count(kPredictedAll)) return; // (all built for this driver: the programs' file is not read for it again) - const std::vector> notes = pipe_notes_read(); - if (notes.empty()) return; - std::vector plan; - if (!plan_read(&plan)) { - logf("vk: predicted pipelines: no plan for these shaders yet (predict_plan_ensure makes it before the precompile)"); - return; + std::unordered_set had = predicted_read(pstamp); + // (all built for this driver: the programs' file is not read for it again. (Triforce, 11-22, the performance audit's + // PIPE-68) Unless the driver has lost them since, which a sample says: predict_check_cold. Cold, the record is gone + // and every predicted program is built again below) + if (had.count(kPredictedAll)) { + if (!predict_check_cold(pstamp)) return; + had.clear(); } - const std::vector> predicted = predict_rebuild(notes, plan); + // ((Triforce, 11-21, the performance audit's PIPE-83, after Source B's) The builder held every note (~27 KB each) and + // a full note of every predicted pipeline until it ended, and it rests through play: about 1 GB for much of a session + // on the starts after a translator or driver update. Now the notes file is gone through once for each key's record, + // the plan kept as record numbers (predict_stream.h, 16 bytes a pipeline), and a worker reads a pipeline's two notes + // through its own handle when it builds it, as the draws, the read-ahead and the warm do (pipe_note_at). + // RX_VK_PREDICT_BG_STREAM=0: all of them held, as before) + static const bool stream = !(getenv("RX_VK_PREDICT_BG_STREAM") && !atoi(getenv("RX_VK_PREDICT_BG_STREAM"))); + const auto no_plan = [] { + logf("vk: predicted pipelines: no plan for these shaders yet (predict_plan_ensure makes it before the precompile)"); + }; + std::vector groups; // (the programs to build, their notes' records) + std::vector> notes, predicted; // (RX_VK_PREDICT_BG_STREAM=0: as before) std::unordered_map> by_program; - for (const auto& n : predicted) by_program[n->prog].push_back(n.get()); std::vector keys; - for (const auto& kv : by_program) - if (!had.count(kv.first)) keys.push_back(kv.first); + if (stream) { + std::unordered_map at; + if (FILE* f = fopen(pipe_notes_path().c_str(), "rb")) { + const auto rec = std::make_unique(); + at = predict_stream::index(f, program_cache_stamp(), sizeof(PipeNote), rec.get(), [&] { return note_sane(*rec); }); + fclose(f); + } + if (at.empty()) return; + std::vector plan; + if (!plan_read(&plan)) return no_plan(); + predict_plan_keys_publish(plan); // ((Triforce, 11-22, PIPE-68) for the counter of play builds: predicted_play_build) + groups = predict_stream::group(plan, at, had); // (the index and the plan end here: freed before the workers start) + for (const predict_stream::Program& g : groups) keys.push_back(g.prog); + } else { + notes = pipe_notes_read(); + if (notes.empty()) return; + std::vector plan; + if (!plan_read(&plan)) return no_plan(); + predicted = predict_rebuild(notes, plan); + for (const auto& n : predicted) by_program[n->prog].push_back(n.get()); + for (const auto& kv : by_program) + if (!had.count(kv.first)) keys.push_back(kv.first); + } if (keys.empty()) { predicted_done(pstamp, kPredictedAll); return; } - const unsigned cpus = std::max(1u, std::thread::hardware_concurrency()); + const unsigned cpus = std::max(1u, cpu_count()); static const unsigned nthreads = getenv("RX_VK_PREDICT_BG_THREADS") ? (unsigned)std::clamp(atoi(getenv("RX_VK_PREDICT_BG_THREADS")), 1, 32) : std::clamp(cpus / 6, 1u, 3u); static const int in_play = getenv("RX_VK_PREDICT_BG_PLAY") ? std::clamp(atoi(getenv("RX_VK_PREDICT_BG_PLAY")), 0, 2) : 0; @@ -11636,6 +12768,17 @@ void predict_background() { SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_IDLE); driver_cache_only = true; int64_t rest_until = 0; // (in play, RX_VK_PREDICT_BG_PLAY=1: this thread's rest after a program) + // ((Triforce, 11-21, PIPE-83) this thread's own handle on the notes file, and its three notes: the two read for a + // pipeline and the predicted one made of them) + struct Notes { + FILE* f = nullptr; + std::unique_ptr v, o, n; + ~Notes() { if (f) fclose(f); } + } in; + if (stream) { + in.f = fopen(pipe_notes_path().c_str(), "rb"); + in.v = std::make_unique(), in.o = std::make_unique(), in.n = std::make_unique(); + } for (;;) { if (prepare_stopped()) return; // (the game in play: see above. Looked at before each program; one already inside the driver finishes) @@ -11658,14 +12801,31 @@ void predict_background() { const int64_t b0 = bg_now_ms(); const size_t i = next++; if (i >= keys.size()) return; - const auto& list = by_program[keys[i]]; + const std::vector* list = stream ? nullptr : &by_program.find(keys[i])->second; + std::unordered_set seen; // ((Triforce, 11-22, PIPE-44) each pipeline once: predict_first) + bool lent = false; program_cache_borrow(keys[i], [&](Program& p) { + lent = true; ensure_layout(p); - for (const PipeNote* n : list) - if (n->variant == p.variant) - if (const VkPipeline pl = build_graphics(p, n->s, note_targets(*n), (VkPrimitiveTopology)n->topo)) - vkDestroyPipeline(dev.device, pl, nullptr), made++; + const auto build = [&](const PipeNote* n) { + if (!predict_first(seen, p, *n)) return; + if (const VkPipeline pl = build_graphics(p, n->s, note_targets(*n), (VkPrimitiveTopology)n->topo)) + vkDestroyPipeline(dev.device, pl, nullptr), made++; + }; + if (stream) { + for (const predict_stream::Entry& e : groups[i].entries) { + if (e.variant != p.variant || !in.f) continue; + // (a note gone or no longer sane: that pipeline is not built, as one whose note the plan lost) + if (!pipe_note_at(in.f, e.vertex_rec, *in.v) || !pipe_note_at(in.f, e.output_rec, *in.o)) continue; + predicted_into(*in.n, *in.v, *in.o, keys[i], e.variant); + build(in.n.get()); + } + return; + } + for (const PipeNote* n : *list) + if (n->variant == p.variant) build(n); }); + if (!lent) st_predict_unlent.fetch_add(1, std::memory_order_relaxed); predicted_done(pstamp, keys[i]); if (in_play == 1 && bg_game_in_play()) rest_until = bg_now_ms() + 3 * (bg_now_ms() - b0); if (++programs_done % 2000 == 0) @@ -11682,6 +12842,8 @@ void predict_background() { "play%s", made.load(), programs_done.load(), std::chrono::duration(std::chrono::steady_clock::now() - t0).count(), rested_ms.load() / 1000.0, all ? ": every predicted program is built for this driver now" : " (stopped: the rest at the next start)"); + logf("vk: predicted pipelines: %lld left out as built already for their program, %lld programs the programs' file did " + "not lend (RX_VK_PREDICT_TYPES)", (long long)st_predict_twice.load(), (long long)st_predict_unlent.load()); }).detach(); } @@ -11698,7 +12860,7 @@ VkComputePipelineCreateInfo compute_create_info(const Program& q) { ci.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT; ci.stage.module = q.stages[0].module; ci.stage.pName = "main"; - pin_subgroup(ci.stage, q.workgroup[0]); + pin_compute(ci.stage, q); ci.layout = q.layout; static const bool noopt = getenv("RX_VK_NOOPT") != nullptr; if (noopt) ci.flags |= VK_PIPELINE_CREATE_DISABLE_OPTIMIZATION_BIT; @@ -11706,6 +12868,7 @@ VkComputePipelineCreateInfo compute_create_info(const Program& q) { } void dispatch(VkCommandBuffer cmd, NvnState& s, uint32_t x, uint32_t y, uint32_t z, uint64_t indirect) { + lab_ab::each(lab_ab::E_DISPATCH); // ((Triforce, 11-22, REPLAY-51) the lab's e_dispatch: a wait at each dispatch, before the clocks PROF(PROF_DISPATCH); DispatchClock dclock; s.grid[0] = indirect ? 1 : x, s.grid[1] = indirect ? 1 : y, s.grid[2] = indirect ? 1 : z; @@ -11715,9 +12878,19 @@ void dispatch(VkCommandBuffer cmd, NvnState& s, uint32_t x, uint32_t y, uint32_t } checkpoint_grid[0] = s.grid[0], checkpoint_grid[1] = s.grid[1], checkpoint_grid[2] = s.grid[2]; const uint8_t types[8]{}; - const auto t_pick = std::chrono::steady_clock::now(); + // ((Triforce, 11-22, the performance audit's DESC-30, after Source A's) The pick's time for the stall line without a clock + // of its own: its slow path (a translation, a cache read: program_pick) is timed by PROF_EXACT, which adds to the thread's + // prof_t.exact, so the change of that across the call is the time, and a pick the memo answers reads no clock and adds + // nothing. It was two clock reads and a sequentially consistent add at every dispatch. With every scope timed + // (prof_full: RX_VK_PROF_ALL=1, RX_VK_SLOW_LOG, RX_VK_BENCH) and in the lab's "diagold" phases, the clock pair as before + const bool pick_clocked = prof_full.load(std::memory_order_relaxed) != 0 || dispatch_diag_old(); + const auto t_pick = pick_clocked ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + const int64_t x_pick = prof_t.exact; const Program* cp = program_for(s, ProgramKind::Compute, types); - st_disp_pick_ns += std::chrono::duration_cast(std::chrono::steady_clock::now() - t_pick).count(); + if (pick_clocked) + st_disp_pick_ns += std::chrono::duration_cast(std::chrono::steady_clock::now() - t_pick).count(); + else if (const int64_t dx = prof_t.exact - x_pick) + st_disp_pick_ns.fetch_add(dx, std::memory_order_relaxed); dclock.t_prog = slow_dispatch_timing() ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; // (a program still being translated is waited for, as a draw's is: a dispatch left out is work never done again. // MKW updates its terrain height clipmaps a region at a time as the camera moves, and makes its sky's tables @@ -11815,15 +12988,20 @@ void dispatch(VkCommandBuffer cmd, NvnState& s, uint32_t x, uint32_t y, uint32_t { // (hookshot, RX_VK_LOG_PUSH=: once, that program's push constants and the c0 words it reads) static const char* want = getenv("RX_VK_LOG_PUSH"); static bool done; - char hex[24]; - snprintf(hex, sizeof hex, "%016llx", (unsigned long long)cp->key); - if (want && !done && !strncmp(hex, want, strlen(want))) { - done = true; - char line[1600]; - int n = snprintf(line, sizeof line, "vk: program %s: %u bytes pushed; c0:", hex, s.cs_push_n); - for (uint32_t off = 0; off < 0x100 && n < (int)sizeof line - 16; off += 4) - n += snprintf(line + n, sizeof line - n, " %08x", driver_bank_word(s, 0, 5, off)); - logf("%s", line); + // ((Triforce, 11-22, the performance audit's DESC-06 and DESC-30, after Source A's) the key's text only while the + // switch is set and its line not written: it was formatted at every dispatch for a switch nobody had set. (And in + // the lab's "diagold" phases, for what that cost) + if ((want && !done) || dispatch_diag_old()) { + char hex[24]; + snprintf(hex, sizeof hex, "%016llx", (unsigned long long)cp->key); + if (want && !done && !strncmp(hex, want, strlen(want))) { + done = true; + char line[1600]; + int n = snprintf(line, sizeof line, "vk: program %s: %u bytes pushed; c0:", hex, s.cs_push_n); + for (uint32_t off = 0; off < 0x100 && n < (int)sizeof line - 16; off += 4) + n += snprintf(line + n, sizeof line - n, " %08x", driver_bank_word(s, 0, 5, off)); + logf("%s", line); + } } } Program& p = const_cast(*cp); @@ -11836,7 +13014,12 @@ void dispatch(VkCommandBuffer cmd, NvnState& s, uint32_t x, uint32_t y, uint32_t on = false; } ~PipeClock() { stop(); } // (a dispatch left out while its pipeline compiles: until there) - } pipe_clock{std::chrono::steady_clock::now(), !p.compute_pipeline || !p.layout}; + }; + // ((Triforce, 11-22, the performance audit's DESC-30) the clock only for a dispatch that has a layout or a pipeline to + // make: it was read at every dispatch and the reading thrown away. (The lab's "diagold" phases read it as before) + const bool pipe_timed = !p.compute_pipeline || !p.layout; + PipeClock pipe_clock{pipe_timed || dispatch_diag_old() ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}, + pipe_timed}; if (!p.dispatched.load(std::memory_order_relaxed) && (p.compute_pipeline || !p.compute_state.load(std::memory_order_relaxed))) (p.compute_pipeline ? st_disp_ready : st_disp_made)++; // (once: a dispatch left out while it compiles asks again) if (!p.layout) { @@ -11865,6 +13048,9 @@ void dispatch(VkCommandBuffer cmd, NvnState& s, uint32_t x, uint32_t y, uint32_t VkPipeline cached = VK_NULL_HANDLE; // ((Triforce) these through the play cache once the preparation is done: cached_create, pcache_play.h) if (dev.vk13.pipelineCreationCacheControl && cached_compute(pcache_use().cache, &asked, &cached) != VK_SUCCESS) cached = VK_NULL_HANDLE; + // ((Triforce, 11-22, PIPE-21) one the light precompile's record lists, not found: counted, built_play_build) + if (!cached && dev.vk13.pipelineCreationCacheControl && built_play_keys.load(std::memory_order_relaxed)) + built_play_build(p.key, 0, true); auto build = [job, info] { Program& q = const_cast(*job); VkComputePipelineCreateInfo ci = info(q); @@ -11914,7 +13100,7 @@ void dispatch(VkCommandBuffer cmd, NvnState& s, uint32_t x, uint32_t y, uint32_t ci.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT; ci.stage.module = p.stages[0].module; ci.stage.pName = "main"; - pin_subgroup(ci.stage, p.workgroup[0]); + pin_compute(ci.stage, p); ci.layout = p.layout; { // (local) RX_VK_NOOPT=1, as for graphics pipelines above static const bool noopt = getenv("RX_VK_NOOPT") != nullptr; @@ -12267,6 +13453,48 @@ int64_t draw_left_out_take() { "colour target drawn for the first time (RX_VK_PIPE_SKIP_NEW=0: left out)", (long long)now[0], (long long)now[1], (long long)now[2], (long long)now[3]); } + if (pipe_ahead_window()) { // ((Triforce, 11-22, PIPE-81) where the first-drawn pipelines stood, when it grew, every 600 presents) + static uint64_t a_told = 0; + static int64_t a_was = 0; + const int64_t none = st_ahead[0].load(), within = st_ahead[1].load(), further = st_ahead[2].load(), behind = st_ahead[3].load(); + const int64_t placed = within + further + behind; + if (calls >= a_told + 600 && none + placed != a_was) { + a_told = calls, a_was = none + placed; + logf("vk: pipelines first drawn, by their place in the order: %lld within %lld places ahead of where the draws are " + "(%lld of them linked by the warm), %lld further ahead, %lld behind, %lld not in it: a builder of whole " + "pipelines just ahead %s (RX_VK_PIPE_AHEAD)", (long long)within, (long long)pipe_ahead_window(), + (long long)st_ahead_warm.load(), (long long)further, (long long)behind, (long long)none, + pipe_ahead::worth(within, placed + none) ? "would meet a third of them or more" : "would meet under a third of them"); + } + } + if (pipe_depth_budget()) { // ((Triforce, 11-22, PIPE-29) the depth budget's counts, and its left-out entries let go) + static uint64_t d_told = 0; + static int64_t d_was[2] = {0, 0}; + const int64_t d_now[2] = {st_pipe_depth_waits.load(), st_pipe_depth_left.load()}; + if (calls >= d_told + 600) { + d_told = calls; + { + std::lock_guard lk(pipe_depth_mu); + pipe_depth_out.clear(); + } + if (std::memcmp(d_now, d_was, sizeof d_now)) { + std::memcpy(d_was, d_now, sizeof d_was); + logf("vk: depth-tested draws into passes drawn once, so far: %lld waited from the frame's pool, %lld left out " + "(RX_VK_PIPE_DEPTH_BUDGET)", (long long)d_now[0], (long long)d_now[1]); + } + } + } + if (translate_wait_ms()) { // ((Triforce, 11-22, PIPE-30) the bounded wait's, when it changed, at most every 600 presents) + static uint64_t t_told = 0; + static int64_t t_was[2] = {0, 0}; + const int64_t t_now[2] = {st_translate_bounded.load(), st_translate_left.load()}; + if (calls >= t_told + 600 && std::memcmp(t_now, t_was, sizeof t_now)) { + t_told = calls; + std::memcpy(t_was, t_now, sizeof t_was); + logf("vk: programs still translating, so far: %lld draws waited from the frame's %d ms, %lld left out " + "(RX_VK_TRANSLATE_WAIT_MS)", (long long)t_now[0], translate_wait_ms(), (long long)t_now[1]); + } + } return hold_pipelines() ? n : 0; } void draw_hold_mode(int mode) { @@ -12486,6 +13714,7 @@ void pipeline_cache_save(bool force) { auto save = [force] { struct Done { ~Done() { busy = false; } } done; const bool checkpoint = force && pcache_checkpoint_now; // ((Triforce) warm_checkpoint.h: pcache_checkpoint_save's) + const bool play_only = force && pcache_play_only_now; // ((Triforce, 11-22, PIPE-23) precompile_end_save's) // ((Triforce, handoff §8) the exit waits for this save (2 s at most: a cut one leaves the old files, .tmp and .part // are put in place whole), and none begins once it has begun: file_writers.h const file_writers::Writing writing; @@ -12495,6 +13724,14 @@ void pipeline_cache_save(bool force) { // below only when the preparation built into its cache since its last save, there is no play cache, or a // preparation saves now (the warm's pcache_saved_now asks the file below for what it built) const int64_t main_built = pcache_main_built.load(); + // ((Triforce, 11-22, the performance audit's PIPE-23; precompile_save.h) the save before the title after a long + // precompile: the play file alone while there is a play cache, which took its compiles. The main cache's hits + // count as built too (cached_create), so the rule below would write the whole main file, 2 to 3 GB, before the + // title. With no play cache the main file is what there is: saved below, as any forced save + if (play_only && pcache_play_cache) { + pcache_play_save(0); + return; + } if (!pcache_play_save(main_built) && pcache_preparing.load() == 0) return; // ((Triforce) through the cache's gate: let go for the PC's RAM (pcache_let_go), it is not saved again this session, // and the file keeps its last save) diff --git a/gpu/vk/vk_draw.h b/gpu/vk/vk_draw.h index 4f7731d..126f372 100644 --- a/gpu/vk/vk_draw.h +++ b/gpu/vk/vk_draw.h @@ -38,10 +38,18 @@ struct PipeNote { struct PredictPlan { uint64_t prog, variant, vertex_note, output_note; }; +/// ((Triforce, 11-22, the performance audit's PIPE-44) What a prediction left out, for its line: noted variants the programs' +/// file has no face for, and vertex layouts that did not fit their program's inputs (RX_VK_PREDICT_TYPES=1: predict_types.h) +struct PredictCounts { + size_t no_face = 0, misfits = 0; +}; std::vector> predict_pipelines(const std::vector>& notes, size_t* programs, - std::vector* plan = nullptr); + std::vector* plan = nullptr, PredictCounts* counts = nullptr); std::vector> predict_rebuild(const std::vector>& notes, const std::vector& plan); +/// ((Triforce, 11-21, PIPE-83) One predicted pipeline's note into n, of the note V whose vertex layout and the note O whose +/// targets and state it takes: what predict_rebuild makes of each plan entry +void predicted_into(PipeNote& n, const PipeNote& V, const PipeNote& O, uint64_t prog, uint64_t variant); } #include @@ -68,6 +76,9 @@ struct DrawArgs { /// A recording starts (its slot's descriptor pool and upload stream are reset); value: the timeline value it signals void draw_begin_recording(int slot, VkCommandBuffer cmd, uint64_t value); +/// ((Triforce, 11-21, the performance audit's REPLAY-04) the slot's descriptor pools and grown streams reset now, before the +/// recording's first packet, so that draw_begin_recording finds it done (vk_draw.cpp) +void draw_slot_reset(int slot); /// Publish CPU writes from this slot's used streams before its commands reach the GPU (also for worker slots). void draw_flush_recording(int slot); /// The recording's stream still holds [at, at + n) in the mapping at base (a reset starts used over) @@ -117,7 +128,17 @@ uint64_t replay_stream_pos(); extern bool occ_live; void occ_draw_enter(); void occ_draw_begin(VkCommandBuffer cmd); +/// ((Triforce, 11-22, GPUWORK-01) a draw that writes colour, its pass open, before occ_draw_begin: counted, and with +/// RX_VK_OCC_DEPTH_ONLY the count it falls in is given up) +void occ_draw_colour(VkCommandBuffer cmd); +/// ((Triforce, 11-22, DESC-82) whether the thread's count has a query open now (after occ_draw_begin); RX_VK_OCC_LEAN=3, the +/// census of the colour-off draws such a query counts (occ_lean.h), on: vk_draw.cpp) +bool occ_counting(); +bool occ_lean_census_on(); void occ_draw_issued(); +/// ((Triforce, 11-22, PIPE-29) whether this thread's draws are inside a samples-passed count now: draw()'s depth budget waits +/// for those (a draw left out there changes what the count says) +bool occ_counting_now(); void occ_pass_end(VkCommandBuffer cmd); void draw_stats(int64_t* draws, int64_t* skipped, int64_t* dispatches, int64_t* pipelines); void draw_stats2(int64_t* passes, int64_t* mesh, double* pipeline_ms); @@ -159,7 +180,7 @@ void memory_trial_orderly_end(); /// The stream's uniform updates (UPDATE_UB, vk_replay.cpp) that overlap [addr, addr + n) and may not be in guest /// memory yet when the GPU reads it: a copy of the range with them applied, at the offset alloc returns in the /// recording's stream (mapped at base), or UINT64_MAX when none overlaps -uint64_t ub_overlay(uint64_t addr, uint64_t n, uint8_t* base, const std::function& alloc); +uint64_t ub_overlay(uint64_t addr, uint64_t n, uint8_t* base, uint64_t (*alloc)(uint64_t, uint64_t), uint64_t align); /// Guest memory as the stream's place sees it: as submitted, the stream's uniform updates before it over it bool ub_read(uint64_t addr, void* dst, uint64_t n); diff --git a/gpu/vk/vk_dump.cpp b/gpu/vk/vk_dump.cpp index 0feeb54..832c994 100644 --- a/gpu/vk/vk_dump.cpp +++ b/gpu/vk/vk_dump.cpp @@ -14,6 +14,9 @@ #include "vk_work.h" // (after the others: counts the commands that do GPU work, for full_barrier) namespace mkwvk { +// ((Triforce, 11-22, the performance audit's DESC-06, after Source A's) The frame being recorded is dumped: read inline by +// dump_on() (vk_dump.h), which every draw asks. It was this file's own `armed`, behind a call +bool dump_armed = false; namespace { enum Kind { KIND_FLOAT, KIND_UINT, KIND_SINT }; @@ -37,7 +40,6 @@ struct Item { std::vector targets; // the presents whose next frame is dumped std::vector counts; // (and how many frames from there: RX_VK_DUMP=+) -bool armed = false; int64_t armed_present = 0, armed_until = 0; int seq = 0; std::vector items; @@ -246,10 +248,6 @@ void write_item(Item& it) { } // namespace -bool dump_on() { - return armed; -} - int dump_seq() { return seq; } @@ -267,10 +265,10 @@ void dump_present(int64_t presents) { if (*q == ',') q++; } } - if (armed) logf("vk: dump of the frame after present %lld: %d steps so far", (long long)armed_present, seq); - if (armed && presents < armed_until) return; // (a run of frames: one folder, steps numbered on) - armed = std::find(targets.begin(), targets.end(), presents) != targets.end(); - if (!armed) return; + if (dump_armed) logf("vk: dump of the frame after present %lld: %d steps so far", (long long)armed_present, seq); + if (dump_armed && presents < armed_until) return; // (a run of frames: one folder, steps numbered on) + dump_armed = std::find(targets.begin(), targets.end(), presents) != targets.end(); + if (!dump_armed) return; armed_present = presents; armed_until = presents + 1; for (size_t i = 0; i < targets.size(); i++) @@ -312,7 +310,7 @@ bool step_wanted(int step, const char* what) { } void dump_image(VkCommandBuffer cmd, Texture* tex, uint32_t level, uint32_t layer, const char* what) { - if (!armed || !tex || !tex->image || tex->samples != VK_SAMPLE_COUNT_1_BIT) return; + if (!dump_armed || !tex || !tex->image || tex->samples != VK_SAMPLE_COUNT_1_BIT) return; if (!step_wanted(seq, what)) return void(seq++); // (RX_VK_DUMP_DEPTH_ONLY=1: the passes' colour targets left out, depth only (per-draw dumps of big targets fit)) static const bool depth_only = getenv("RX_VK_DUMP_DEPTH_ONLY") != nullptr; @@ -420,7 +418,7 @@ std::vector buf_items; } // namespace void dump_buffer(VkCommandBuffer cmd, VkBuffer src, VkDeviceSize off, VkDeviceSize n, const char* what) { - if (!armed || !src || !n) return; + if (!dump_armed || !src || !n) return; n = std::min(n, 65536) & ~3ull; if (!n) return; BufItem it; diff --git a/gpu/vk/vk_dump.h b/gpu/vk/vk_dump.h index dece3f3..6a156ad 100644 --- a/gpu/vk/vk_dump.h +++ b/gpu/vk/vk_dump.h @@ -8,8 +8,12 @@ namespace mkwvk { -/// The frame being recorded is dumped -bool dump_on(); +/// The frame being recorded is dumped. ((Triforce, 11-22, the performance audit's DESC-06, after Source A's) Read inline: +/// every draw asks, and it was a call into vk_dump.cpp for one bool, which is that file's to set +extern bool dump_armed; +inline bool dump_on() { + return dump_armed; +} /// The number the next step's dump gets (draws logged before it are in it) int dump_seq(); /// A present (the count so far): arms the dump of the frame after it, or ends the one before diff --git a/gpu/vk/vk_framegen.cpp b/gpu/vk/vk_framegen.cpp index aacdad3..1582783 100644 --- a/gpu/vk/vk_framegen.cpp +++ b/gpu/vk/vk_framegen.cpp @@ -23,6 +23,8 @@ #include "vk_framegen.h" #include "framegen_rows.h" // (Triforce: the Frame generation row's values, AMD FSR 4 among them) +#include "framegen_idle.h" // ((Triforce, 11-21, PRESENT-18) its images let go once it has been off a while) +#include "framegen_refresh.h" // ((Triforce, 11-21, the upscaler audit's UPS-05) a fixed multiplier and the display's refresh) #include "vk_dlssg.h" #include "vk_fsr3_inputs.h" // (AMD FSR 3.1 frame interpolation's depth and motion) #include "vk_fsr3fi.h" @@ -1243,7 +1245,24 @@ bool framegen_record(VkCommandBuffer cmd, VkImageView src, uint32_t sw, uint32_t set->busy = true; fg.next_set = (fg.next_set + 1) % 3; } - const int cur = fg.cur, prev = cur ^ 1, m = multiplier(); + // ((Triforce, 11-20) the row's multiplier as it is now, and no more than the frames the sets were made with (ensure_size, + // above): the menu runs on the display thread while this one waited for a set, and a step up of the row in that wait + // had the game's frame copied into an image never made, which crashed in the driver (framegen_rows.h set_multiplier)) + const int cur = fg.cur, prev = cur ^ 1; + int m = framegen_rows::set_multiplier(multiplier(), fg.mult); + // ((Triforce, 11-21, the upscaler audit's UPS-05) RX_VK_FRAMEGEN_REFRESH=1: a fixed multiplier (2x to 8x) makes no more + // frames than the display's refreshes that fall in one frame of the game (framegen_refresh.h): the owner's 5x on 165 Hz + // showed 331 frames a second, half of them never on the screen. Off unless set; never more than the sets' frames (m) + static const bool refresh_cap = framegen_refresh::on(getenv("RX_VK_FRAMEGEN_REFRESH")); + if (refresh_cap && opts.framegen >= 2 && opts.framegen <= kMaxMult) { + const int row = m; + m = framegen_refresh::frames(m, fg.game_ns, framegen_refresh::period_ns(display_refresh_mhz())); + static int told = 0; + if (m != told) + told = m, logf("vk: frame interpolation: %d frame(s) for each of the game's, of the row's %d (the game's frames %.2f ms " + "apart, the display at %.1f Hz: RX_VK_FRAMEGEN_REFRESH)", m, row, fg.game_ns / 1e6, + display_refresh_mhz() / 1000.0); + } const bool blend = fg.have_prev && !gap; static uint64_t sequence = 0; sequence++; @@ -1310,6 +1329,9 @@ bool framegen_record(VkCommandBuffer cmd, VkImageView src, uint32_t sw, uint32_t for (int k = 1; k < m; k++) make_frame(cmd, prev, cur, set->frame[k - 1], (float)k / (float)m, uw, uh); set->made = m - 1; } + // ((Triforce, 11-20) the game's frame after the made ones, in a frame the set has: the row gone to AMD FSR 4 in the wait + // above, over sets made for the game's frames alone, has one made frame and no second image) + set->made = framegen_rows::game_slot(set->made, fg.mult); const VkImageCopy c{{VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}, {0, 0, 0}, {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}, {0, 0, 0}, {uw, uh, 1}}; vkCmdCopyImage(cmd, fg.frames[cur].image, VK_IMAGE_LAYOUT_GENERAL, set->frame[set->made].image, @@ -1330,6 +1352,48 @@ void framegen_queue(uint64_t value) { fg.cv.notify_all(); } +/// ((Triforce, 11-21, the performance audit's PRESENT-18, after Source A's) Each present, after its queue (vk_replay.cpp, +/// after framegen_queue): frame generation's images let go once it has been off for framegen_idle::kPresents presents, the +/// display thread done with its sets and every queue idle first, as ensure_size does before it drops them. They are made +/// again at the next frame that needs them. framegen_idle.h has why and the rule; RX_VK_FG_KEEP=1: kept +void framegen_idle(bool recorded) { + static framegen_idle::Counts counts; + static const bool keep = getenv("RX_VK_FG_KEEP") && atoi(getenv("RX_VK_FG_KEEP")) != 0; + const bool on = framegen_on(); + bool have_hudless; + { + std::lock_guard lk(hl.mu); + have_hudless = hl.w || hl.taken[0].w || hl.taken[1].w; + } + const framegen_idle::Release go = + framegen_idle::step(counts, {on, recorded, fsr4fg::frames_wanted(), fg.w != 0, have_hudless}, keep); + if (!go.shared && !go.hudless) return; + { // (as ensure_size: the display thread done with the sets, then every queue idle) + std::unique_lock lk(fg.mu); + fg.cv.wait(lk, [] { return fg.jobs.empty() && !fg.sets[0].busy && !fg.sets[1].busy && !fg.sets[2].busy; }); + } + { + std::lock_guard qk(dev.present_mu); + device_wait_idle_locked(); + } + if (go.shared) { + // (the frames and the sets' made frames; the lumas and flows are small) + const uint64_t mb = ((uint64_t)fg.w * fg.h * 4 * (2 + 3 * (uint64_t)std::max(fg.mult, 1))) >> 20; + logf("vk: frame generation off for %u presents: its images (%ux%u, %dx) let go, about %llu MB (RX_VK_FG_KEEP=1: kept)", + framegen_idle::kPresents, fg.w, fg.h, fg.mult, (unsigned long long)mb); + drop_size(); + } + if (go.hudless) { + std::lock_guard lk(hl.mu); + const uint64_t mb = ((uint64_t)hl.taken[0].w * hl.taken[0].h * 4 * 2 + (uint64_t)hl.w * hl.h * 4 * 2) >> 20; + for (Hudless::Taken& t : hl.taken) drop_img(t.img), t = {}; + drop_img(hl.srgb), drop_img(hl.out), hl.w = hl.h = 0; + hl.want_w = hl.want_h = 0; + logf("vk: AMD FSR frame generation off for %u presents: the images of the picture without the HUD let go, about %llu MB", + framegen_idle::kPresents, (unsigned long long)mb); + } +} + void framegen_queue_held(VkImage image, VkImageView view, uint32_t w, uint32_t h, const int crop[4]) { if (!image || !fg.thread_started) return; // (no display thread yet: nothing of the game's was shown through it) std::lock_guard lk(fg.mu); diff --git a/gpu/vk/vk_framegen.h b/gpu/vk/vk_framegen.h index 7199fa3..2b254a2 100644 --- a/gpu/vk/vk_framegen.h +++ b/gpu/vk/vk_framegen.h @@ -17,6 +17,9 @@ bool framegen_record(VkCommandBuffer cmd, VkImageView src, uint32_t sw, uint32_t VkImage framegen_made_image(uint32_t* w, uint32_t* h); /// After that recording's submit (value): the display thread shows its frames void framegen_queue(uint64_t value); +/// ((Triforce, 11-21, the performance audit's PRESENT-18) Each present, after framegen_queue: its images let go once it has +/// been off for 120 presents. recorded: this present recorded made frames (framegen_idle.h) +void framegen_idle(bool recorded); /// (Triforce) The hold of the last picture (present_hold.h, vk_replay.cpp do_present). A frame held is not given to /// framegen_record: the display thread shows the held picture (image GENERAL, w x h, crop as above or null; view: for the /// window's passes, or null) once, after the frames it has still to show, and nothing is made from it diff --git a/gpu/vk/vk_fsr3.cpp b/gpu/vk/vk_fsr3.cpp index 379ce66..cc24986 100644 --- a/gpu/vk/vk_fsr3.cpp +++ b/gpu/vk/vk_fsr3.cpp @@ -321,6 +321,8 @@ bool make_pipeline(Pipeline& p, const PassInfo& info) { ci.layout = p.layout; // (hookshot, a stutter: through the pipeline cache kept across runs. Made with none, FSR 3.1's dozen passes were compiled // again at every start, inside the first upscaled frame: a 2.7 s first frame of play against 1.0 s without FSR, 2026-11-10) + // ((Triforce, 11-22, the performance audit's PIPE-95) and through the play cache, so that a pass the main cache lacks is + // in the small file a save in play writes, not left to the exit's save of the whole main file: shared_cache_compute) const VkResult made = shared_cache_compute(&ci, &p.pipeline); vkDestroyShaderModule(dev.device, m, nullptr); if (made != VK_SUCCESS) { diff --git a/gpu/vk/vk_fsr3_inputs.cpp b/gpu/vk/vk_fsr3_inputs.cpp index 24e6bc4..127342c 100644 --- a/gpu/vk/vk_fsr3_inputs.cpp +++ b/gpu/vk/vk_fsr3_inputs.cpp @@ -22,6 +22,7 @@ #include "dlss_checks.h" // ((Triforce) scene_size_fits: the scenes the gameplay upscale's output can come from) #include "dlss_math.h" +#include "fg_motion.h" // ((Triforce, 11-22) DLSS frame generation given the camera's motion alone: the owner's flicker) #include "framegen_rows.h" // ((Triforce) pose_of, camera_cut: a camera cut starts a new history) #include "vk_draw.h" // (descriptors_disturbed) #include "vk_fsr3.h" @@ -454,6 +455,10 @@ bool made_this_frame() { bool chain_motion(VkCommandBuffer cmd, VkImage motion, uint32_t width, uint32_t height) { // (RX_VK_FG_OWN_MOTION=0: the camera's motion alone, as before, for comparing) static const bool off = env_switch("RX_VK_FG_OWN_MOTION") == 0; + // ((Triforce, 11-22) and NVIDIA DLSS frame generation the camera's alone too, by default: with the chain's motion the + // owner's flicker of 11-20 came back, with the camera's it did not (fg_motion.h says what was run). + // RX_VK_DLSSG_OWN_MOTION=1: the chain's for it as well, as before) + if (!fg_motion::chain(dlssg::active(), getenv("RX_VK_FG_OWN_MOTION"), getenv("RX_VK_DLSSG_OWN_MOTION"))) return false; State& s = st(); std::lock_guard lk(s.mu); if (off || !cmd || !motion || !s.have_frame || s.frame.frame != texture_frame_no() || !s.motion.image || diff --git a/gpu/vk/vk_game_tex.h b/gpu/vk/vk_game_tex.h index d6e8b16..fefc482 100644 --- a/gpu/vk/vk_game_tex.h +++ b/gpu/vk/vk_game_tex.h @@ -277,6 +277,20 @@ inline bool make_skin(std::vector&& bntx_bytes, Skin& skin) { if (out.name == "SystemBgSh_00") out.img = shrink_mask(out.img, 4); skin.images.push_back(std::move(out)); } + // ((Triforce) 11-18) The game's own pictures for the achievements' icons (vk_menu.cpp ach_icon), by the names his own game's + // archive lists them under: the Spiritual Stones and the medallions as the load list shows them, a song's icon, a Gold + // Skulltula's mark, a rupee. Taken when the archive has them and they are of a form this reads; the skin is whole + // without them, and the icons are then drawn + static const char* const more[] = { + "DataFileImportantIcon_00", "DataFileImportantIcon_01", "DataFileImportantIcon_02", "DataFileImportantIcon_03", "DataFileImportantIcon_04", + "DataFileImportantIcon_05", "DataFileImportantIcon_06", "DataFileImportantIcon_07", "DataFileImportantIcon_08", + "OcarinaMelodyIcon_00", "MapIconCollectSkullSign_00", "RupeeIconNew_00"}; + for (const char* n : more) { + const Bntx::Entry* e = bntx.find(n); + Skin::Entry out; + out.name = n; + if (e && bntx.decode(*e, out.img)) skin.images.push_back(std::move(out)); + } const Image *shape = skin.get("BtnThinBase_00"), *grain = skin.get("BtnThinBase_05"); Skin::Entry row{"plate_row", make_plate(*shape, *grain, skin.get("BtnThinBase_04"), kPlateBlack, kPlateWhite, kPlateLine)}; Skin::Entry row_sel{"plate_row_sel", row.img}; // (the selected row's plate is the same: the game marks it with the cursor alone) diff --git a/gpu/vk/vk_gpu_clock.cpp b/gpu/vk/vk_gpu_clock.cpp new file mode 100644 index 0000000..aebca33 --- /dev/null +++ b/gpu/vk/vk_gpu_clock.cpp @@ -0,0 +1,124 @@ +// (Triforce) The GPU held at amdgpu's stable power state while the game runs: Linux only, the rule in gpu_clock.h +// (RX_VK_GPU_CLOCK). The render node Vulkan names (VK_EXT_physical_device_drm) is opened, checked to be that device, and given an +// amdgpu context of the renderer's own whose stable power state is set; the file stays open to the process's end, when the kernel +// frees the context and gives the level back (a crash or kill -9 too). Elsewhere this file is empty. +#include "vk_gpu_clock.h" + +#ifdef __linux__ +#include "gpu_clock.h" +#include "vk_internal.h" // (before sys/sysmacros.h: its major and minor are macros) + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace mkwvk { + +namespace { + +// (union drm_amdgpu_ctx: what goes in, and what an allocation gives back) +union AmdgpuCtx { + struct { + uint32_t op, flags, ctx_id; + int32_t priority; + } in; + struct { + uint32_t ctx_id, pad; + } alloc; + uint64_t words[2]; +}; +static_assert(sizeof(AmdgpuCtx) == gpu_clock::kCtxArgBytes, "union drm_amdgpu_ctx is 16 bytes"); + +int held_fd = -1; // (kept open to the process's end: closing it gives the level back) + +/// power_dpm_force_performance_level's first word, into out; false when it cannot be read +bool read_level(long long minor, char* out, size_t cap) { + char path[96]; + snprintf(path, sizeof path, "/sys/class/drm/renderD%lld/device/power_dpm_force_performance_level", minor); + FILE* f = fopen(path, "r"); + if (!f) return false; + const bool ok = fgets(out, (int)cap, f) != nullptr; + fclose(f); + if (!ok) return false; + out[strcspn(out, " \t\r\n")] = 0; + return *out != 0; +} + +} // namespace + +void gpu_clock_device_created(bool drm_ext) { + if (dev.props.vendorID != 0x1002) return; + const bool integrated = dev.props.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU; + const char* sw = getenv("RX_VK_GPU_CLOCK"); + if (sw && !strcmp(sw, "0")) { + logf("vk: GPU clock: left to the driver (RX_VK_GPU_CLOCK=0)"); + return; + } + if (!drm_ext) { + logf("vk: GPU clock: left to the driver (no VK_EXT_physical_device_drm to name the GPU's render node)"); + return; + } + VkPhysicalDeviceDrmPropertiesEXT drm{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRM_PROPERTIES_EXT}; + VkPhysicalDeviceProperties2 p2{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2}; + p2.pNext = &drm; + vkGetPhysicalDeviceProperties2(dev.physical, &p2); + if (!drm.hasRender) { + logf("vk: GPU clock: left to the driver (the GPU has no render node)"); + return; + } + char level[32] = {}; + const bool have_level = read_level((long long)drm.renderMinor, level, sizeof level); + const gpu_clock::Choice c = gpu_clock::choose(sw, dev.props.vendorID, integrated, dev.props.deviceName, getenv("SteamDeck"), + have_level ? level : nullptr); + if (c.want == gpu_clock::kNone) { + logf("vk: GPU clock: left to the driver (%s; level %s)", c.why, have_level ? level : "unread"); + return; + } + char node[64]; + snprintf(node, sizeof node, "/dev/dri/renderD%lld", (long long)drm.renderMinor); + const int fd = open(node, O_RDWR | O_CLOEXEC); + if (fd < 0) { + logf("vk: GPU clock: left to the driver (%s cannot be opened, errno %d)", node, errno); + return; + } + struct stat st{}; + if (fstat(fd, &st) || !S_ISCHR(st.st_mode) || major(st.st_rdev) != (unsigned)drm.renderMajor || + minor(st.st_rdev) != (unsigned)drm.renderMinor) { + logf("vk: GPU clock: left to the driver (%s is not the device Vulkan names, %lld:%lld)", node, (long long)drm.renderMajor, + (long long)drm.renderMinor); + close(fd); + return; + } + AmdgpuCtx a{}; + a.in.op = gpu_clock::kCtxOpAlloc; + if (ioctl(fd, gpu_clock::kCtxIoctl, &a)) { + logf("vk: GPU clock: left to the driver (no amdgpu context, errno %d)", errno); + close(fd); + return; + } + AmdgpuCtx s{}; + s.in.op = gpu_clock::kCtxOpSetStablePstate; + s.in.flags = c.want == gpu_clock::kPeak ? gpu_clock::kPstatePeak : gpu_clock::kPstateStandard; + s.in.ctx_id = a.alloc.ctx_id; + if (ioctl(fd, gpu_clock::kCtxIoctl, &s)) { + // (EBUSY: another holds a stable power state, a profiler or RADV_PROFILE_PSTATE; EINVAL: a kernel before 5.18) + logf("vk: GPU clock: left to the driver (the stable power state was refused, errno %d)", errno); + close(fd); + return; + } + held_fd = fd; + logf("vk: GPU clock: held at amdgpu's %s stable power state while the game runs (%s; level was %s; the kernel gives it back at " + "the end; RX_VK_GPU_CLOCK=0: left to the driver)", c.want == gpu_clock::kPeak ? "peak" : "standard", c.why, + have_level ? level : "unread"); +} + +} // namespace mkwvk +#endif diff --git a/gpu/vk/vk_gpu_clock.h b/gpu/vk/vk_gpu_clock.h new file mode 100644 index 0000000..3914783 --- /dev/null +++ b/gpu/vk/vk_gpu_clock.h @@ -0,0 +1,10 @@ +// (Triforce) The GPU held at amdgpu's stable power state while the game runs (Linux; gpu_clock.h has the rule, RX_VK_GPU_CLOCK) +#pragma once + +#ifdef __linux__ +namespace mkwvk { +/// Once, after vkCreateDevice (vk_main.cpp create_device). drm_ext: the device has VK_EXT_physical_device_drm, which names its +/// render node. A log line on AMD GPUs; nothing at all on others +void gpu_clock_device_created(bool drm_ext); +} // namespace mkwvk +#endif diff --git a/gpu/vk/vk_imgui.cpp b/gpu/vk/vk_imgui.cpp index 178b411..ef9e855 100644 --- a/gpu/vk/vk_imgui.cpp +++ b/gpu/vk/vk_imgui.cpp @@ -233,7 +233,10 @@ void art_make() { // (the presenting thread, before a frame: the images as ImG memcpy(t->GetPixels(), e.img.rgba.data(), e.img.rgba.size()); t->UseColors = true; ImGui::RegisterUserTexture(t); - arts_.push_back({e.name, Art{t, (float)e.img.w, (float)e.img.h}}); + arts_.push_back({e.name, Art{t, (float)e.img.w, (float)e.img.h, e.img.mask}}); + if (!e.name.compare(0, 8, "DataFile") || !e.name.compare(0, 7, "Ocarina") || !e.name.compare(0, 7, "MapIcon") || !e.name.compare(0, 5, "Rupee")) + // ((Triforce) the achievements' icons: which form each is, once) + logf("vk: ui: the game's own %s: %u x %u, %s", e.name.c_str(), e.img.w, e.img.h, e.img.mask ? "a mask" : "a picture"); } skin_.images.clear(), skin_.images.shrink_to_fit(); art_state.store(3, std::memory_order_release); diff --git a/gpu/vk/vk_imgui.h b/gpu/vk/vk_imgui.h index 823d09d..6e29cf1 100644 --- a/gpu/vk/vk_imgui.h +++ b/gpu/vk/vk_imgui.h @@ -24,6 +24,7 @@ struct Fonts { struct Art { ImTextureData* tex = nullptr; float w = 0, h = 0; + bool mask = false; // ((Triforce) one channel, white with its shape in alpha: the game's layout gives it its colour, as the drawing must) }; const Art* art(const char* name); /// (Triforce) The button icons' font asked for: read at the next begin, the one the game was served this run diff --git a/gpu/vk/vk_internal.h b/gpu/vk/vk_internal.h index 1152b92..9d8096b 100644 --- a/gpu/vk/vk_internal.h +++ b/gpu/vk/vk_internal.h @@ -12,6 +12,7 @@ #include "vk_api.h" #include "vk_vendor.h" #include "write_gen.h" // (WriteGen: a page write generation, 64 bits) +#include "memory_priority.h" // ((Triforce, 11-22, TEX-47) RX_VK_MEMORY_PRIORITY's tiers #if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) #include @@ -55,6 +56,7 @@ struct Device { graphics_pipeline_library = false, custom_border_color = false, dynamic_state3 = false, depth_clip_control = false, null_descriptor = false, index_uint8 = false, push_descriptor = false, conditional_rendering = false, provoking_vertex = false, vertex_divisor = false, stencil_export = false, memory_budget = false, + memory_priority = false, // ((Triforce, 11-22, TEX-47) VK_EXT_memory_priority and its feature, asked for with RX_VK_MEMORY_PRIORITY alone checkpoints = false, // (RX_VK_CHECKPOINTS: VK_NV_device_diagnostic_checkpoints) buffer_marker = false, // (RX_VK_CHECKPOINTS=1 without the NV extension: VK_AMD_buffer_marker, AMD and Intel) device_fault = false, // (VK_EXT_device_fault: what a lost device faulted on) @@ -79,6 +81,7 @@ struct Device { VkShaderStageFlags sg32_stages = 0; // (stages that run at 32 lanes when asked: the size the translated programs assume) uint32_t subgroup_size = 32, subgroup_max = 32; // (the lanes a stage not pinned runs at, and the most it can have) VkDriverId driver_id{}; + bool radv_gcn = false; // ((Triforce, 11-22, MEM-23) RADV on a GCN 1 to 4 GPU: vk_vendor.h radv_gcn, set in create_device) VendorFixes fixes{}; // (hookshot) the vendor and driver workarounds this device needs (vk_vendor.h; vk_main.cpp create_device) uint32_t max_push_descriptors = 0; // no window (HOS_HEADLESS): presents go to an image of their own, which RX_VK_SHOTS can write out @@ -95,6 +98,7 @@ struct Device { std::vector swap_rendered; // present waits belong to acquired images, not frame slots std::mutex present_mu; // (the swapchain: presents and resizes) std::atomic swap_resize{0}; // (a new window size, width << 32 | height: the next present makes the swapchain) + uint64_t swap_generation = 0; // ((Triforce) one more for each swapchain made: an image present_image holds belongs to one) // (hookshot) a queue of a compute family for vkQueuePresentKHR alone (vk_main.cpp create_device; RX_VK_PRESENT_QUEUE=0 / 1). // VK_NULL_HANDLE: presents go on `queue` under present_mu. Its own lock: taken after present_mu, never before it VkQueue present_queue = VK_NULL_HANDLE; @@ -129,6 +133,14 @@ bool pools_mirrored(); // ((Triforce, upstream #2464) vk_memory.cpp mirror_on: void memory_ram_watch(uint64_t presents); bool memory_ram_low(); // (under 3 GB available, or a tenth of a PC of more than 30 GB) bool memory_ram_short(); // (under 1.5 GB available at two looks in a row) +// ((Triforce, 11-22, the performance audit's MEM-141) the swap-aware pressure hold (vk_memory.cpp, linux_pressure.h; off unless +// RX_VK_SWAP_RESERVE_MB is set, never on Windows): whether it is on, the cut of an integrated GPU's texture budget, the bytes +// a valve gave (1: staging arenas, 2: capture spares; the imports count themselves), and the imports' limit for the episode +bool memory_swap_hold(); +uint64_t memory_swap_cut(); +void memory_swap_valve(int valve, uint64_t bytes); +uint64_t memory_swap_episode_want(); +void memory_swap_gave(uint64_t bytes); bool memory_ram_fits(uint64_t bytes); // (a copy of that many bytes fits in the RAM available, with 2 GB to spare) bool memory_ram_fits_now(uint64_t bytes); // ((Triforce) the same with the RAM looked at now: the preparation's checkpoints) void memory_process(uint64_t* working_set, uint64_t* private_bytes); // (the process's, asked now; 0 when not known) @@ -287,6 +299,10 @@ bool create_buffer_cached(uint64_t size, VkBufferUsageFlags usage, VkBuffer& buf void** mapped, bool dedicated = false); // (vk_vma.cpp: cached host memory, mapped) void flush_mapped(VmaAllocation allocation, uint64_t size); bool host_cached(VmaAllocation allocation); // (its memory type has VK_MEMORY_PROPERTY_HOST_CACHED_BIT) +// ((Triforce, 11-22, MEM-39) vk_vma.cpp: the lab's capture buffer in uncached host memory; an allocation's memory type) +bool create_buffer_uncached(uint64_t size, VkBufferUsageFlags usage, VkBuffer& buffer, VmaAllocation& allocation, + void** mapped, bool dedicated = false); +uint32_t alloc_memory_type(VmaAllocation allocation, VkMemoryPropertyFlags* flags); void flush_mapped_range(VmaAllocation allocation, uint64_t offset, uint64_t size); void invalidate_mapped(VmaAllocation allocation, uint64_t size); // (the CPU is to read what the GPU wrote: a no-op in coherent memory) void memory_stamp_gpu(uint64_t guest, uint64_t size); // (a GPU write: memory_stamp, and the pages noted) @@ -305,6 +321,8 @@ void memory_drop_shadow(uint64_t guest, uint64_t size); // (a write the captur /// (EvilBoyce) the newest submission value whose write-back reaches into [guest, guest + size) (0: none ever); *ever set /// when any write-back ever did uint64_t memory_writeback_value(uint64_t guest, uint64_t size, bool* ever); +/// ((Triforce, THREAD-25) page_marks.h, no lock) false: no write-back ever reached a page of the range: *ever above is false +bool memory_writeback_marked(uint64_t guest, uint64_t size); void memory_note_readback(uint64_t guest, uint64_t size); // (PR 430) a CPU read of GPU results: see memory_rb_refresh /// (Triforce) around the write of an NVN event's word by a command buffer's signal: its write-back is not one the captures of /// its page wait for (vk_memory.cpp). This thread's bindings while on @@ -317,6 +335,9 @@ void memory_note_gpu_write(uint64_t gpu, uint64_t size); /// (vk_replay.cpp) The GPU's progress on the recordings' timeline, and a wait for it to reach v uint64_t vk_timeline_completed(); void vk_timeline_wait(uint64_t v); +/// ((Triforce, 11-22, the performance audit's SYNC-122) the same wait for a caller other than a capture's take: who is +/// wb_rec_wait.h's Caller (1 a read of a page put off, 2 a binding of one); its held part is counted under it +void vk_timeline_wait_from(uint64_t v, int who); // ((Triforce) RX_VK_LATE_QUEUE: the highest timeline value on the queue; a wait for a sent one (no flush; false: not reached in 1 s); // the late pages' fixes applied (dev.present_mu held; queue_flush_locked does it first); vk_memory.cpp's side uint64_t vk_timeline_signal_sent(); @@ -328,6 +349,7 @@ void memory_late_fix_apply(uint64_t done); /// ((Triforce, wp20) of this thread's last vk_timeline_wait, the part after the writer's recording was done (ns): wb_rec_wait.h) int64_t vk_timeline_wait_held_last(); WriteGen memory_generation(); +WriteGen memory_generation_any_thread(); // ((Triforce, 11-22, MEM-85) the same, read with an atomic load: from a thread that makes none) uintptr_t memory_stamp_who(uint64_t guest, WriteGen* gen, uintptr_t* anchor); // (RX_VK_STAMP_SRC: who wrote that page last) /// 1: a page of [guest, guest + size) was written after generation gen; 0: none was bool memory_written_since(uint64_t guest, uint64_t size, WriteGen gen); @@ -353,7 +375,12 @@ bool has_stencil(VkFormat f); // device memory (vk_vma.cpp, Vulkan Memory Allocator) bool vma_init(PFN_vkGetInstanceProcAddr gipa); +/// ((Triforce, 11-22, the performance audit's TEX-47, after Source A's) RX_VK_MEMORY_PRIORITY's tiers where the device has +/// VK_EXT_memory_priority (dev.memory_priority), else none: memory_priority.h. First asked once the device is made +const memory_priority::Tiers& memory_tiers(); extern uint64_t test_card_ballast; // (vk_vma.cpp, RX_VK_TEST_CARD_MB: 0 outside that lab test) +/// (vk_memory.cpp) The card's video memory in bytes: its largest device-local heap, less RX_VK_TEST_CARD_MB's ballast +uint64_t device_vram_bytes(); void vma_blocks_ahead_check(); // (vk_vma.cpp: the blocks made ahead given back while VRAM is short; every 600 presents) /// (textures: from the textures' own pool, the one image_compact_* moves allocations in: vk_texture.cpp's compaction) bool create_image(const VkImageCreateInfo& ci, VkImage& image, VmaAllocation& allocation, bool dedicated = false, bool target = false, @@ -418,9 +445,19 @@ void vma_log_image_blocks(); // (the image pools' blocks: what they hold, and PFN_vkGetInstanceProcAddr instance_proc(); // vk_main.cpp bool create_swapchain(uint32_t width, uint32_t height); +/// (Triforce, 11-22, the performance audit's PRESENT-45, after Source A's; vk_main.cpp, surface_lost.h) A result of an +/// acquire, a present or a swapchain's creation is a lost surface that is recovered from: VK_ERROR_SURFACE_LOST_KHR, with +/// RX_VK_SURFACE_RECOVER on (unless =0) and no DLSS frame generation +bool surface_recover(VkResult result); +/// ...and what hangs on the lost surface let go, the surface last (dev.present_mu held; no present in hand on the present +/// thread): the next create_swapchain makes a surface for the window again +void invalidate_lost_surface(); bool replay_start(); // vk_replay.cpp bool precompile_wanted(); // vk_draw.cpp void precompile_run(void (*progress)(size_t done, size_t total)); +void texture_pipelines_unprepared(); // ((Triforce) vk_draw.cpp: no precompile: the cache read, the converters made) +VkResult shared_cache_compute(const VkComputePipelineCreateInfo* ci, VkPipeline* out); // (vk_draw.cpp: through that cache) +void texture_pipelines_warm(); // ((Triforce) vk_texture.cpp: the texture converters' pipelines, before the game) bool prepare_run(void (*progress)(size_t done, size_t total), bool background); bool pcache_carry_rebuild(); // ((Triforce) after prepare_run: the carried pipeline cache over its cap, a fresh one: pcache_carry.h) void pcache_carry_done(); // ((Triforce) the preparation gone through and saved: a carried history no longer open: pcache_carry.h) @@ -435,6 +472,9 @@ void predict_run(void (*progress)(size_t done, size_t total)); // (after a pre /// what is done kept (the program cache, the driver's) and not counted as a start that ended inside them void prepare_stop(); bool prepare_stopped(); +/// ((Triforce, 11-22, PIPE-23) Whether this start only prepares the shaders and plays no game (vk_main.cpp prepare_only): its +/// end saves the pipeline cache itself, so the precompile's end leaves that to it (vk_draw.cpp precompile_end_save) +bool prepare_only_start(); /// What prepare_run is doing, for the start-up screen: 0 the shaders translated, 1 the graphics driver's part of them int prepare_pass(); /// Whether prepare_run has that second pass at all here (off unless the stage libraries are switched on): the screen's steps diff --git a/gpu/vk/vk_jobs.cpp b/gpu/vk/vk_jobs.cpp index e146c0b..8aacca6 100644 --- a/gpu/vk/vk_jobs.cpp +++ b/gpu/vk/vk_jobs.cpp @@ -11,6 +11,7 @@ #include #include "vk_jobs.h" +#include "cpu_count.h" // ((Triforce, 11-22, OS-49) the threads to size by: HOS_SIM_CPU #include "vk_internal.h" #include "vk_draw.h" // (ub_thread_off) @@ -61,7 +62,7 @@ void worker() { void start() { // (hookshot: with 16 hardware threads or more, 6 workers; RX_VK_WORKERS=: n, 1 to 16) - const unsigned hw = std::thread::hardware_concurrency(); + const unsigned hw = cpu_count(); unsigned n = hw >= 16 ? 6u : std::clamp(hw / 3, 2u, 4u); if (const char* e = getenv("RX_VK_WORKERS")) n = (unsigned)std::clamp(atoi(e), 1, 16); for (unsigned i = 0; i < n; i++) std::thread(worker).detach(); diff --git a/gpu/vk/vk_main.cpp b/gpu/vk/vk_main.cpp index 1079d2a..1c61a01 100644 --- a/gpu/vk/vk_main.cpp +++ b/gpu/vk/vk_main.cpp @@ -3,6 +3,7 @@ // swapchain, and the rx_* table (vk_api.h). The NVN replay itself is vk_replay.cpp. #include "vk_dlssg.h" #include "vk_paths.h" +#include "cpu_count.h" // ((Triforce, 11-22, OS-49) the threads to size by: HOS_SIM_CPU #include "vk_loader.h" #include @@ -21,6 +22,7 @@ #include "vk_internal.h" #include "gpu_choice.h" #include "present_mode.h" // (the swapchain's present mode: the VSync row and the switches) +#include "surface_lost.h" // ((Triforce, 11-22, PRESENT-45) the window's surface lost: create_swapchain, invalidate_lost_surface) #include "vk_menu.h" #include "vk_dlss.h" #include "vk_fsr4.h" @@ -29,6 +31,11 @@ #include "vk_progress.h" #include "vk_dump.h" // (write_png: RX_VK_STARTUP_SHOTS) #include "foliage_shadow.h" // ((Triforce) RX_VK_FOLIAGE_SHADOW_VRS: whether the fragment shading rate is asked for) +#include "recorder_auto.h" // ((Triforce) the recorder thread's default by the GPU and the cores) +#include "vk_gpu_clock.h" // ((Triforce, Linux) the GPU's stable power state: gpu_clock_device_created) +#include "mesa_spilling.h" // ((Triforce, 11-22, PIPE-55) shader_spilling_rate=2 for Mesa's Intel compiler, Linux build) +#include "new_wait.h" // ((Triforce, 11-22, OS-04) a `new` refused at Windows' commit limit waits before it throws: new_refused) +#include namespace vkx { #define VKX_DEFINE(f) PFN_##f f; @@ -128,6 +135,13 @@ bool create_instance() { for (const char* want : {"VK_KHR_xlib_surface", "VK_KHR_xcb_surface", "VK_KHR_wayland_surface"}) if (has_extension(all, want)) exts[n_exts++] = want; } +#endif +#ifndef _WIN32 + // ((Triforce, 11-22, the performance audit's PIPE-55, after Source A's) Mesa's Intel compiler spills registers at rate 2 + // (mesa_spilling.h): on an Iris Xe one compute program was not built after 20 minutes and 13 GB at the driver's own rate + // and took 32 s and 1.7 GB at 2. Set before the instance is made, when the driver reads its options, and without + // overwriting, so a value the player set wins. No other driver reads it. RX_VK_NO_SPILLING_RATE=1: the driver's own rate) + if (mesa_spilling::wanted(false, getenv("RX_VK_NO_SPILLING_RATE"))) setenv("shader_spilling_rate", "2", 0); #endif VkApplicationInfo app{VK_STRUCTURE_TYPE_APPLICATION_INFO}; app.pApplicationName = "mkw-recomp"; @@ -159,6 +173,24 @@ bool create_instance() { return true; } +// (Triforce) The physical cores the process may run on (recorder_auto.h; 0: not known): the RelationProcessorCore entries with a +// logical processor in the process's affinity. The core kept for the replay thread (host->reserved_cpus) counts too: on Linux +// the process's mask is the one the loading thread had (host/posix/win32_posix.c), which may already be kept off that core +static unsigned process_physical_cores() { + DWORD len = 0; + GetLogicalProcessorInformation(nullptr, &len); + std::vector info(len / sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION) + 1); + len = (DWORD)(info.size() * sizeof info[0]); + if (!GetLogicalProcessorInformation(info.data(), &len)) return 0; + DWORD_PTR proc = 0, sys = 0; + if (!GetProcessAffinityMask(GetCurrentProcess(), &proc, &sys) || !proc) return 0; + std::vector masks; + for (DWORD i = 0; i < len / sizeof info[0]; i++) + if (info[i].Relationship == RelationProcessorCore) masks.push_back((uint64_t)info[i].ProcessorMask); + const uint64_t affinity = (uint64_t)proc | (host && host->reserved_cpus ? host->reserved_cpus() : 0); + return recorder_default::physical_cores(masks.data(), (int)masks.size(), affinity); +} + bool pick_device() { menu_init(); uint32_t n = 0; @@ -167,6 +199,7 @@ bool pick_device() { vkEnumeratePhysicalDevices(dev.instance, &n, all.data()); std::vector valid_gpus; + std::vector valid_types; // ((Triforce) 11-22: their VkPhysicalDeviceType, for gpu_choice::unnamed_integrated) gpu_device_names.clear(); for (uint32_t i = 0; i < n; i++) { VkPhysicalDeviceProperties p; @@ -176,6 +209,7 @@ bool pick_device() { if (p.apiVersion >= VK_API_VERSION_1_3) { valid_gpus.push_back(i); gpu_device_names.push_back(p.deviceName); + valid_types.push_back((int)p.deviceType); } } menu_set_gpu_count((int)valid_gpus.size()); @@ -213,8 +247,11 @@ bool pick_device() { int best = -1, best_score = -1; uint32_t best_driver = 0; int target_raw_idx = -1; + bool unnamed_integrated = false; // ((Triforce) 11-22: the row's number alone on the processor graphics: Auto, the row follows) if (const char* want = getenv("RX_VK_DEVICE")) { if (*want) target_raw_idx = atoi(want); + } else if (gpu_choice::unnamed_integrated(menu_gpu_name(), gpu_device_names, valid_types, opts.gpu_device)) { + unnamed_integrated = true; } else if (const int k = gpu_choice::pick(menu_gpu_name(), gpu_device_names, opts.gpu_device); k >= 0) { target_raw_idx = (int)valid_gpus[k]; if (opts.gpu_device != k + 1) { // ((Triforce) 11-20: the saved name found the card under another number: the row follows it) @@ -267,10 +304,36 @@ bool pick_device() { logf("vk: no Vulkan 1.3 GPU"); return false; } + if (unnamed_integrated) { // ((Triforce) 11-22) the row says the card Auto took: the menu's next save keeps it with its name + for (size_t k = 0; k < valid_gpus.size(); k++) { + if (valid_gpus[k] != (uint32_t)best) continue; + logf("vk: GPU device: the row's %d is the processor graphics (%s) and no name was saved with it (a choice counted in " + "Windows' order before 11-20): %s, as Auto takes it; the row now says %d", + opts.gpu_device, gpu_device_names[(size_t)opts.gpu_device - 1].c_str(), gpu_device_names[k].c_str(), (int)k + 1); + opts.gpu_device = (int)k + 1; + } + } dev.physical = all[best]; vkGetPhysicalDeviceProperties(dev.physical, &dev.props); - // (Triforce, 11-16) the recorder thread by default on NVIDIA with 12 logical processors or more (vk_recorder.h mode_from_env) - recorder_auto(dev.props.vendorID == 0x10DE && std::thread::hardware_concurrency() >= 12); + // (Triforce, 11-16) the recorder thread's default (vk_recorder.h mode_from_env): NVIDIA with 12 logical processors or more; + // (Triforce, 11-21) AMD with 6 physical cores or more, but for its older Windows driver (recorder_auto.h) + { + VkPhysicalDeviceDriverProperties drv{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES}; + VkPhysicalDeviceProperties2 p2{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2}; + p2.pNext = &drv; + vkGetPhysicalDeviceProperties2(dev.physical, &p2); + VendorFacts facts; + facts.vendor_id = p2.properties.vendorID; + facts.driver_id = drv.driverID; + facts.device_type = p2.properties.deviceType; + facts.driver_info = drv.driverInfo; + const unsigned logical = cpu_count(), physical = process_physical_cores(); + const recorder_default::Why why = recorder_default::why(facts.vendor_id, logical, physical, amd_older_compiler(facts)); + recorder_auto(recorder_default::on(why)); + if (!getenv("RX_VK_RECORDER")) + logf("vk: recorder thread %s by default (%s: %u logical processors, %u physical cores; RX_VK_RECORDER=thread or =off " + "decides)", recorder_default::on(why) ? "on" : "off", recorder_default::text(why), logical, physical); + } vkGetPhysicalDeviceMemoryProperties(dev.physical, &dev.memory); return true; } @@ -305,6 +368,11 @@ bool create_device() { optional(VK_EXT_VERTEX_ATTRIBUTE_DIVISOR_EXTENSION_NAME, dev.vertex_divisor); optional(VK_EXT_SHADER_STENCIL_EXPORT_EXTENSION_NAME, dev.stencil_export); optional(VK_EXT_MEMORY_BUDGET_EXTENSION_NAME, dev.memory_budget); + // ((Triforce, 11-22, the performance audit's TEX-47, after Source A's) VK_EXT_memory_priority: which device memory the + // system keeps in video memory when the card is short, the render targets before the sampled textures. Asked for only + // with RX_VK_MEMORY_PRIORITY (1, or ,: memory_priority.h); unset, nothing of it is on) + const bool prio_asked = memory_priority::parse(getenv("RX_VK_MEMORY_PRIORITY")).on; + if (prio_asked) optional(VK_EXT_MEMORY_PRIORITY_EXTENSION_NAME, dev.memory_priority); optional(VK_EXT_IMAGE_2D_VIEW_OF_3D_EXTENSION_NAME, dev.view_2d_of_3d); // ((Triforce) the far cascades' foliage at one fragment per 2 x 2 pixels, foliage_shadow.h: asked for only where mesh programs // can draw natively and RX_VK_FOLIAGE_SHADOW_VRS is not off; enabled below only with pipelineFragmentShadingRate) @@ -412,6 +480,8 @@ bool create_device() { if (dev.device_fault) *tail = &fault, tail = &fault.pNext; VkPhysicalDeviceImage2DViewOf3DFeaturesEXT v2d3d{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_2D_VIEW_OF_3D_FEATURES_EXT}; if (dev.view_2d_of_3d) *tail = &v2d3d, tail = &v2d3d.pNext; + VkPhysicalDeviceMemoryPriorityFeaturesEXT mem_prio{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PRIORITY_FEATURES_EXT}; + if (dev.memory_priority) *tail = &mem_prio, tail = &mem_prio.pNext; // ((Triforce, 11-22, TEX-47) only with the switch) VkPhysicalDeviceCooperativeMatrixFeaturesKHR coop{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_FEATURES_KHR}; if (dev.coop_matrix) *tail = &coop, tail = &coop.pNext; VkPhysicalDeviceDescriptorBufferFeaturesEXT descriptor_buffer{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_BUFFER_FEATURES_EXT}; @@ -424,6 +494,9 @@ bool create_device() { vkGetPhysicalDeviceFeatures2(dev.physical, &f); descriptor_buffer.descriptorBufferCaptureReplay = VK_FALSE; dev.view_2d_of_3d = dev.view_2d_of_3d && v2d3d.sampler2DViewOf3D; + dev.memory_priority = dev.memory_priority && mem_prio.memoryPriority; // ((Triforce, 11-22, TEX-47) the allocator's bit: vma_init) + if (prio_asked && !dev.memory_priority) + logf("vk: RX_VK_MEMORY_PRIORITY: this driver has no VK_EXT_memory_priority: nothing changes"); coop.cooperativeMatrixRobustBufferAccess = VK_FALSE; dev.coop_matrix = dev.coop_matrix && coop.cooperativeMatrix && f12.vulkanMemoryModel && f12.vulkanMemoryModelDeviceScope; @@ -483,6 +556,12 @@ bool create_device() { facts.driver_id = drv.driverID; facts.device_type = p2.properties.deviceType; facts.driver_info = drv.driverInfo; + // ((Triforce, 11-22, the performance audit's MEM-23, after Source A's) RADV on a GCN 1 to 4 GPU, by its missing 16-bit + // storage input and output: a fact for the rules that need those cards told apart (the mesh fallback's stage sizes). + // No line and nothing else on any other device + facts.storage_io16 = f11.storageInputOutput16 != VK_FALSE; + dev.radv_gcn = radv_gcn(facts); + if (dev.radv_gcn) logf("vk: RADV on a GCN 1 to 4 GPU (no 16-bit storage input and output)"); dev.fixes = vendor_fixes(facts); amd_older = amd_older_compiler(facts); vendor_fixes_env(dev.fixes, [](const char* name) -> const char* { return getenv(name); }); @@ -525,6 +604,12 @@ bool create_device() { dev.driver_id = drv.driverID; logf("vk: subgroups: default %u lanes, %u-%u, pinned to 32 on stages %#x", sg.subgroupSize, sc.minSubgroupSize, sc.maxSubgroupSize, (unsigned)dev.sg32_stages); + // ((Triforce, 11-22, PIPE-55) the rate create_instance asked Mesa's Intel compiler for, as this process sees it) + if (mesa_spilling::told(drv.driverID == VK_DRIVER_ID_INTEL_OPEN_SOURCE_MESA)) { + const char* rate = getenv(mesa_spilling::kName); + logf("vk: Mesa's Intel compiler spills at rate %s (driconf shader_spilling_rate: 2 unless set)", + rate && *rate ? rate : "of its own"); + } } if (!f13.dynamicRendering || !f13.synchronization2 || !f12.timelineSemaphore) { logf("vk: the GPU lacks dynamic rendering, synchronization2 or timeline semaphores"); @@ -630,6 +715,10 @@ bool create_device() { coopmat_init(); reflex_device_created(); antilag_device_created(); +#ifdef __linux__ + // ((Triforce) the Steam Deck's GPU held at amdgpu's peak stable power state while the game runs: vk_gpu_clock.cpp, RX_VK_GPU_CLOCK) + gpu_clock_device_created(has_extension(exts, VK_EXT_PHYSICAL_DEVICE_DRM_EXTENSION_NAME)); +#endif if (dev.push_descriptor) { VkPhysicalDevicePushDescriptorPropertiesKHR pp{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PUSH_DESCRIPTOR_PROPERTIES_KHR}; VkPhysicalDeviceProperties2 p2{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2}; @@ -692,10 +781,51 @@ bool gpu_device_luid(uint8_t out[8]) { return true; } +// ---- (Triforce, 11-22, the performance audit's PRESENT-45, after Source A's) The window's surface lost (surface_lost.h), +// RX_VK_SURFACE_RECOVER (on unless =0) +namespace { bool open_window_surface(bool log_failure); } // (below, with the window's own code: start) +bool surface_recover(VkResult result) { + static const bool on = surface_lost::on(getenv("RX_VK_SURFACE_RECOVER")); + return surface_lost::recovers(result == VK_ERROR_SURFACE_LOST_KHR, on, dlssg::active()); +} +/// What hangs on the lost surface let go, in Vulkan's order: the device idle first (frames in flight name the swapchain's +/// images), then the swapchain as create_swapchain lets an old one go (Reflex off it, the overlays' views, the images' +/// semaphores, the swapchain), then the surface. The first 8 losses are said. dev.present_mu held +void invalidate_lost_surface() { + static int said; + if (said++ < 8) + logf("vk: the window's surface is lost: its swapchain and the surface are let go, and made again at the next present " + "(RX_VK_SURFACE_RECOVER=0: not handled, as before)"); + device_wait_idle_locked(); + if (dev.swapchain) { + reflex_swapchain_destroyed(); + swap_views_drop(); + for (auto semaphore : dev.swap_rendered) vkDestroySemaphore(dev.device, semaphore, nullptr); + vkDestroySwapchainKHR(dev.device, dev.swapchain, nullptr); + dev.swapchain = VK_NULL_HANDLE; + dev.swap_rendered.clear(); + dev.swap_images.clear(); + } + if (dev.surface) vkDestroySurfaceKHR(dev.instance, dev.surface, nullptr); + dev.surface = VK_NULL_HANDLE; +} + bool create_swapchain(uint32_t width, uint32_t height) { + // ((Triforce, 11-22, PRESENT-45) no surface: it was lost and let go (invalidate_lost_surface; start makes the first one + // before it comes here). Made again with the startup's calls while the window has a size, one try each 250 ms at most) + if (!dev.surface) { + static int64_t tried_ms; + const int64_t now_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()).count(); + if (!surface_lost::retry_due(now_ms, tried_ms)) return false; + tried_ms = now_ms; + if (!open_window_surface(true)) return false; + logf("vk: the window's surface is made again"); + } VkSurfaceCapabilitiesKHR caps; VkResult result = vkGetPhysicalDeviceSurfaceCapabilitiesKHR(dev.physical, dev.surface, &caps); if (result != VK_SUCCESS) { + if (surface_recover(result)) return invalidate_lost_surface(), false; // ((Triforce, PRESENT-45) lost at the query) logf("vk: querying surface capabilities failed (VkResult %d)", (int)result); return false; } @@ -712,6 +842,7 @@ bool create_swapchain(uint32_t width, uint32_t height) { uint32_t n = 0; result = vkGetPhysicalDeviceSurfaceFormatsKHR(dev.physical, dev.surface, &n, nullptr); if (result != VK_SUCCESS || !n) { + if (surface_recover(result)) return invalidate_lost_surface(), false; // ((Triforce, PRESENT-45) lost at the query) logf("vk: querying surface formats failed (VkResult %d, count %u)", (int)result, n); return false; } @@ -831,6 +962,9 @@ bool create_swapchain(uint32_t width, uint32_t height) { dev.swap_images.clear(); } if (result != VK_SUCCESS) { + // ((Triforce, 11-22, PRESENT-45) the surface lost at the creation: it goes too (the old swapchain went just above), + // and the next present's creation makes one for the window again) + if (surface_recover(result)) return invalidate_lost_surface(), false; logf("vk: vkCreateSwapchainKHR failed (VkResult %d, extent %ux%u, format %d, usage %#x, mode %d, images %u)", (int)result, width, height, (int)fmt.format, ci.imageUsage, (int)mode, ci.minImageCount); return false; @@ -862,6 +996,7 @@ bool create_swapchain(uint32_t width, uint32_t height) { dev.swap_images = std::move(images); logf("vk: swapchain %ux%u, %u images, %s%s", width, height, n, present_mode::name(chosen), force_fifo ? " (RX_VK_FIFO)" : want_immediate ? " (RX_VK_IMMEDIATE)" : opts.vsync ? "" : " (VSync off)"); + dev.swap_generation++; // ((Triforce) an image held from the one before is let go: present_image, RX_VK_ACQUIRE_FENCE) present_mode_now.store((int)chosen, std::memory_order_relaxed); swap_views_make(ci.imageUsage, mutable_view ? view_formats[1] : fmt.format); reflex_swapchain_created(sc); // (NVIDIA Reflex: its sleep mode on the new swapchain) @@ -1317,6 +1452,7 @@ bool started_up() { if (FILE* f = fopen(skipped_path().c_str(), "rb")) { fclose(f); logf("vk: the shaders' preparation was skipped at the build: left to when each is first drawn"); + texture_pipelines_unprepared(); // ((Triforce) no precompile: the texture converters' pipelines made here all the same) return true; } if (precompile_wanted()) { // (a release build, before the game: nothing to build during play) @@ -1391,6 +1527,50 @@ static void choose_present_queue() { logf("vk: presents on a queue of their own, of compute family %u (RX_VK_PRESENT_QUEUE=0: off)", dev.present_family); } +/// (Triforce, 11-22, the performance audit's PRESENT-45, after Source A's) A surface for the window again, after the one +/// before was lost and let go (invalidate_lost_surface): the calls start() makes below, for the window it was given. +/// False, with no surface, while the window has no size (minimized: it waits), or when the surface cannot be made or the +/// replay's queue cannot present to it (said when log_failure, the first 8 times). The presents' own queue is chosen +/// again for the new surface. From create_swapchain, dev.present_mu held +bool open_window_surface(bool log_failure) { + static int said; +#ifdef _WIN32 + if (!main_window) return false; + RECT r{}; + GetClientRect(main_window, &r); + if (r.right - r.left <= 0 || r.bottom - r.top <= 0) return false; + VkWin32SurfaceCreateInfoKHR si{VK_STRUCTURE_TYPE_WIN32_SURFACE_CREATE_INFO_KHR}; + si.hinstance = GetModuleHandleA(nullptr); + si.hwnd = main_window; + if (vkCreateWin32SurfaceKHR(dev.instance, &si, nullptr, &dev.surface) != VK_SUCCESS) { + dev.surface = VK_NULL_HANDLE; + if (log_failure && said++ < 8) logf("vk: the window's surface could not be made again: tried again every 250 ms"); + return false; + } +#else + const MkwVkWindow* w = prep_window; + if (!w) return false; + int width = 0, height = 0; + w->drawable_size(w->window, &width, &height); + if (width <= 0 || height <= 0) return false; + if (!w->create_surface(w->window, (void*)dev.instance, (void*)&dev.surface)) { + dev.surface = VK_NULL_HANDLE; + if (log_failure && said++ < 8) logf("vk: the window's surface could not be made again: tried again every 250 ms"); + return false; + } +#endif + VkBool32 ok = VK_FALSE; + if (vkGetPhysicalDeviceSurfaceSupportKHR(dev.physical, dev.queue_family, dev.surface, &ok) != VK_SUCCESS || !ok) { + if (log_failure && said++ < 8) + logf("vk: the replay's queue cannot present to the window's new surface: tried again every 250 ms"); + vkDestroySurfaceKHR(dev.instance, dev.surface, nullptr); + dev.surface = VK_NULL_HANDLE; + return false; + } + choose_present_queue(); + return true; +} + int start(void* hwnd) { main_window = (HWND)hwnd; if (!hwnd) { // (HOS_HEADLESS: no window; the frames are rendered all the same, RX_VK_SHOTS writes them out) @@ -1464,6 +1644,8 @@ const char* describe() { MkwVkRenderer table; } // namespace +/// ((Triforce, 11-22, PIPE-23) prepare_only, above, for vk_draw.cpp's precompile_end_save (vk_internal.h)) +bool prepare_only_start() { return prepare_only(); } } // namespace mkwvk // (hookshot) The window is being closed (host/display.c, host/posix/display_sdl.c, before they end the process): while the @@ -1483,6 +1665,37 @@ void mkwvk_close_asked(void) { if (WaitForSingleObject(prep_ended, 10000) != WAIT_OBJECT_0) logf("vk: the preparation did not stop within 10 s: closing anyway"); } +// ---- (Triforce, 11-22, the performance audit's OS-04, after Source A's and Source B's: new_wait.h) a refused `new` ---- +namespace mkwvk { +/// The system's commit left (false: not known: elsewhere a card's memory is not charged to it) +bool new_commit_left(uint64_t* avail) { +#ifdef _WIN32 + MEMORYSTATUSEX ms{}; + ms.dwLength = sizeof ms; + if (!GlobalMemoryStatusEx(&ms)) return false; + *avail = ms.ullAvailPageFile; + return true; +#else + (void)avail; + return false; +#endif +} +/// The renderer's new handler: at Windows' commit limit a refused allocation sleeps 50 ms and is tried again, up to +/// HOS_COMMIT_WAIT_MS (5 s) a thread; then, or with 4 GiB of commit or more left, or in a NoWait scope, it throws as before +void new_refused() { + static const int max_ms = new_wait::wait_ms(getenv("HOS_COMMIT_WAIT_MS")); + uint64_t avail = 0; + const bool known = new_commit_left(&avail); + if (!new_wait::waits(known, avail, new_wait::t_waited_ms, max_ms, new_wait::t_no_wait)) throw std::bad_alloc(); + static std::atomic told{0}; + if (told.fetch_add(1) < 8) + logf("vk: an allocation refused with %llu MB of the system's commit left: tried again for up to %d ms (HOS_COMMIT_WAIT_MS)", + (unsigned long long)(avail >> 20), max_ms); + Sleep(new_wait::kStepMs); + new_wait::t_waited_ms += new_wait::kStepMs; +} +} // namespace mkwvk + extern "C" #ifdef _WIN32 __declspec(dllexport) @@ -1493,6 +1706,7 @@ const MkwVkRenderer* mkwvk_init(const MkwVkHost* h) { using namespace mkwvk; if (!h || h->version != MKWVK_API_VERSION) return nullptr; host = h; + std::set_new_handler(new_refused); // ((Triforce, OS-04) a `new` refused at the commit limit waits before it throws) if (!load_global()) { logf("vk: vulkan-1.dll not found"); return nullptr; @@ -1505,6 +1719,9 @@ const MkwVkRenderer* mkwvk_init(const MkwVkHost* h) { dev.props.deviceName, VK_VERSION_MAJOR(dev.props.apiVersion), VK_VERSION_MINOR(dev.props.apiVersion), dev.props.driverVersion, dev.mesh_shader ? "yes" : "no", dev.external_memory_host ? "yes" : "no", dev.graphics_pipeline_library ? "yes" : "no"); + // ((Triforce, 11-22, the performance audit's PRESENT-46, after Source A's) the card is known now, which menu_init (inside + // pick_device) was not given: a first start on a weak card takes its Render resolution, with RX_VK_FIRST_START_AUTO + menu_first_start(); if (!dev.mesh_shader && !dev.mesh_fallback && !getenv("RX_VK_NO_MESH_WARNING")) { // (the game draws its foliage, grass and many effects with mesh shaders: without them those draws are left out and the // picture is broken (an RX 580's "static"-like scenes, 2026-11-08). Said once, on its own thread, and the game goes on.) diff --git a/gpu/vk/vk_memory.cpp b/gpu/vk/vk_memory.cpp index dbf8a43..fce004c 100644 --- a/gpu/vk/vk_memory.cpp +++ b/gpu/vk/vk_memory.cpp @@ -31,21 +31,38 @@ #include "vk_internal.h" #include "mem_parts.h" +#include "cpu_count.h" // ((Triforce, 11-22, OS-49) the threads to size by: HOS_SIM_CPU #include "vk_texture.h" // (upload_command_buffer) #include "vk_gprof.h" // (RX_VK_PASS_TIMING) #include "import_hold.h" #include "vk_paths.h" // (game_module_path: triforce_mirror_keep.txt) +#include "import_budget.h" // ((Triforce, 11-22, MEM-22) what the idle imports' valve measures the heap against) +#include "dmabuf_gate.h" // ((Triforce, 11-22) the performance audit's OS-08: no udmabuf dma-bufs offered to NVIDIA's driver +#include "linux_pressure.h" // ((Triforce, 11-22) the performance audit's MEM-141: the swap-aware pressure hold, RX_VK_SWAP_RESERVE_MB #include "mirror_inherit.h" #include "write_gen_table.h" #include "wb_lazy_state.h" // (the lazy write-back's page states) #include "lab_ab.h" // (Triforce, lab: in-run A/B switches, RX_VK_LAB_AB) +#include "mirror_budget.h" // ((Triforce) the VRAM mirrors' budget by the card: mirror_budget) +#include "lock_census.h" // ((Triforce, 11-22, THREAD-68) who holds the lock a thread waits for, RX_VK_LOCK_CENSUS) +#include "page_marks.h" // ((Triforce, 11-22, THREAD-25) the pages a write-back ever reached, asked without a lock: wb_pages_note) +#include "new_wait.h" // ((Triforce, 11-22, OS-04) a nothrow new that must answer at once: NoWait) #include "wb_lazy_payers.h" // (Triforce: how many of its payments copy at once, RX_VK_LAZY_PAYERS) #include "demote_watch.h" // (Triforce: a pool's trial in guest memory far over its baseline ends early) #include "upload_torn.h" // ((Triforce, wp18-flash) memory_rewritten_since_capture: the pages a re-upload's rule compares) #include "import_ahead.h" // (Triforce: chunks earlier runs imported on their own threads, imported ahead) #include "import_touch.h" // (Triforce: a range's pages read before the driver locks them, RX_VK_IMPORT_TOUCH) #include "capture_behind.h" // (Triforce: a capture's copies behind the game's ordering point, RX_VK_CAPTURE_BEHIND) +#include "cap_spare.h" // ((Triforce, 11-22, MEM-33) which spare a capture takes, and what the spares may hold, RX_VK_CAP_SPARE_FIT) +#include "table_stream.h" // ((Triforce, 11-22, SYNC-04) the page table's changes streamed into 64 KiB transfers, RX_VK_TABLE_STREAM) +#include "range_walk.h" // ((Triforce, 11-22, MEM-42) a loop's pages asked of a sorted range list by a forward cursor, RX_VK_RANGE_WALK) +#include "piece_idle.h" // ((Triforce, 11-22, MEM-26's first step) what the mirrors' pieces hold that no game pool covers: counted +#include "late_inline.h" // ((Triforce, 11-22, MEM-80) the late pages' partial rule alone, RX_VK_LATE_INLINE +#include "retired_give_back.h" // ((Triforce, 11-22, MEM-21) a retired pool's import given back after a wait, RX_VK_KEEP_RETIRED=wait +#include "msync_pass.h" // ((Triforce, 11-21, MEM-48) a mirror sync's pages from the captures in one compute dispatch +#include "mirror_spare.h" // ((Triforce, 11-21, MEM-27's first step) what the mirrors' sparse buffers cost: counted #include "shadow_prev.h" // (Triforce: the texture shadow's pages taken again with bytes it holds, RX_VK_SHADOW_KEEP / _PREV) +#include "wb_excl_clip.h" // ((Triforce, 11-22) no write-back into memory the capture leaves out, RX_VK_WB_EXCL_CLIP) #ifdef __linux__ #include #include @@ -77,10 +94,16 @@ struct Pool { WriteGen synced_gen; // every page stamped up to this generation is in the mirror uint64_t synced_epoch = 0; // the submission that last brought it up to date uint64_t sync_seg = 0; // (and the parallel replay's segment whose uploads did) + // ((Triforce, 11-21, MEM-48) what wrote the mirror so far in an upload command buffer (msync_pass.h kCopies, kPass), of + // the recording up_value: UpWrites) + VkCommandBuffer up_cmd = VK_NULL_HANDLE; + uint64_t up_value = 0; + uint8_t up_kinds = 0; uint64_t last_value = 0; // (the newest submission that bound it: its mirror is in use until that one is done) uint64_t imported_value = 0; // (Triforce: the recording being made when this import went into the table: imports_give_back) uint64_t direct_write_epoch = 0; // (the part that last wrote it in guest memory, before it had a mirror) uint64_t retired_value = 0; // (retired: the submission that may still use it) + int64_t retired_ms = 0; // ((Triforce, 11-22, MEM-21) and when, steady clock ms: RX_VK_KEEP_RETIRED=wait's, retired_give_back.h) bool was_mirrored = false; // (its mirror was given back: its pages stay in the captures, for the next one) // (hookshot) the GPU wrote it past what is copied back (memory_note_gpu_write: a bare entry's guessed range, the pointer // table's, a wide binding's): that is in its mirror alone, so the mirror is never given back (mirrors_give_back) @@ -119,6 +142,9 @@ struct Pool { std::vector res_bits; std::vector res_allocs; uint64_t res_bytes = 0; + // ((Triforce, 11-22, MEM-26's first step; only with RX_VK_MIRROR_PIECE_IDLE set) the submission a binding last reached each + // piece at (mirror_epoch's low 32 bits): sized once with res_bits, written without the lock as res_bits is read + std::vector piece_last; // (the pieces left in guest memory because the CPU and the GPU both write pages of them (demote_trials): a bit a piece, read // without the lock as res_bits; conf_bits: the pieces a sync found such a page in, since the last trial; trial_bits: the // ones the trial under way took out) @@ -143,8 +169,22 @@ Pool* pool_lookup(uint64_t guest, uint64_t size); thread_local const char* tl_sync_why = nullptr; struct SrwMutex { // (a slim reader/writer lock: half std::mutex's cost, taken for every buffer a draw binds) SRWLOCK l = SRWLOCK_INIT; - void lock() { AcquireSRWLockExclusive(&l); } - void unlock() { ReleaseSRWLockExclusive(&l); } + // ((Triforce, 11-22, THREAD-68) the lock census, RX_VK_LOCK_CENSUS: off, the lock is taken as before; on, tried first, and + // a wait for it counted against the thread that holds it: lock_census.h) + std::atomic owner{nullptr}; + struct Raw { + SRWLOCK* l; + void lock() { AcquireSRWLockExclusive(l); } + bool try_lock() { return TryAcquireSRWLockExclusive(l) != 0; } + }; + void lock() { + Raw raw{&l}; + lock_census::lock(raw, owner, "pools_mu"); + } + void unlock() { + lock_census::unlocking(owner); + ReleaseSRWLockExclusive(&l); + } }; SrwMutex pools_mu; uint32_t generation = ~0u; @@ -155,6 +195,9 @@ ImportHolds import_holds; // (issue 218: a failed import is not tried a std::mutex import_holds_mu; thread_local bool import_cooling = false; thread_local VkResult import_result = VK_SUCCESS; +// ((Triforce, 11-22) when an import was last refused for want of room (out of device or host memory; import_now_ms, 0: never): +// retired_imports_give_back gives the retired imports back for a minute after) +std::atomic import_room_refused_ms{0}; thread_local bool import_on_bg_thread = false; // ((Triforce) bg_main's thread: the only imports another thread's submit waits beside) // ((Triforce) lookups waiting now for an import on the background thread (pool_lookup): while there is one, that thread's read of // a range's pages first stops, and the driver takes the rest as it did (import_touch.h). The read shortens the allocate, beside @@ -190,7 +233,9 @@ constexpr VkBufferUsageFlags kUsage = // 3-8 fps with 11 ms of GPU work. A pool larger than udmabuf's size limit (/sys/module/udmabuf/parameters/size_limit_mb, // 64 MB unless raised) is made of several dma-bufs of at most that size, bound one after another into a sparse buffer, so the // pool is still one buffer with one device address and nobody has to raise the limit. RX_VK_NO_DMABUF=1: host pointers only, -// as before; RX_VK_DMABUF_PIECE_MB=: pieces of at most n MB (to try the sparse path with the limit raised) +// as before; RX_VK_DMABUF_PIECE_MB=: pieces of at most n MB (to try the sparse path with the limit raised). +// (Triforce, 11-22, the performance audit's OS-08) NVIDIA's proprietary driver refuses every such dma-buf, so with it none is +// made and the pools are host-pointer imports at once (dmabuf_gate.h) struct UdmabufCreate { uint32_t memfd, flags; uint64_t offset, size; }; constexpr unsigned long kUdmabufCreate = (1ul << 30) | (sizeof(UdmabufCreate) << 16) | ('u' << 8) | 0x42; // (_IOW('u', 0x42, ...)) int dmabuf_memfd = -2, dmabuf_dev = -1, dmabuf_failures = 0; @@ -338,7 +383,15 @@ bool import_dma_buf(uint64_t lo, uint64_t hi, Pool& p) { typedef int (*MemfdFn)(void); const MemfdFn fn = getenv("RX_VK_NO_DMABUF") ? nullptr : (MemfdFn)dlsym(RTLD_DEFAULT, "hos_posix_guest_memfd"); const int fd = fn ? fn() : -1; - if (fd >= 0 && dev.dma_buf_import && vkGetMemoryFdPropertiesKHR) dmabuf_dev = open("/dev/udmabuf", O_RDWR | O_CLOEXEC); + // ((Triforce, 11-22, the performance audit's OS-08, after Source A's) not with NVIDIA's proprietary driver: it refuses + // every dma-buf of guest memory (VK_ERROR_OUT_OF_DEVICE_MEMORY, a kernel message each start, up to 198 ms for one + // refusal during a load) and the pool ends as a host-pointer import anyway. dmabuf_gate.h has the rule; NVK and RADV + // have other driver ids. RX_VK_DMABUF_NVIDIA=1: offered to it too, as before + dmabuf_gate::Facts gate; + gate.fn = fn != nullptr, gate.memfd = fd >= 0, gate.ext = dev.dma_buf_import && vkGetMemoryFdPropertiesKHR; + gate.nvidia = dev.driver_id == VK_DRIVER_ID_NVIDIA_PROPRIETARY && + !(getenv("RX_VK_DMABUF_NVIDIA") && atoi(getenv("RX_VK_DMABUF_NVIDIA")) != 0); + if (dmabuf_gate::try_udmabuf(gate)) dmabuf_dev = open("/dev/udmabuf", O_RDWR | O_CLOEXEC); if (fd >= 0 && dmabuf_dev >= 0) { dmabuf_memfd = fd; dmabuf_limit = 64ull << 20; @@ -351,9 +404,7 @@ bool import_dma_buf(uint64_t lo, uint64_t hi, Pool& p) { if (const char* e = getenv("RX_VK_DMABUF_PIECE_MB")) if (atoll(e) > 0) piece = std::min(piece, (uint64_t)atoll(e) << 20); } - logf("vk: guest memory as dma-bufs: %s", dmabuf_memfd >= 0 ? "yes" : !fn ? "no (no memfd window, or RX_VK_NO_DMABUF)" - : fd < 0 ? "no (the window is anonymous memory)" : !dev.dma_buf_import ? "no (VK_EXT_external_memory_dma_buf)" - : "no (/dev/udmabuf cannot be opened)"); + logf("vk: guest memory as dma-bufs: %s", dmabuf_gate::answer(gate, dmabuf_dev >= 0)); if (dmabuf_memfd >= 0) logf("vk: dma-bufs of at most %llu MB (udmabuf's limit %llu MB); larger pools %s", (unsigned long long)(piece >> 20), (unsigned long long)(dmabuf_limit >> 20), @@ -545,6 +596,8 @@ bool import_pool(uint64_t lo, uint64_t hi, Pool& p) { const VkResult allocated = vkAllocateMemory(dev.device, &ai, nullptr, &p.memory); if (allocated != VK_SUCCESS) { import_result = allocated; + if (allocated == VK_ERROR_OUT_OF_DEVICE_MEMORY || allocated == VK_ERROR_OUT_OF_HOST_MEMORY) + import_room_refused_ms.store(import_now_ms(), std::memory_order_relaxed); vkDestroyBuffer(dev.device, p.buffer, nullptr); p.buffer = VK_NULL_HANDLE; import_remember_failure(lo, hi); @@ -876,11 +929,18 @@ void memory_table_begin() { } table_defer = guest_pages != nullptr; } -// (at the end of the recording's uploads: the changes, the last one for each page, as transfers into the table; the -// upload command buffer's barriers order them after the work submitted before and before this submission's) -bool memory_table_flush() { - table_defer = false; - if (table_pending.empty()) return false; +// ((Triforce, 11-22, the performance audit's SYNC-04, after Source A's; RX_VK_TABLE_STREAM, on; =0: the sort, as before) +// The flush below made a pair for every page of every change and sorted them all: table_stream.h says what that cost and +// writes the same transfers without it +bool table_stream_on() { + static const bool on = [] { + const char* e = getenv("RX_VK_TABLE_STREAM"); + return !(e && *e && atoi(e) == 0); + }(); + return on; +} +/// The flush as it was (RX_VK_TABLE_STREAM=0): every page of table_pending as a pair, sorted, the last of each page kept +bool table_flush_sorted() { std::vector> pages; // (page, value), in the order they were set for (const TableRun& r : table_pending) for (uint64_t i = 0; i < r.count; i++) pages.push_back({r.first + i, r.address ? r.address + (i << 12) : 0}); @@ -904,6 +964,30 @@ bool memory_table_flush() { emit(); return true; } +// (at the end of the recording's uploads: the changes, the last one for each page, as transfers into the table; the +// upload command buffer's barriers order them after the work submitted before and before this submission's) +bool memory_table_flush() { + table_defer = false; + if (table_pending.empty()) return false; + if (!table_stream_on()) return table_flush_sorted(); + const auto t0 = std::chrono::steady_clock::now(); + VkCommandBuffer cmd = upload_command_buffer(); + GprofSpan gp(cmd, "ptable update (guest page table)"); + gp.items = 0; + static thread_local table_stream::Scratch scratch; // (each recording thread's: table_pending is) + const uint64_t written = table_stream::stream(table_pending.data(), table_pending.size(), scratch, [&](uint64_t first, const uint64_t* values, size_t n) { + vkCmdUpdateBuffer(cmd, guest_pages_buffer, first * 8, n * 8, values), gp.items++, gp.bytes += n * 8; + }); + const size_t runs = table_pending.size(); + table_pending.clear(); + // (a flush that still takes long: its size, the first 16 times) + const double ms = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); + static std::atomic told{0}; + if (ms > 1.0 && told.fetch_add(1, std::memory_order_relaxed) < 16) + logf("vk: the page table's flush took %.2f ms: %llu pages in %zu changes, %d transfers (RX_VK_TABLE_STREAM)", ms, + (unsigned long long)written, runs, (int)gp.items); + return true; +} uint64_t memory_guest_pages() { static std::mutex mu; @@ -960,9 +1044,9 @@ bool memory_init() { std::atomic memsync_lock_wait_ns{0}; std::atomic memsync_part_ns[3]; void memory_sync() { - static std::mutex sync_mu; // (called from every thread that touches memory) + static lock_census::CensusMutex sync_mu{"sync_mu"}; // (called from every thread that touches memory; (Triforce, THREAD-68) counted by the lock census) const auto w0 = std::chrono::steady_clock::now(); - std::lock_guard sync_lk(sync_mu); + std::lock_guard sync_lk(sync_mu); struct Waited { // (diagnostics: only the lock waits, added up; nothing else changes) std::chrono::steady_clock::time_point t; ~Waited() {} @@ -1080,12 +1164,26 @@ namespace { int cap_only_gpu(); bool cap_excl_hit(uint64_t lo, uint64_t hi); bool cap_excl_page(uint64_t g); +uint64_t cap_excl_kept(uint64_t lo, uint64_t hi, std::vector>& out); void cap_born_take(std::vector>& out); void cap_excl_note(uint64_t gpu, uint64_t n, const char* who); void cap_pool_born_now(uint64_t gpu, uint64_t size); } const std::vector>& capture_ranges_merged(); void shadow_written(const uint32_t* pages, int64_t n); // (the write-back's diff, below: pages the captures took) +// ((Triforce, 11-22, the performance audit's MEM-42, after Source A's; RX_VK_RANGE_WALK, on; =0: as before) The capture and +// collection loops asked of every page whether it lies in a sorted range list by a binary search. Each loop has a +// range_walk::RangeWalk now, a cursor that only moves forward (range_walk.h): the same answers in any order. A walk is +// made inside its loop's hold of the list's lock, never kept across an unlock. This is what a walk is made with: true +// has it search every time, as before (the switch, or the lab knob norangewalk in its phases) +using range_walk::RangeWalk; +bool range_walk_search() { + static const bool off = [] { + const char* e = getenv("RX_VK_RANGE_WALK"); + return e && *e && atoi(e) == 0; + }(); + return off || lab_ab::on("norangewalk"); +} // ---- lazy write-back (Triforce; docs/lazy-writeback.md; the page states and their rules: wb_lazy_state.h) ---- // The Debug tab's Lazy write-back row, off unless set; RX_VK_LAZY_WB=1 or 0 for one run, whatever the row says, and @@ -1372,10 +1470,13 @@ LONG CALLBACK lazy_veh(EXCEPTION_POINTERS* x) { const uintptr_t a = (uintptr_t)r->ExceptionInformation[1], base = (uintptr_t)host->guest_base(); if (a < base || a - base >= host->guest_size()) return EXCEPTION_CONTINUE_SEARCH; const uint64_t page = (uint64_t)(a - base) >> 12; - if (!lz->tracked(page) || tl_lz_paying) return EXCEPTION_CONTINUE_SEARCH; + if (tl_lz_paying) return EXCEPTION_CONTINUE_SEARCH; + // ((Triforce, 11-21) on_fault, not tracked() then fault(): a page given its access back by another thread's payment just + // after this fault is uncounted after that, and tracked() alone sent the fault on to the crash handler; on_fault runs the + // access again when the page lets it through by now. Untracked and still without access: not ours, as before) const DWORD last = GetLastError(); std::vector jobs; - const wb_lazy::Fault f = lz->fault(page, r->ExceptionInformation[0] == 1, kLzChunkPages - 1, lz_now(), jobs); + const wb_lazy::Fault f = lz->on_fault(page, r->ExceptionInformation[0] == 1, kLzChunkPages - 1, lz_now(), jobs); if (f == wb_lazy::PAY) lazy_pay(jobs, wb_lazy::BY_FAULT); SetLastError(last); return f == wb_lazy::NOT_OURS ? EXCEPTION_CONTINUE_SEARCH : EXCEPTION_CONTINUE_EXECUTION; @@ -1587,17 +1688,14 @@ void memory_collect_written() { // (the captured ranges once a call: a lock, a pool-list refresh and three searches for every dirty page cost the // replay thousands of lock cycles a submit, the cost that retired gpu_pages' lock below) const auto& owned = capture_ranges_merged(); - const auto in_capture = [&owned](uint64_t guest) { - auto it = std::upper_bound(owned.begin(), owned.end(), std::make_pair(guest, ~uint64_t(0))); - return it != owned.begin() && guest < std::prev(it)->second; - }; + RangeWalk in_capture{owned, range_walk_search()}; // ((Triforce, 11-22, MEM-42) this thread's copy of the list: one walk for the loop) const bool only_gpu = cap_only_gpu() == 1; for (int64_t i = 0; i < n; i++) { const uint64_t guest = (uint64_t)pages[i] << kPageShift; // (RX_VK_CAP_ONLY_GPU=1: heap memory between the pools of an imported chunk, which no GPU address reaches: not // stamped, so no mirror sync copies it either) if (only_gpu && cap_excl_page(guest)) continue; - if (!owned.empty() && in_capture(guest)) { // (written after its submit's capture: the next capture takes it, as it is then) + if (!owned.empty() && in_capture.in(guest)) { // (written after its submit's capture: the next capture takes it, as it is then) uint64_t avail = 0; if (const uint8_t* q = host->gpu_ptr(guest, &avail)) host->mark_written(q, 4096); continue; @@ -1616,6 +1714,11 @@ void memory_collect_written() { WriteGen memory_generation() { return gens.now; } +/// ((Triforce, 11-22, the performance audit's MEM-85) gens.now read with one atomic load (write_gen.h gen_load), for the game's +/// thread at a release it queues (RX_VK_RELEASE_QUEUE): next_gen moves it on the replaying thread meanwhile +WriteGen memory_generation_any_thread() { + return gen_load(gens.now); +} /// (RX_VK_STAMP_SRC) Who wrote the page at guest last (stamp_src above), its generation, and memory_stamp's own address to place it by uintptr_t memory_stamp_who(uint64_t guest, WriteGen* gen, uintptr_t* anchor) { const WriteGen g = page_gen(guest >> kPageShift); @@ -1730,6 +1833,7 @@ struct Capture { size_t nbytes = 0; std::vector late; // (indices into pages: copied again by the replay once late_value is done) uint64_t late_value = 0; + uint64_t late_done = 0; // ((Triforce, 11-22, MEM-80) the GPU's progress capture_make found late pages by: late_inline.h) // (hookshot) "shadow staging": the texture pages of this capture, copied from guest memory into ordinary cached RAM too, for // take_capture's texture shadow to read instead of the buffer above (often write-combined here: CPU reads ~0.3 GB/s). // sidx[i] = the staging slot of pages[i] (4 KB a slot), or -1: that page is read from the buffer. Empty sidx: no staging. @@ -1790,7 +1894,10 @@ bool stage_get(Capture& c, size_t need) { // (a block of at least need bytes i } } const size_t sz = (need + (1u << 20) - 1) & ~size_t((1u << 20) - 1); // (rounded to 1 MB: blocks of near sizes fit each other) - c.stage.reset(new (std::nothrow) uint8_t[sz]); + { + new_wait::NoWait at_once; // ((Triforce, OS-04) no block: the caller goes without, it does not wait at the commit limit) + c.stage.reset(new (std::nothrow) uint8_t[sz]); + } c.stage_bytes = c.stage ? sz : 0; return c.stage != nullptr; } @@ -1805,7 +1912,7 @@ void stage_put(Capture& c) { // (the block back to the kept ones, at most four } c.stage.reset(), c.stage_bytes = 0; } -std::mutex cap_mu; +lock_census::CensusMutex cap_mu{"cap_mu"}; // ((Triforce, THREAD-68) counted by the lock census, RX_VK_LOCK_CENSUS; a std::mutex otherwise) std::deque> captures; std::vector> spare; // (taken ones, their buffers kept) // ---- the captures' pages, for the mirror syncs ---- @@ -1855,27 +1962,157 @@ std::mutex cap_free_mu; // (the submitting threads take buffers from cap_free, uint64_t cap_free_bytes = 0; // (cap_free_mu) // (the spare buffers: 12 and, on a card of 6 GB or less, 32 MB of them; RX_VK_CAP_SPARE_MB=: that many, 0: as before. // After a load the retired ones, 40-60 MB each, stayed as spares for good) +// ((Triforce, 11-22, after Casual Gamer's fix) and 32 MB on a PC of under 20 GB of RAM (mem_parts.h spare_capped): on a 16 GB +// laptop the spares held ~0.45 GB of RAM that Windows paged out and in again; capped, 10-18% fewer page-ins) +bool ram_now(uint64_t* avail, uint64_t* total); // (below) uint64_t cap_spare_max() { static const uint64_t v = [] { if (const char* e = getenv("RX_VK_CAP_SPARE_MB")) return e[0] && atoll(e) ? (uint64_t)atoll(e) << 20 : ~0ull; - uint64_t used = 0, allowed = 0; - return vram_usage(&used, &allowed) && allowed <= (6ull << 30) + (512ull << 20) ? 32ull << 20 : ~0ull; + uint64_t used = 0, allowed = 0, avail = 0, total = 0; + const bool small_card = vram_usage(&used, &allowed) && allowed <= (6ull << 30) + (512ull << 20); + // (not on AMD: there a capture buffer made anew costs about 100 ms, and with the spares capped more captures make one: + // a hitch each, where the RAM rule was measured on an NVIDIA laptop alone. AMD keeps its spares, as before) + if (!ram_now(&avail, &total) || dev.props.vendorID == 0x1002) total = 0; + const bool capped = mem_parts::spare_capped(small_card, total); + if (capped && !small_card) + logf("vk: capture buffers: 32 MB of spares at most, a PC of %llu MB of RAM (RX_VK_CAP_SPARE_MB=0: as before)", + (unsigned long long)(total >> 20)); + return capped ? 32ull << 20 : ~0ull; }(); return v; } -void cap_buf_free(VkBuffer buffer, VmaAllocation alloc, void* map, uint64_t size, bool from_worker = false) { - std::lock_guard lk(cap_free_mu); - if (from_worker && cap_shutting_down.load()) return; - // (hookshot, RX 7800 XT: buffers up to 256 MB are kept for the next capture, not only up to 64 MB: a new one costs ~100 ms of - // vkAllocateMemory on AMD's driver. RX_VK_CAPTURE_KEEP_OLD=1: as before) - static const bool keep_old = getenv("RX_VK_CAPTURE_KEEP_OLD") != nullptr; - if (cap_free.size() < 12 && size <= (keep_old ? 64ull << 20 : 256ull << 20) && - cap_free_bytes + size <= std::min(cap_spare_max(), keep_old ? ~0ull : 768ull << 20)) { - cap_free.push_back({buffer, alloc, map, size}); - cap_free_bytes += size; - } else { - destroy_buffer(buffer, alloc); +// ((Triforce) RX_VK_CAP_FREE_ASYNC, on by default; =0: as before) A retired capture buffer the spares cannot keep is destroyed +// on a thread of its own, not where it was retired: take_capture retires on the replay's thread with pools_mu held, and +// freeing a 64-256 MB host-cached buffer makes the driver unmap and release every page, a slow take's "retires" part. Only +// buffers the GPU is done with get here (the retire waits for the work that read them). Once the process starts to exit +// nothing more is destroyed: what is queued goes with the process +bool cap_free_async_on() { + static const bool v = [] { + const char* e = getenv("RX_VK_CAP_FREE_ASYNC"); + return !e || !env_value_off(e); + }(); + return v; +} +void cap_exit_note(); +std::mutex cap_destroy_mu; // (held by the destroying thread for each destroy, and by cap_exit_note as it sets the flag) +std::atomic cap_destroyed_later{0}, cap_destroyed_later_bytes{0}; // (handed to the thread, for the 600-take line) +void destroy_buffer_later(VkBuffer buffer, VmaAllocation alloc, uint64_t size) { + struct Later { VkBuffer buffer; VmaAllocation alloc; }; + struct Queue { + std::mutex mu; // (a leaf: nothing else is taken while it is held) + std::condition_variable cv; + std::vector todo; + int state = 0; // (0 no thread yet, 1 running, 2 none could be started) + }; + static Queue* q = new Queue; // (never freed: the detached thread outlives the statics at exit) + { + std::unique_lock lk(q->mu); + if (q->state == 0) { + static const bool exit_hook = [] { return std::atexit(cap_exit_note) == 0; }(); + (void)exit_hook; + try { + std::thread([] { + std::vector batch; + std::unique_lock lk(q->mu); + for (;;) { + q->cv.wait(lk, [] { return !q->todo.empty(); }); + batch.swap(q->todo); + lk.unlock(); + for (const Later& b : batch) { + std::lock_guard dg(cap_destroy_mu); + if (cap_shutting_down.load()) break; + destroy_buffer(b.buffer, b.alloc); + } + batch.clear(); + lk.lock(); + } + }).detach(); + q->state = 1; + } catch (...) { // (no thread: destroyed where it was retired, as before) + q->state = 2; + } + } + if (q->state == 1) { + q->todo.push_back({buffer, alloc}); + cap_destroyed_later.fetch_add(1, std::memory_order_relaxed); + cap_destroyed_later_bytes.fetch_add((int64_t)size, std::memory_order_relaxed); + q->cv.notify_one(); + return; + } } + destroy_buffer(buffer, alloc); +} +// ((Triforce, 11-22, the performance audit's MEM-33, after Source A's; RX_VK_CAP_SPARE_FIT, on; =0: as before) cap_spare.h +// says what was so. With it: a capture with no spare makes its buffer with cap_free_mu let go; it leaves a big spare far +// over its class to the load that wants it; a big spare taken is made again on a worker; and what the workers are making +// is counted with the spares (cap_reserved_bytes, cap_reserved_n), so a spare made ahead is not thrown away as it lands) +bool cap_spare_fit_on() { + static const bool on = [] { + const char* e = getenv("RX_VK_CAP_SPARE_FIT"); + const bool v = !(e && *e && atoi(e) == 0); + if (!v) logf("vk: capture buffers made under the spares' lock and any fitting spare taken, as before (RX_VK_CAP_SPARE_FIT=0)"); + return v; + }(); + return on; +} +/// (RX_VK_CAP_STEP_MB=, off unless set) the steps of a 16 to 32 MB capture's buffer where spares are capped at 32 MB +uint64_t cap_step_mb() { + static const uint64_t v = [] { + const char* e = getenv("RX_VK_CAP_STEP_MB"); + const uint64_t n = e && atoi(e) > 0 ? (uint64_t)atoi(e) : 0; + if (n) logf("vk: capture buffers over 16 MB in steps of %llu MB where spares are capped at 32 MB (RX_VK_CAP_STEP_MB)", (unsigned long long)n); + return n; + }(); + return v; +} +/// ((Triforce, 11-22, MEM-39) lab only, RX_VK_CAP_UNCACHED=1) capture buffers asked for as memory written in order, which +/// the allocator takes from an uncached (write-combined) type where the driver has one: the slow case a driver can hand +/// out unasked, made on any PC. Off unless set +bool cap_uncached() { + static const bool on = [] { + const char* e = getenv("RX_VK_CAP_UNCACHED"); + const bool v = e && atoi(e) != 0; + if (v) logf("vk: lab: capture buffers asked for as uncached host memory (RX_VK_CAP_UNCACHED=1)"); + return v; + }(); + return on; +} +VkBufferUsageFlags cap_usage(); // (below) +/// A capture buffer: cached host memory, mapped (own: a memory object of its own) +bool cap_buf_make(uint64_t size, VkBuffer& buffer, VmaAllocation& alloc, void** map, bool own) { + return cap_uncached() ? create_buffer_uncached(size, cap_usage(), buffer, alloc, map, own) + : create_buffer_cached(size, cap_usage(), buffer, alloc, map, own); +} +uint64_t cap_reserved_bytes = 0; // (cap_free_mu: the spares the workers are making, counted with the ones held) +size_t cap_reserved_n = 0; // (cap_free_mu: and how many) +std::atomic cap_made_n{0}, cap_made_us{0}; // (buffers made on a submitting thread, and their time: the 600-take line) +void cap_buf_free(VkBuffer buffer, VmaAllocation alloc, void* map, uint64_t size, bool from_worker = false) { + { + std::lock_guard lk(cap_free_mu); + if (from_worker && cap_shutting_down.load()) return; + // ((Triforce, 11-22, MEM-33) a worker's own spare: its reservation ends in the same hold that keeps it) + if (from_worker && cap_spare_fit_on()) cap_reserved_bytes -= size, cap_reserved_n--; + // (hookshot, RX 7800 XT: buffers up to 256 MB are kept for the next capture, not only up to 64 MB: a new one costs ~100 ms of + // vkAllocateMemory on AMD's driver. RX_VK_CAPTURE_KEEP_OLD=1: as before) + static const bool keep_old = getenv("RX_VK_CAPTURE_KEEP_OLD") != nullptr; + // ((Triforce, 11-22, the performance audit's MEM-141) while the swap hold is on, at most 32 MB of spares + // (linux_pressure.h): the buffer goes instead of waiting for a capture that may not come. Never on Windows + const bool swap_hold_on = memory_swap_hold(); + if (cap_spare::keeps(cap_free.size(), cap_reserved_n, cap_free_bytes, cap_reserved_bytes, size, + linux_pressure::spare_cap(swap_hold_on, cap_spare_max()), keep_old)) { + cap_free.push_back({buffer, alloc, map, size}); + cap_free_bytes += size; + return; + } + if (swap_hold_on) memory_swap_valve(2, size); + if (!cap_free_async_on()) { // (=0: destroyed here, under the lock, as before) + destroy_buffer(buffer, alloc); + return; + } + } + // (past the spares' limits: decided under the lock, destroyed after it) + if (!cap_shutting_down.load()) destroy_buffer_later(buffer, alloc, size); + else destroy_buffer(buffer, alloc); } void cap_buf_retire(std::map::iterator it) { cap_bytes -= it->second.size; @@ -1914,10 +2151,29 @@ bool cap_prewarm_on() { }(); return v; } +// ((Triforce, 11-21, the performance audit's MEM-48, after Source A's) A mirror sync's pages from the captures go to the +// mirror by the write-back diff's compute pass, one dispatch a sync, not as copy regions: msync_pass.h, msync_pass_add below. +// RX_VK_MSYNC_PASS: on for NVIDIA when not set, 1 on every GPU, 0 the copies as before +bool msync_pass_on() { + static const bool on = [] { + const bool v = msync_pass::on(getenv("RX_VK_MSYNC_PASS"), dev.props.vendorID == 0x10DE); + logf("vk: mirror syncs' pages from the captures %s (RX_VK_MSYNC_PASS: 1 the pass on every GPU, 0 copy regions)", + v ? "by a compute pass" : "as copy regions, as before"); + return v; + }(); + return on; +} +/// A capture buffer's usage: the pass reads it by its address +VkBufferUsageFlags cap_usage() { + static const VkBufferUsageFlags u = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | + (msync_pass_on() ? VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT : 0); + return u; +} std::atomic cap_prewarm_busy{0}; std::atomic cap_prewarm_class_busy[4]{}; // (32, 64, 128 and 192 MB spares being made) void cap_exit_note() { std::lock_guard lk(cap_free_mu); // (a worker inside cap_buf_free finishes first; one after this sees the flag under the same mutex) + std::lock_guard dg(cap_destroy_mu); // (and a destroy on destroy_buffer_later's thread; the next one sees the flag) cap_shutting_down = true; } int cap_prewarm_index(uint64_t size) { return size <= (32ull << 20) ? 0 : size <= (64ull << 20) ? 1 : size <= (128ull << 20) ? 2 : size <= (192ull << 20) ? 3 : -1; } @@ -1926,7 +2182,7 @@ void cap_spare_make(uint64_t size, int index) { VmaAllocation a = nullptr; void* m = nullptr; const auto t0 = std::chrono::steady_clock::now(); - if (!cap_shutting_down.load() && create_buffer_cached(size, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, b, a, &m, !getenv("RX_VK_CAP_BLOCKS")) && m) { + if (!cap_shutting_down.load() && cap_buf_make(size, b, a, &m, !getenv("RX_VK_CAP_BLOCKS")) && m) { static std::atomic told{0}; if (told++ < 8) logf("vk: a %llu MB capture buffer made ahead in %.1f ms, off the game's thread (RX_VK_CAP_PREWARM)", (unsigned long long)(size >> 20), @@ -1934,6 +2190,9 @@ void cap_spare_make(uint64_t size, int index) { // (a spare, within the same limits as any other: past them it is let go. Not once the process is exiting: the list and the device // may be going, and the memory goes with the process) cap_buf_free(b, a, m, size, true); + } else if (cap_spare_fit_on()) { // ((Triforce, 11-22, MEM-33) no buffer: what was reserved for it is free again) + std::lock_guard lk(cap_free_mu); + cap_reserved_bytes -= size, cap_reserved_n--; } cap_prewarm_class_busy[index]--; cap_prewarm_busy--; @@ -1942,10 +2201,14 @@ void cap_spare_make(uint64_t size, int index) { void cap_prewarm_class(uint64_t size) { if (!cap_prewarm_on()) return; const int index = cap_prewarm_index(size); - if (index < 0 || size > cap_spare_max()) return; // (not the 256 MB class, and not past what spares may hold) + const bool fit = cap_spare_fit_on(); + if (index < 0 || (!fit && size > cap_spare_max())) return; // (not the 256 MB class, and not past what spares may hold) static const uint64_t kCls[4] = {32, 64, 128, 192}; const uint64_t cls = kCls[index]; const uint64_t bytes = cls << 20; + // ((Triforce, 11-22, MEM-33) with the spares held and the ones being made it must fit what spares may hold, or it + // would be destroyed as it lands) + if (fit && !cap_spare::room(cap_free_bytes, cap_reserved_bytes, bytes, cap_spare_max())) return; for (const FreeBuf& f : cap_free) if (f.size >= bytes) return; if (cap_prewarm_class_busy[index].load() > 0 || cap_prewarm_busy.load() >= 3) return; // (three at a time: each commits its memory) @@ -1953,15 +2216,18 @@ void cap_prewarm_class(uint64_t size) { (void)exit_hook; cap_prewarm_busy++; cap_prewarm_class_busy[index]++; + if (fit) cap_reserved_bytes += bytes, cap_reserved_n++; // (before its thread starts: cap_spare_make gives it back) try { std::thread(cap_spare_make, bytes, index).detach(); } catch (...) { // (no thread: no spare, the capture goes on) cap_prewarm_class_busy[index]--; cap_prewarm_busy--; + if (fit) cap_reserved_bytes -= bytes, cap_reserved_n--; } } bool cap_buf_get(uint64_t need, VkBuffer& buffer, VmaAllocation& alloc, void*& map, uint64_t& size) { - std::lock_guard lk(cap_free_mu); + const bool fit = cap_spare_fit_on(); + std::unique_lock lk(cap_free_mu); if (cap_prewarm_on()) { static bool first = true; if (first) { // (the first capture: the classes a load asks for) @@ -1969,19 +2235,49 @@ bool cap_buf_get(uint64_t need, VkBuffer& buffer, VmaAllocation& alloc, void*& m for (uint64_t c : {32ull << 20, 64ull << 20, 128ull << 20, 192ull << 20}) cap_prewarm_class(c); } } - size_t best = cap_free.size(); - for (size_t i = 0; i < cap_free.size(); i++) - if (cap_free[i].size >= need && (best == cap_free.size() || cap_free[i].size < cap_free[best].size)) best = i; - if (best < cap_free.size()) { + const bool capped = cap_spare_max() <= cap_spare::kBig; // (a card of 6.5 GB or less: spares hold 32 MB) + const uint64_t cls = cap_buf_size(need); + // (the smallest spare that fits. (Triforce, 11-22, MEM-33) Unless any_size: not a big one far over this capture's class, + // which a load wants; and the spare a capture of a class of 32 MB or more takes is made again on a worker, for the + // next load. Not for a small capture that took a big spare: that one comes back. cap_free_mu held) + const auto take_spare = [&](bool any_size) { + const size_t best = cap_spare::pick(cap_free.size(), [](size_t i) { return cap_free[i].size; }, need, cls, any_size); + if (best == cap_free.size()) return false; buffer = cap_free[best].buffer, alloc = cap_free[best].alloc, map = cap_free[best].map, size = cap_free[best].size; cap_free_bytes -= size; cap_free.erase(cap_free.begin() + (ptrdiff_t)best); - return map != nullptr; - } - size = cap_buf_size(need); + if (fit && cls >= cap_spare::kBig) cap_prewarm_class(size); + return true; + }; + if (take_spare(!fit || capped)) return map != nullptr; + size = fit ? cap_spare::stepped(need, cls, cap_step_mb(), capped) : cls; map = nullptr; - return create_buffer_cached(size, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, buffer, alloc, &map, - size >= (16ull << 20) && !getenv("RX_VK_CAP_BLOCKS")) && map; + const bool own = size >= (16ull << 20) && !getenv("RX_VK_CAP_BLOCKS"); + const auto t0 = std::chrono::steady_clock::now(); + const auto made_note = [&] { + cap_made_n.fetch_add(1, std::memory_order_relaxed); + cap_made_us.fetch_add((uint64_t)std::chrono::duration_cast(std::chrono::steady_clock::now() - t0).count(), + std::memory_order_relaxed); + }; + if (!fit) { // (RX_VK_CAP_SPARE_FIT=0: made with the lock held, as before) + const bool ok = cap_buf_make(size, buffer, alloc, &map, own) && map; + made_note(); + return ok; + } + // ((Triforce, 11-22, MEM-33) no spare: made with the lock let go, so the replay's take, which holds pools_mu, does + // not wait behind this thread's allocation to give a buffer back) + lk.unlock(); + const bool made = cap_buf_make(size, buffer, alloc, &map, own); + made_note(); + if (made && map) return true; + if (made) destroy_buffer(buffer, alloc), buffer = VK_NULL_HANDLE, alloc = nullptr; // (made but not mapped: not kept) + // (no memory for it: any spare that fits, then the exact size without a memory object of its own) + lk.lock(); + if (take_spare(true)) return map != nullptr; + lk.unlock(); + size = need; + map = nullptr; + return cap_buf_make(size, buffer, alloc, &map, false) && map; } std::vector> cap_ranges; // (the mirrored pools the captures take, as of the last one) std::mutex cap_ranges_mu; @@ -2117,6 +2413,19 @@ bool cap_excl_page(uint64_t g) { while (cur < mine.size() && mine[cur].second <= g) cur++; return cur < mine.size() && mine[cur].first < g + 4096; } +/// ((Triforce, 11-22) RX_VK_WB_EXCL_CLIP) the parts of [lo, hi) outside the left-out ranges, appended to out; the bytes left +/// out returned (wb_excl_clip.h: a write-back from the mirror there put stale words over the CPU's newer ones) +uint64_t cap_excl_kept(uint64_t lo, uint64_t hi, std::vector>& out) { + static thread_local std::vector> mine; + static thread_local uint64_t ver = 0; + const uint64_t now = cap_excl_ver.load(std::memory_order_acquire); + if (ver != now) { + std::lock_guard lk(cap_excl_mu); + mine = cap_excl; + ver = cap_excl_ver.load(std::memory_order_relaxed); + } + return wb_excl_clip::kept(lo, hi, mine, out); +} void pool_records_ranges(std::vector>& out) { std::lock_guard lk(pool_rec_mu); for (const auto& [lo, hi] : pool_records) out.push_back({lo & ~4095ull, (hi + 4095) & ~4095ull}); @@ -2728,6 +3037,9 @@ struct WbPages { } bool empty() const { return !any; } } wb_pages; +// ((Triforce, 11-22, the performance audit's THREAD-25, after Source A's) every page wb_pages ever held a value for, set +// before the value and under wb_pages_mu, read with no lock: page_marks.h, memory_writeback_marked) +PageMarks wb_marks; struct WbRun { uint64_t lo, hi, value; }; // (a write-back's range in guest memory and its submission) std::vector wb_runs_recent; // (wb_pages_mu held) uint64_t wb_runs_lost = 0; // (the ranges of submissions up to this value were dropped unseen) @@ -2735,6 +3047,7 @@ void wb_pages_note(uint64_t guest_lo, uint64_t guest_hi, uint64_t value) { static const bool off = getenv("RX_VK_NO_READBACK_WAIT") != nullptr; if (off || !value || guest_hi <= guest_lo) return; std::lock_guard lk(wb_pages_mu); + wb_marks.mark(guest_lo, guest_hi); // ((Triforce, THREAD-25) before the pages have their value: a reader with no mark has no value to wait for) for (uint64_t p = guest_lo >> kPageShift; p <= (guest_hi - 1) >> kPageShift; p++) wb_pages.set(p, value); // (Triforce) and the range itself, while its submission may still be on the GPU: take_capture's late pages (wb_inflight_in) if (wb_runs_recent.size() >= 4096) { @@ -2804,7 +3117,7 @@ void late_unresolved_read(uint64_t page0, uint64_t page1) { st_late_read++; if (v > vk_timeline_completed()) { const auto t0 = std::chrono::steady_clock::now(); - vk_timeline_wait(v); + vk_timeline_wait_from(v, 1); // ((Triforce, 11-22, SYNC-122) wb_rec_wait.h kRead: the held part by caller) st_late_read_ms += std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); st_late_read_waited++; } @@ -2832,6 +3145,11 @@ uint64_t memory_writeback_value(uint64_t guest, uint64_t size, bool* ever) { v = wb_pages.max_in(guest >> kPageShift, (guest + size - 1) >> kPageShift, ever); return v; } +/// ((Triforce, THREAD-25) no lock) whether memory_writeback_value's "ever" can be true for the range: a write-back ever reached a +/// page of it, or the range lies where the marks are not kept. False: "ever" is false, now and until a write-back is noted there +bool memory_writeback_marked(uint64_t guest, uint64_t size) { + return wb_marks.any(guest, size); +} /// A copy that fails instead of faulting on a page unmapped meanwhile (structured exception handling: the common case /// costs a memcpy; ReadProcessMemory per page was ~2.3 us each). Not inlined, and the copy a call: without @@ -3057,7 +3375,7 @@ void copy_helper_main() { int capture_copy_helpers() { // (copy_pages and copy_pairs share one pool of helpers, started by the first call that splits) static const int helpers = getenv("RX_VK_CAPTURE_THREADS") ? std::clamp(atoi(getenv("RX_VK_CAPTURE_THREADS")), 0, 4) - : (getenv("RX_VK_LOW_SPEC") ? 0 : std::thread::hardware_concurrency() >= 12 ? 4 : 2); // (four where the threads are spare) + : (getenv("RX_VK_LOW_SPEC") ? 0 : cpu_count() >= 12 ? 4 : 2); // (four where the threads are spare) return helpers; } /// (hookshot) whole's n copies in helpers + 1 parts: the first on the calling thread, the others queued for the helpers. @@ -3190,7 +3508,7 @@ std::unique_ptr capture_make(bool behind) { if (!spare.empty()) c = std::move(spare.back()), spare.pop_back(); } if (!c) c = std::make_unique(); - c->pages.clear(), c->copied.clear(), c->slots.clear(), c->nbytes = 0, c->late.clear(), c->late_value = 0; + c->pages.clear(), c->copied.clear(), c->slots.clear(), c->nbytes = 0, c->late.clear(), c->late_value = 0, c->late_done = 0; c->behind.reset(); if (c->map) cap_buf_free(c->buffer, c->alloc, c->map, c->bsize), c->buffer = VK_NULL_HANDLE, c->map = nullptr; stage_put(*c); @@ -3250,6 +3568,7 @@ std::unique_ptr capture_make(bool behind) { uint64_t v = 0; if (cap_wait) { const uint64_t done = vk_timeline_completed(); + c->late_done = done; // ((Triforce, 11-22, MEM-80) kept for take_capture's partial rule: one read of the progress) std::lock_guard lk(wb_pages_mu); if (!wb_pages.empty()) for (int64_t i = 0; i < n; i++) { @@ -3290,16 +3609,26 @@ std::unique_ptr capture_make(bool behind) { // (only where the capture buffer is not cached memory: a cached one the shadow reads as fast as the staging, and the // extra copy would be the game thread's for nothing) c->buf_cached = host_cached(c->alloc); + if (!c->buf_cached) { // ((Triforce, 11-22, MEM-39) once: what the driver gave instead of cached memory + static std::atomic told{false}; + if (!told.exchange(true)) { + VkMemoryPropertyFlags flags = 0; + const uint32_t type = alloc_memory_type(c->alloc, &flags); + logf("vk: a capture buffer is not in cached host memory: memory type %u (flags %#x), %llu MB%s. The CPU reads " + "such memory slowly, so its texture pages are staged in ordinary memory and read from there (RX_VK_SHADOW_STAGE)", + type, (unsigned)flags, (unsigned long long)(c->bsize >> 20), cap_uncached() ? " (asked for: RX_VK_CAP_UNCACHED=1)" : ""); + } + } if (stage_on() && !texr.empty() && !c->buf_cached) { c->sidx.assign((size_t)n, -1); int32_t tn = 0; size_t lj = 0; + RangeWalk in_texr{texr, range_walk_search()}; // ((Triforce, 11-22, MEM-42) texr is this thread's: one walk for the loop) for (int64_t i = 0; i < n; i++) { while (lj < c->late.size() && c->late[lj] < (uint32_t)i) lj++; if (lj < c->late.size() && c->late[lj] == (uint32_t)i) continue; const uint64_t g = (uint64_t)pages[i] << kPageShift; - auto it = std::upper_bound(texr.begin(), texr.end(), std::make_pair(g, ~uint64_t(0))); - if (it != texr.begin() && g < std::prev(it)->second) c->sidx[(size_t)i] = tn++; + if (in_texr.in(g)) c->sidx[(size_t)i] = tn++; } if (!tn || !stage_get(*c, (size_t)tn * 4096)) c->sidx.clear(); } @@ -3312,12 +3641,13 @@ std::unique_ptr capture_make(bool behind) { ahead.assign((size_t)n, 0); int64_t na = 0; for (uint32_t i : c->late) na += !ahead[i], ahead[i] = 1; - if (!texr.empty()) + if (!texr.empty()) { + RangeWalk in_texr{texr, range_walk_search()}; // ((Triforce, 11-22, MEM-42) the pages are not always sorted: a step back searches) for (int64_t i = 0; i < n; i++) { const uint64_t g = (uint64_t)pages[i] << kPageShift; - auto it = std::upper_bound(texr.begin(), texr.end(), std::make_pair(g, ~uint64_t(0))); - if (!ahead[(size_t)i] && it != texr.begin() && g < std::prev(it)->second) ahead[(size_t)i] = 1, na++; + if (!ahead[(size_t)i] && in_texr.in(g)) ahead[(size_t)i] = 1, na++; } + } if (capture_behind::worth_behind(capture_copy_helpers(), n, na)) { b = std::make_shared(); b->pages.assign(pages, pages + n); @@ -3445,7 +3775,8 @@ void late_slow_log(const Capture& c, uint64_t v, uint64_t done0, double ms, doub } { std::lock_guard sk(shadow_mu); - for (size_t j = 0; j < m; j++) is_tex[j] = !tex_merged.empty() && in_tex_merged((uint64_t)c.pages[c.late[j]] << kPageShift); + RangeWalk in_tex{tex_merged, range_walk_search()}; // ((Triforce, 11-22, MEM-42) under shadow_mu, for this loop) + for (size_t j = 0; j < m; j++) is_tex[j] = !tex_merged.empty() && in_tex.in((uint64_t)c.pages[c.late[j]] << kPageShift); } { const uint64_t done = vk_timeline_completed(); @@ -3587,7 +3918,13 @@ bool take_capture(std::unique_ptr c, uint64_t value, uint64_t completed // alone cut (RX_VK_LATE_TEST=2), stayed clean: the wait itself orders something the game's frame depends on, not found // yet. What is known is in the comments here and at late_partial, deferred_late, late_unresolved) static const bool late_new_env = getenv("RX_VK_LATE_NEW") && atoi(getenv("RX_VK_LATE_NEW")) != 0; - const bool late_all = !(late_new_env || lab_ab::on("latenew")); // (lab: the knob turns the rules on for its phases) + // ((Triforce, 11-22, the performance audit's MEM-80, after Source B's) RX_VK_LATE_INLINE=1 or the knob lateinline: the + // partial rule alone, the other three off (late_inline.h); unset, the rules are RX_VK_LATE_NEW's as before + static const char* late_inline_env = getenv("RX_VK_LATE_INLINE"); + const late_inline::Rules rules = late_inline::rules(late_new_env || lab_ab::on("latenew"), // (lab: the knob turns the rules on for its phases) + late_inline::on(late_inline_env, lab_ab::on("lateinline")), late_same_on(), + late_part_on(), late_defer_on()); + const bool late_all = !rules.any; uint64_t v = c->late_value; if (!late_all) { static thread_local std::vector need; @@ -3601,11 +3938,12 @@ bool take_capture(std::unique_ptr c, uint64_t value, uint64_t completed if (it != imported.begin() && (--it, guest >= it->second.lo && guest < it->second.hi)) p = &it->second; // (a page of no imported pool: as before) if (p && p->mstate != Pool::MIRROR_ON && p->mstate != Pool::MIRROR_WANTED) { - late_skipped_now.push_back(i); + if (rules.skip) late_skipped_now.push_back(i); + else need.push_back(i); // ((Triforce, 11-22, MEM-80) the partial rule alone: waited for, as by default) continue; } // (the same bytes as when the page was last synced: nothing of the CPU's to bring to the mirror. See late_prev) - if (late_same_on() && std::binary_search(c->slots.begin(), c->slots.end(), i)) { + if (rules.same && std::binary_search(c->slots.begin(), c->slots.end(), i)) { // (as last synced: the copy of the capture that stamped it last, while that is kept; else the bytes // noted when it was last waited for, if no capture stamped it since) // (by the captures' own entries, not the page's stamp: the GPU's write-backs stamp the page too) @@ -3621,12 +3959,13 @@ bool take_capture(std::unique_ptr c, uint64_t value, uint64_t completed if (pv != late_prev.end() && pv->second.gen == e.gen) was = pv->second.bytes.data(); } if (was) cap_ready(*c, i); // ((Triforce) copied ahead: a late page; behind_ready's quick look) + // ((Triforce, 11-22, MEM-39) a read of the capture buffer, cached or not: a late page is never staged) if (was && !memcmp(was, (const uint8_t*)c->map + (uint64_t)i * 4096, 4096)) { late_same_now.push_back(i); continue; } } - if (late_part_on() && std::binary_search(c->slots.begin(), c->slots.end(), i)) late_part_cand.push_back(i); + if (rules.part && std::binary_search(c->slots.begin(), c->slots.end(), i)) late_part_cand.push_back(i); else need.push_back(i); } } @@ -3645,7 +3984,9 @@ bool take_capture(std::unique_ptr c, uint64_t value, uint64_t completed } st_late_same += (int64_t)late_same_now.size(); if (!late_part_cand.empty()) { // (the parts of each no submission on the GPU writes back: see late_partial) - const uint64_t done = vk_timeline_completed(); + // ((Triforce, 11-22, MEM-80) against the progress capture_make read before its copies, not a second read: a + // write-back done since then counts as in flight, and its range is not synced from the older copy) + const uint64_t done = c->late_done; std::lock_guard wk(wb_pages_mu); for (uint32_t i : late_part_cand) { std::pair g[LatePartial::kRanges]; @@ -3654,7 +3995,7 @@ bool take_capture(std::unique_ptr c, uint64_t value, uint64_t completed lp.n = 0; // (not known, or nothing of it on the GPU any more: taken again as before; all of it written back: waited for) const bool known = wb_inflight_in(c->pages[i], done, g, LatePartial::kRanges, &n); - if (known && n && g[0].first == 0 && g[0].second == 4096 && late_defer_on() && + if (known && n && g[0].first == 0 && g[0].second == 4096 && rules.defer && !par_active.load(std::memory_order_relaxed)) { // (written back whole: put off, see deferred_late) late_defer_now.push_back(i); continue; @@ -3753,9 +4094,11 @@ bool take_capture(std::unique_ptr c, uint64_t value, uint64_t completed bool tex = false; if (tex_capture_on()) { std::lock_guard sk(shadow_mu); - if (!tex_merged.empty()) + if (!tex_merged.empty()) { + RangeWalk in_tex{tex_merged, range_walk_search()}; // ((Triforce, 11-22, MEM-42) under shadow_mu, for this loop) for (uint32_t i : c->late) - if (in_tex_merged((uint64_t)c->pages[i] << 12)) { tex = true; break; } + if (in_tex.in((uint64_t)c->pages[i] << 12)) { tex = true; break; } + } } size_t pending = 0; { std::lock_guard fl(late_fix_mu); pending = late_fixes.size(); } @@ -3899,16 +4242,25 @@ bool take_capture(std::unique_ptr c, uint64_t value, uint64_t completed size_t sh_any = 0; if (tex_same && !c->copied.empty() && c->map && tex_capture_on()) { std::lock_guard sk(shadow_mu); + // ((Triforce, 11-22, the performance audit's MEM-39, after Source A's) The compare below read the capture buffer, + // which a driver may hand out as write-combined memory though cached was asked for (the CPU reads that at about + // 0.3 GB/s: 3 to 14 microseconds a page, for every texture page compared, under pools_mu and shadow_mu). It reads + // the staging now when that holds the page, as the shadow's copies below do: the same bytes, copied there first + // and to the buffer from there. With no staging (a cached buffer, RX_VK_SHADOW_STAGE=0) cap_page_bytes is the + // buffer, as before. The lab knob texsamemap reads the buffer in its phases) + const bool same_from_map = lab_ab::on("texsamemap"); if (!tex_merged.empty() && shadow_n.load(std::memory_order_relaxed)) { + RangeWalk in_tex{tex_merged, range_walk_search()}; // ((Triforce, 11-22, MEM-42) under shadow_mu, for this loop) for (size_t i = 0; i < c->copied.size(); i++) { const uint32_t p = c->copied[i]; if (!dropped.empty() && dropped[c->slots[i]]) continue; - if (!in_tex_merged((uint64_t)p << 12)) continue; + if (!in_tex.in((uint64_t)p << 12)) continue; const auto s = shadow.find(p); if (s == shadow.end() || !s->second.b) continue; cap_ready(*c, c->slots[i]); // ((Triforce) texture storage: copied ahead, unless its ranges grew since) if (sh_on && !sh_any++) sh_looks.assign(c->copied.size(), shadow_prev::Look{}); // (the first look: sized) - if (memcmp(s->second.b.get(), (const uint8_t*)c->map + (uint64_t)c->slots[i] * 4096, 4096) != 0) { + const uint8_t* const cap_b = same_from_map ? (const uint8_t*)c->map + (uint64_t)c->slots[i] * 4096 : cap_page_bytes(*c, c->slots[i]); + if (memcmp(s->second.b.get(), cap_b, 4096) != 0) { st_tex_changed++; if (sh_on) // (ready above: the copies before the last changes compared with the capture's) sh_looks[i] = shadow_prev::look(false, s->second.b.get(), s->second.prev, cap_page_bytes(*c, c->slots[i]), sh_prev, @@ -3958,6 +4310,8 @@ bool take_capture(std::unique_ptr c, uint64_t value, uint64_t completed cap_ready(*c, i); LatePrev& lp = late_prev[c->pages[i]]; lp.gen = g; + // ((Triforce, 11-22, MEM-39) a read of the capture buffer, cached or not: a late page is never staged, its + // second copy is in the buffer alone; a few a take) memcpy(lp.bytes.data(), (const uint8_t*)c->map + (uint64_t)i * 4096, 4096); } } @@ -3998,15 +4352,18 @@ bool take_capture(std::unique_ptr c, uint64_t value, uint64_t completed // (hookshot: those copied now are overwritten in place below, their buffers kept: erasing every written page and // inserting the copied ones again was a free, an allocation and two hash operations a page, for the pages the // game rewrites every frame. c->slots[i] is the index in c->pages of c->copied[i], in order) + // ((Triforce, 11-22, MEM-42) the loops below each have their walk over tex_merged: shadow_mu is held from here + // to the end of them, and nothing in them publishes the spans) { size_t j = 0; + RangeWalk in_tex{tex_merged, range_walk_search()}; for (size_t i = 0; i < c->pages.size(); i++) { if (j < c->slots.size() && c->slots[j] == i) { j++; continue; } const uint32_t p = c->pages[i]; - if (in_tex_merged((uint64_t)p << 12)) shadow_erase(p); + if (in_tex.in((uint64_t)p << 12)) shadow_erase(p); } } bool swept = false; @@ -4030,14 +4387,16 @@ bool take_capture(std::unique_ptr c, uint64_t value, uint64_t completed static int64_t st_sh_back_d[shadow_prev::kMaxVers]; // ((Triforce) the versions given back, by how far back) sh_pages.clear(), sh_left.clear(); if (sh_any) { + RangeWalk in_tex{tex_merged, range_walk_search()}; for (uint32_t p : c->copied) - if (in_tex_merged((uint64_t)p << 12)) sh_pages.push_back(p); + if (in_tex.in((uint64_t)p << 12)) sh_pages.push_back(p); std::sort(sh_pages.begin(), sh_pages.end()); } size_t i = 0; + RangeWalk in_tex{tex_merged, range_walk_search()}; // (this loop's, and the one's that ends it below) for (; i < c->copied.size(); i++) { const uint32_t p = c->copied[i]; - if (!in_tex_merged((uint64_t)p << 12)) continue; + if (!in_tex.in((uint64_t)p << 12)) continue; if (shadow_n.load(std::memory_order_relaxed) >= kShadowMax && !shadow.count(p)) { if (swept) break; swept = true; @@ -4091,6 +4450,7 @@ bool take_capture(std::unique_ptr c, uint64_t value, uint64_t completed const int32_t sx = c->sidx.empty() ? -1 : c->sidx[c->slots[i]]; cap_ready(*c, c->slots[i]); // ((Triforce) as for tex_same above) if (sx >= 0) pairs.push_back({slot.b.get(), (const uint8_t*)c->stage.get() + (uint64_t)sx * 4096}), from_stage++; + // ((Triforce, 11-22, MEM-39) a read of the capture buffer: unstaged pages only, so a cached buffer's, or a late page's) else pairs.push_back({slot.b.get(), (const uint8_t*)c->map + (uint64_t)c->slots[i] * 4096}), from_map++; } // (shadow_mu still held: no slot taken here goes before its copy; the sweep above drops only older ones) @@ -4107,6 +4467,15 @@ bool take_capture(std::unique_ptr c, uint64_t value, uint64_t completed "host-cached, %lld not (write-combined: staged)%s", (long long)st_from_stage, (long long)st_from_map, (long long)st_cached, (long long)(600 - st_cached), stage_on() ? "" : " (RX_VK_SHADOW_STAGE=0)"); st_from_stage = st_from_map = 0, st_cached = 0; + // ((Triforce, 11-22, MEM-33) the capture buffers no spare served, made on a submitting thread) + static uint64_t made_told = 0, made_us_told = 0; + if (const uint64_t n = cap_made_n.load(std::memory_order_relaxed); n != made_told) { + const uint64_t us = cap_made_us.load(std::memory_order_relaxed); + logf("vk: capture buffers made on a submitting thread (600 takes): %llu in %.1f ms, %llu so far (no spare fit; " + "%s; RX_VK_CAP_SPARE_FIT)", (unsigned long long)(n - made_told), (double)(us - made_us_told) / 1000.0, + (unsigned long long)n, cap_spare_fit_on() ? "with the spares' lock let go" : "with the spares' lock held, as before"); + made_told = n, made_us_told = us; + } } for (const auto& [sp, slot_i] : sh_left) { // ((Triforce) RX_VK_SHADOW_CHECK=1: the copies left, against the capture's) const auto hash = [](const uint8_t* b, uint64_t n) { return upload_same::content_hash(b, n); }; @@ -4134,7 +4503,7 @@ bool take_capture(std::unique_ptr c, uint64_t value, uint64_t completed } } for (; i < c->copied.size(); i++) // (the shadow full: the copied pages not replaced lose their old copies) - if (in_tex_merged((uint64_t)c->copied[i] << 12)) shadow_erase(c->copied[i]); + if (in_tex.in((uint64_t)c->copied[i] << 12)) shadow_erase(c->copied[i]); } } const auto tk1 = std::chrono::steady_clock::now(); @@ -4213,13 +4582,16 @@ bool take_capture(std::unique_ptr c, uint64_t value, uint64_t completed "for (%lld ranges unknown, %lld none on the GPU now, %lld written back whole), %lld not: no mirror, %lld not: " "the same bytes, %lld synced in part, %lld put off (%lld synced later, %lld at a binding); %lld of those taken " "again at the replay's first read, %lld after a wait, %.3f ms a take), pools lock %.3f, retires " - "and stamps %.3f, bytes copied %.3f; %zu kept (%llu MB), %zu spare buffers, %zu texture pages copied (%llu MB)", + "and stamps %.3f, bytes copied %.3f; %zu kept (%llu MB), %zu spare buffers, %lld retired ones (%lld MB) destroyed " + "on a thread of their own (RX_VK_CAP_FREE_ASYNC), %zu texture pages copied (%llu MB)", st_tk_all / 600, st_tk_late / 600, (long long)st_tk_late_n, (long long)st_late_needed, (long long)st_late_why[0], (long long)st_late_why[1], (long long)st_late_why[2], (long long)st_late_skipped, (long long)st_late_same, (long long)st_late_part, (long long)st_late_deferred, (long long)st_late_ready, (long long)st_late_jit, (long long)st_late_read, (long long)st_late_read_waited, st_late_read_ms / 600, st_tk_lock / 600, st_tk_stamp / 600, st_tk_copy / 600, - cap_bufs.size(), (unsigned long long)(cap_bytes >> 20), cap_free.size(), shadow_n.load(std::memory_order_relaxed), + cap_bufs.size(), (unsigned long long)(cap_bytes >> 20), cap_free.size(), + (long long)cap_destroyed_later.exchange(0, std::memory_order_relaxed), + (long long)(cap_destroyed_later_bytes.exchange(0, std::memory_order_relaxed) >> 20), shadow_n.load(std::memory_order_relaxed), (unsigned long long)(shadow_n.load(std::memory_order_relaxed) >> 8)); if (late_queue_on()) { int64_t qw, qns, qto; @@ -4559,10 +4931,34 @@ void import_ahead_look() { } } +bool mirror_pieces_on(); // (below, with the mirrors in pieces) +/// ((Triforce, 11-22, the performance audit's MEM-16, after Source A's) Whether an import's chunk is 4 MB and not 16. An import +/// covers whole chunks of the game's address space, and a whole mirror copies all of its import into video memory, the +/// padding too: Source A measured Hyrule Field's mirrors at 1,236-1,268 MB with 4 MB chunks against 1,645-1,657 with 16, for +/// about half again as many imports (91-94 against 59-65 in five minutes). The mirrors in pieces bind memory only where +/// bindings reach, so the gain is where the mirrors are whole: NVIDIA's cards before Turing by default, and any without sparse +/// residency buffers. Only NVIDIA's: an import stalled the replay 20 to 75 ms on an RX 9060 XT (import_in_background's +/// note), so AMD's and Intel's whole mirrors keep 16 MB until a lab run there. The card's maker is asked first: on the other +/// makes' cards mirror_pieces_on asks the memory trial, which is not this rule's to start +bool small_chunks_wanted() { + return dev.props.vendorID == 0x10de && mirror_on() && !mirror_pieces_on(); +} /// (imports by 16 MB chunk (RX_VK_IMPORT_CHUNK_MB), not by pool: each import cost 5-45 ms in the driver, more as their /// number grew, and the game makes ~13 pools a second; a chunk covers the pools after it at no cost) +/// ((Triforce, 11-22, MEM-16) 4 MB where small_chunks_wanted says so; RX_VK_IMPORT_CHUNK_MB still comes first, and =16 is the +/// size as it was. Worked out at the first import, once the device is there, and said once) uint64_t import_chunk() { - static const uint64_t chunk = (uint64_t)(getenv("RX_VK_IMPORT_CHUNK_MB") ? std::max(0, atoi(getenv("RX_VK_IMPORT_CHUNK_MB"))) : 16) << 20; + static const uint64_t chunk = [] { + const char* e = getenv("RX_VK_IMPORT_CHUNK_MB"); + const bool small = !e && small_chunks_wanted(); + const uint64_t mb = e ? (uint64_t)std::max(0, atoi(e)) : small ? 4 : 16; + logf("vk: imports in chunks of %llu MB%s", (unsigned long long)mb, + e ? " (RX_VK_IMPORT_CHUNK_MB)" + : small ? " (whole mirrors on an NVIDIA card: less of an import's padding copied into video memory; " + "RX_VK_IMPORT_CHUNK_MB=16: as before)" + : ""); + return mb << 20; + }(); return chunk; } /// (hookshot) A game pool across a chunk's end had each chunk imported on its own, at the first binding in it, and a later @@ -4778,6 +5174,15 @@ bool mirror_on() { }(); return on != 0; } +/// ((Triforce, 11-22, the performance audit's MEM-51, after Source A's) No mirrors, so no write-back: nothing reads what +/// memory_cpu_owned, shadow_written_to and guest_cpu_wrote note for one (the replay's own report words, which a write-back +/// leaves out, and the pages the CPU wrote, which the write-back diff's shadows must read again). The readers, +/// cut_cpu_words and shadows_forget_written, run only for a submission with write-backs, and a write-back is queued only +/// past gpu_buffer_bind's !mirror_on() return, for a pool with a mirror. Without mirrors (Intel's integrated GPUs, +/// RX_VK_MIRROR=0) the lists grew to 65,536 words and 8 million pages (32 MB), were dropped and grew again, for nothing. +/// mirror_on() is fixed for the run, so the notes never start part-way. Not asked before the device is made: mirror_on() +/// decides once, by the card. The switch back is the mirrors' own: with RX_VK_MIRROR=1 everything is noted, as before +bool wb_notes_off() { return dev.device != VK_NULL_HANDLE && !mirror_on(); } /// (hookshot, smoothness) A read binding in a chunk not imported yet: its import goes to the background thread and the binding is /// served from a copy in the recording's stream, rather than the replay waiting for the import. Imports on the replay thread were @@ -4834,19 +5239,18 @@ uint64_t device_vram_bytes() { bool preset_caps_memory() { return false; } namespace { uint64_t mirror_budget() { - // (a quarter of the card, at most the 3 GB it was for all: on a 6 GB laptop those and the textures were past what the card has) + // (by the card, mirror_budget.h: a quarter on discrete cards over 4.5 GB, at most 3 GB; an eighth on smaller ones and on + // integrated GPUs up to 6.5 GB. On 8 GB cards, a 3070 or a 4060, the game's memory read over PCIe instead of mirrored left the + // GPU at a few percent and the frame stuttering) static const uint64_t b = [] { if (const char* e = getenv("RX_VK_MIRROR_MB")) return (uint64_t)atoi(e) << 20; const uint64_t vram = device_vram_bytes(); - // (cards of 6 GB or less only: on 8 GB ones, a 3070 or a 4060, the game's memory read over PCIe instead left the GPU at a few - // percent and the frame stuttering) - if (vram && vram <= (6ull << 30) + (512ull << 20)) return vram / 8; - return vram ? std::min(vram / 4, 3072ull << 20) : (uint64_t)(3072ull << 20); + const bool discrete = dev.props.deviceType == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU; + const uint64_t n = mirror_size::bytes(vram, discrete); + logf("vk: VRAM mirrors' budget %llu MB (%llu MB of video memory, %s; RX_VK_MIRROR_MB= sets it)", + (unsigned long long)(n >> 20), (unsigned long long)(vram >> 20), discrete ? "discrete" : "not discrete"); + return n; }(); - // (the Insert menu's quality preset sets one, 256 MB at the 4 GB preset then: it was never read, and an 8 GB card kept ~2 GB of - // mirrors with the 4GB preset on, 2026-11-08) - // (only on a card the preset is for: preset_caps_memory) - if (opts.mirror_mb > 0 && !getenv("RX_VK_MIRROR_MB") && preset_caps_memory()) return (uint64_t)opts.mirror_mb << 20; return b; } uint64_t mirror_bytes = 0, mirror_epoch = 0; @@ -4894,7 +5298,7 @@ void cpu_words_merge() { // (holding cpu_words_mu) } // namespace void memory_cpu_owned(uint64_t guest, uint64_t size) { static const bool off = getenv("RX_VK_WB_REPORTS") != nullptr; - if (off || !guest) return; + if (off || !guest || wb_notes_off()) return; // ((Triforce, 11-22, MEM-51) no mirrors: no write-back cuts these words out) std::lock_guard lk(cpu_words_mu); if (cpu_words.size() + cpu_words_recent.size() + cpu_words_new.size() > 65536) { // (as before: forgotten when it gets this big) cpu_words_merge(); // (which leaves cpu_words_new empty) @@ -4943,6 +5347,7 @@ void retire_pool(std::map::iterator it) { shadow_free(*old); // (its write-backs in this submission, if any, copied whole) guest_pages_set(old->lo, old->hi, 0); old->retired_value = std::max(old->last_value, mem_rec_value); + old->retired_ms = std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); std::erase(mirror_wanted, old); // (the new pool gets the mirror) if (old->mstate == Pool::MIRROR_WANTED) old->mstate = Pool::MIRROR_NONE; // (EvilOOT, 11-07) what the GPU wrote into its mirror and nowhere else (the culling's bare table entries past their first @@ -4967,6 +5372,44 @@ std::unordered_map> direct_by_pool; // (bind std::unordered_map direct_bytes_by_pool; // (and the bytes of those bindings: what the GPU may read over PCIe, a binding's whole range) int64_t live_seen = 0; // (pool -> pages its syncs read live, since the last report) +// ---- ((Triforce, 11-21, MEM-48) the mirror syncs' pass: what is declared ahead of the syncs; the rest after wbd_table_get) ---- +int64_t st_msync_dispatches = 0, st_msync_chunks = 0, st_msync_fallbacks = 0, st_msync_barriers = 0; // (pools_mu) +/// (pools_mu held) The writes one sync or fill makes into p's mirror in the upload command buffer being recorded, kept in +/// order with the pass: copies into a buffer are in order among themselves, a dispatch is not, so where the mirror was +/// written earlier in that command buffer and one of the two is the pass, a full barrier goes first (msync_pass.h +/// barrier_first). What was there before is looked at once, at the first write: a sync's own copies and its pass never +/// share a page. With the pass off nothing is noted and no barrier is added +struct UpWrites { + Pool& p; + VkCommandBuffer cmd = VK_NULL_HANDLE; + bool looked = false; + uint8_t prior = 0, mine = 0; + explicit UpWrites(Pool& pool) : p(pool) {} + UpWrites(const UpWrites&) = delete; + void write(uint8_t kind) { + if (!msync_pass_on()) return; + if (!looked) { + looked = true, cmd = upload_command_buffer(); + prior = p.up_cmd == cmd && p.up_value == mem_rec_value ? p.up_kinds : 0; + } + if (msync_pass::barrier_first(prior, kind)) { + full_barrier(cmd, "barrier (a mirror written by copies and by the syncs' pass)"); + prior = 0, st_msync_barriers++; + } + mine |= kind; + } + ~UpWrites() { + if (mine) p.up_cmd = cmd, p.up_value = mem_rec_value, p.up_kinds = (uint8_t)(prior | mine); + } +}; +/// (pools_mu held) Before copies into p's mirror in the uploads that are no sync's: a fill, a late page, a new mirror's +void mirror_up_copy(Pool& p) { + UpWrites(p).write(msync_pass::kCopies); +} +bool msync_pass_add(Pool& p, VkBuffer buf, const std::vector& rs); +void msync_pass_flush(UpWrites& up); +void msync_pass_report(); + void transfer_barrier(VkCommandBuffer cmd) { full_barrier(cmd, "barrier (write-back or sync)"); // (the same barrier; one right after another is left out) } @@ -5086,6 +5529,28 @@ inline bool pieces_resident(const Pool* p, uint64_t off, uint64_t size) { } return true; } +// ((Triforce, 11-22, the performance audit's MEM-26, its first step, after Source A's; RX_VK_MIRROR_PIECE_IDLE=, +// off unless set) piece_idle.h says what is counted and why nothing is released yet. Every 1200 submissions the "mirrors in +// pieces" line is followed by the MB of pieces no game pool covers; with the switch each binding that is given a mirror in +// pieces also notes the submission in its pieces (a relaxed store a piece), and the line says how much of those MB no +// binding reached for that many submissions (Source A waits 600). Unset or 0: nothing is noted at a binding +uint64_t piece_idle_for() { + static const uint64_t v = [] { + const char* e = getenv("RX_VK_MIRROR_PIECE_IDLE"); + const uint64_t n = e && atoi(e) > 0 ? (uint64_t)atoi(e) : 0; + if (n) logf("vk: mirrors in pieces: the pieces no binding reached for %llu submissions are counted (RX_VK_MIRROR_PIECE_IDLE; " + "nothing is released)", (unsigned long long)n); + return n; + }(); + return v; +} +/// A binding was given the pool's mirror over [off, off + size): its pieces were reached in this submission +inline void pieces_touch(Pool* p, uint64_t off, uint64_t size) { + if (!piece_idle_for() || !p->mpieces || !size || p->piece_last.empty()) return; + const uint32_t now = (uint32_t)mirror_epoch; + for (uint64_t i = off >> p->piece_shift, e = std::min((off + size - 1) >> p->piece_shift, p->piece_last.size() - 1); i <= e; i++) + __atomic_store_n(&p->piece_last[i], now, __ATOMIC_RELAXED); +} /// Copies into the pool's mirror (by dstOffset; by_src: out of it, by srcOffset), cut down to its resident pieces void clip_resident(const Pool& p, std::vector& copies, bool by_src = false) { if (!p.mpieces || copies.empty()) return; @@ -5137,6 +5602,7 @@ void mirror_captures_over(Pool* p, uint64_t a, uint64_t b) { di.memoryBarrierCount = 1; di.pMemoryBarriers = &mb; vkCmdPipelineBarrier2(upload_command_buffer(), &di); + mirror_up_copy(*p); // ((Triforce, 11-21, MEM-48) in order with the syncs' pass) for (const auto& [buf, rs] : over) vkCmdCopyBuffer(upload_command_buffer(), buf, p->mirror, (uint32_t)rs.size(), rs.data()); } @@ -5189,6 +5655,7 @@ bool mirror_pieces_make(Pool& p, uint64_t off, uint64_t size, bool captures, std memory_trial_used(MEMTRIAL_PIECES); // (Triforce: the mirrors in pieces ran, for the memory trial; once a session) // (Triforce, lazy write-back: the copies read guest memory: nothing owed there) for (const VkBufferCopy& c : copies) lazy_settle(p.lo + c.srcOffset, p.lo + c.srcOffset + c.size, wb_lazy::BY_GPU); + mirror_up_copy(p); // ((Triforce, 11-21, MEM-48) in order with the syncs' pass) { GprofSpan gp(upload_command_buffer(), "mpiece copy (a part of a pool -> its mirror's new pieces)"); gp.items = (int)copies.size(); @@ -5237,6 +5704,7 @@ void mirror_sync(Pool& p) { p.synced_epoch = mirror_epoch; p.sync_seg = texture_segment(); mirror_synced.push_back(&p); + UpWrites up(p); // ((Triforce, 11-21, MEM-48) this sync's writes into the mirror, in order with the pass) std::vector runs; // (from guest memory) std::map> cap_runs; // (from the captures' copies) uint64_t run_lo = 0, run_hi = 0; @@ -5384,19 +5852,31 @@ void mirror_sync(Pool& p) { demote_wanted.push_back(p.lo); } } + // ((Triforce, 11-21, the performance audit's MEM-48, after Source A's) the captures' pages as the pass's chunks, one + // dispatch for the sync where their copy regions were (msync_pass.h); a buffer whose runs are not whole words is copied + // as before, and so is the sync when no chunk table has room (msync_pass_flush). The lab's knob "nomsyncpass": copies) + const bool pass = msync_pass_on() && !lab_ab::on("nomsyncpass"); for (auto& [buf, rs] : cap_runs) { clip_resident(p, rs); // (Triforce: a mirror in pieces takes the pages of its resident pieces only) if (rs.empty()) continue; + if (pass && msync_pass_add(p, buf, rs)) { + for (const VkBufferCopy& r : rs) st_sync_bytes += (int64_t)r.size; + st_sync_copies += (int64_t)rs.size(); + continue; + } + up.write(msync_pass::kCopies); GprofSpan gp(upload_command_buffer(), "msync copies (CPU pages from a capture -> mirror)"); gp.items = (int)rs.size(); vkCmdCopyBuffer(upload_command_buffer(), buf, p.mirror, (uint32_t)rs.size(), rs.data()); for (const VkBufferCopy& r : rs) st_sync_bytes += (int64_t)r.size, gp.bytes += r.size; st_sync_copies += (int64_t)rs.size(); } + msync_pass_flush(up); // (ahead of the live runs: other pages of the mirror) clip_resident(p, runs); if (runs.empty()) return; // (Triforce, lazy write-back: the copies read guest memory: nothing owed there) for (const VkBufferCopy& r : runs) lazy_settle(p.lo + r.srcOffset, p.lo + r.srcOffset + r.size, wb_lazy::BY_GPU); + up.write(msync_pass::kCopies); checkpoint_mark(upload_command_buffer(), p.lo, 'm'); GprofSpan gp(upload_command_buffer(), "msync copies (CPU pages live from guest memory -> mirror)"); gp.items = (int)runs.size(); @@ -5558,6 +6038,8 @@ bool memory_rb_refresh(VkCommandBuffer up) { clip_resident(*pp, rs); // (Triforce: a page of a piece with no memory is read in guest memory itself) if (rs.empty()) continue; for (const VkBufferCopy& r : rs) lazy_settle(pp->lo + r.srcOffset, pp->lo + r.srcOffset + r.size, wb_lazy::BY_GPU); // (Triforce) + // ((Triforce, 11-21, MEM-48) after the full barrier above: whatever wrote the mirror before, the pass too, is done) + if (msync_pass_on()) pp->up_cmd = up, pp->up_value = mem_rec_value, pp->up_kinds = msync_pass::kCopies; vkCmdCopyBuffer(up, pp->buffer, pp->mirror, (uint32_t)rs.size(), rs.data()); gp.items += (int)rs.size(); for (const VkBufferCopy& r : rs) gp.bytes += r.size; @@ -5595,6 +6077,7 @@ void memory_pool_created(uint64_t gpu, uint64_t size) { bg_cv.notify_one(); } +bool imports_leaving_overlap(uint64_t lo, uint64_t hi); // (below, with evicted_pools) int memory_guest_imported(uint64_t lo, uint64_t hi) { std::lock_guard lk(pools_mu); auto it = imported.upper_bound(hi); // (pools by lo: those starting below hi, the last ones checked first) @@ -5605,6 +6088,18 @@ int memory_guest_imported(uint64_t lo, uint64_t hi) { } for (const auto& nh : retired_pools) if (nh.mapped().memory && nh.mapped().lo < hi && lo < nh.mapped().hi) return 1; + // ((Triforce, 11-22, after Casual Gamer's fix) and the imports given back and not freed yet (imports_leaving_overlap): out of the + // table, but the driver keeps their pages locked until the GPU is past them. A free there discarded or decommitted pages + // under a live lock, and the device was lost (v0.4.0, a 16 GB laptop: 2 tours of 5 without this, 0 of 53 with it). The + // game frees far more since its mapped pages count as used (host/hos_mem.c query_used). RX_VK_IMPORT_LEAVING=0: not, as before) + static const bool leaving = !(getenv("RX_VK_IMPORT_LEAVING") && atoi(getenv("RX_VK_IMPORT_LEAVING")) == 0); + if (leaving && imports_leaving_overlap(lo, hi)) { + static int told = 0; // (pools_mu held) + if (++told <= 8 || told % 64 == 0) + logf("vk: the game's pages %#llx..%#llx freed under an import still being given back: zeroed, not discarded (%d so far)", + (unsigned long long)lo, (unsigned long long)hi, told); + return 1; + } // ((Triforce) and the imports the background thread holds or is making, not in the table yet: bg_held. RX_VK_BG_IMPORT_HOLD=0: // not, as before: the lab's A/B of whether this was the device lost of test11 and test12, with RX_VK_IMPORT_AHEAD=1) static const bool hold = !(getenv("RX_VK_BG_IMPORT_HOLD") && atoi(getenv("RX_VK_BG_IMPORT_HOLD")) == 0); @@ -5741,7 +6236,7 @@ void late_deferred_bound(uint64_t guest, uint64_t size) { } const uint64_t done0 = vk_timeline_completed(); const auto w0 = std::chrono::steady_clock::now(); - if (v > done0) vk_timeline_wait(v); + if (v > done0) vk_timeline_wait_from(v, 2); // ((Triforce, 11-22, SYNC-122) wb_rec_wait.h kBind: the held part by caller) st_late_jit++; { // (diagnostics, RX_VK_LOG_LATE: the bindings that reached a page put off, and what they waited) static const bool log_late = getenv("RX_VK_LOG_LATE") != nullptr; @@ -5764,6 +6259,7 @@ void late_deferred_bound(uint64_t guest, uint64_t size) { if (!pieces_resident(&p, a - p.lo, b - a)) continue; // (Triforce: no memory under it: the GPU reads guest memory there) const VkBufferCopy r{a - p.lo, a - p.lo, b - a}; lazy_settle(a, b, wb_lazy::BY_GPU); // (Triforce, lazy write-back: the copy reads guest memory) + mirror_up_copy(p); // ((Triforce, 11-21, MEM-48) in order with the syncs' pass) vkCmdCopyBuffer(upload_command_buffer(), p.buffer, p.mirror, 1, &r); st_sync_bytes += (int64_t)r.size, st_sync_copies++; } @@ -5872,7 +6368,10 @@ BufferRef gpu_buffer_bind(uint64_t gpu, uint64_t size, bool written) { return remember(in_guest({p->buffer, off, size, 0})); if (p->mstate == Pool::MIRROR_ON && p->synced_epoch == mirror_epoch && p->sync_seg <= texture_segment()) { // (Triforce: a mirror in pieces: only where it has memory; a piece left in guest memory for good: the import; else below) - if (pieces_resident(p, off, size)) return remember({p->mirror, off, size, 0}); + if (pieces_resident(p, off, size)) { + pieces_touch(p, off, size); // ((Triforce, 11-22, MEM-26's count) nothing unless RX_VK_MIRROR_PIECE_IDLE is set + return remember({p->mirror, off, size, 0}); + } if (pieces_direct(p, off, size)) return remember(in_guest({p->buffer, off, size, 0})); } } @@ -5988,14 +6487,40 @@ BufferRef gpu_buffer_bind(uint64_t gpu, uint64_t size, bool written) { if (kb) logf("vk: write-backs only of written bindings up to %llu KB (RX_VK_WB_MAX_KB)", (unsigned long long)kb); return kb << 10; }(); - if (!wb_max || size <= wb_max) - writebacks.push_back({p, off, std::min(off + size, p->hi - p->lo), tl_event_write}); + // ((Triforce, 11-22) RX_VK_WB_EXCL_CLIP, on by default with RX_VK_CAP_ONLY_GPU=1; =0: as before) not the parts in memory + // the capture leaves out: the CPU's writes there never reach the mirror, and its stale bytes copied back went over the + // game's heap (a bare table entry's 64-byte head over a 12-byte block's header 0x18 past it: the 0.6.0 crashes at area + // changes in 0x803b44e0 / 0x80a30a00 / 0x80a31030; wb_excl_clip.h) + static const bool excl_clip = [] { + const bool v = wb_excl_clip::on(getenv("RX_VK_WB_EXCL_CLIP"), cap_only_gpu()); + logf("vk: write-backs %s (RX_VK_WB_EXCL_CLIP)", v ? "left out of the memory the capture leaves out" : "of every written binding whole"); + return v; + }(); + const uint64_t wb_end = std::min(off + size, p->hi - p->lo); + if (!wb_max || size <= wb_max) { + if (excl_clip && cap_excl_hit(p->lo + off, p->lo + wb_end)) { + static thread_local std::vector> parts; + parts.clear(); + const uint64_t dropped = cap_excl_kept(p->lo + off, p->lo + wb_end, parts); + for (const auto& [a, b] : parts) writebacks.push_back({p, a - p->lo, b - p->lo, tl_event_write}); + static std::atomic st_clipped{0}, st_clipped_bytes{0}; + const int64_t k = st_clipped.fetch_add(1, std::memory_order_relaxed); + st_clipped_bytes.fetch_add((int64_t)dropped, std::memory_order_relaxed); + if (k < 20 || (k + 1) % 100000 == 0) + logf("vk: write-back of %#llx +%#llx: %llu bytes in memory the capture leaves out not copied back (%lld so far, " + "%lld KB; RX_VK_WB_EXCL_CLIP=0: copied)", (unsigned long long)(p->lo + off), (unsigned long long)(wb_end - off), + (unsigned long long)dropped, (long long)(k + 1), (long long)(st_clipped_bytes.load(std::memory_order_relaxed) >> 10)); + } else { + writebacks.push_back({p, off, wb_end, tl_event_write}); + } + } rb_hazard_locked(p->lo + off, p->lo + std::min(off + size, p->hi - p->lo)); // (memory_rb_refresh) const uint64_t np = (p->hi - p->lo + 4095) >> 12; if (p->gpu_bits.size() < (np + 63) / 64) p->gpu_bits.resize((np + 63) / 64); const uint64_t a = off >> 12, b = std::min((off + std::max(size, 1) - 1) >> 12, np - 1); for (uint64_t q = a; q <= b; q++) p->gpu_bits[q >> 6] |= 1ull << (q & 63); } + pieces_touch(p, off, size); // ((Triforce, 11-22, MEM-26's count) nothing unless RX_VK_MIRROR_PIECE_IDLE is set return remember({p->mirror, off, size, 0}); } @@ -6012,7 +6537,7 @@ uint64_t st_mirror_bytes_freed = 0; // last_value counts the draws and dispatches that bound it, not the shaders that reach it through the guest page table: // a culling or fog compute pass already running read a mirror destroyed under it (device lost in 53caac5f and ee7764a6, // issues 132 and 195, AMD after a scene change). The table goes back to guest memory at once, as before -struct DeadMirror { VkBuffer buffer; VmaAllocation alloc; uint64_t value; std::vector pieces; }; // (pieces: a sparse one's memory) +struct DeadMirror { VkBuffer buffer; VmaAllocation alloc; uint64_t value; std::vector pieces; uint64_t sparse_bytes = 0; }; // (pieces: a sparse one's memory; (Triforce, 11-21, MEM-27) sparse_bytes: its address space) std::deque dead_mirrors; // A retired pool's import, given back the same way. Each import is a userptr buffer object to amdgpu, and RADV lists every // buffer object in every submission (buffer device addresses): the kernel checks every page of every userptr object at each @@ -6024,6 +6549,13 @@ std::deque dead_mirrors; struct DeadImport { VkBuffer buffer; VkDeviceMemory memory; uint64_t value; uint64_t bytes; std::vector pieces; uint64_t lo, hi; }; std::deque dead_imports; uint64_t st_retired_imports_freed = 0, st_retired_import_bytes_freed = 0; +uint64_t dead_import_bytes = 0; // ((Triforce, 11-22, MEM-21) the bytes in dead_imports: given back, not freed yet (pools_mu)) +// ((Triforce, 11-21, the performance audit's MEM-27, its first step) what the mirrors' sparse buffers cost to make and to +// destroy, and how many creates a freed one could have served: mirror_spare.h. pools_mu held at both places) +mirror_spare::Counts sparse_ct; +double sparse_ms_since(std::chrono::steady_clock::time_point t0) { + return std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); +} void reap_mirrors() { // (pools_mu held) while (!dead_mirrors.empty() && dead_mirrors.front().value <= mem_completed) { if (getenv("RX_VK_LOG_ADDR")) { @@ -6041,7 +6573,9 @@ void reap_mirrors() { // (pools_mu held) for (const PieceBind& b : piece_binds) pending = pending || b.buffer == dead_mirrors.front().buffer; if (pending) break; } + const auto destroy_t0 = std::chrono::steady_clock::now(); vkDestroyBuffer(dev.device, dead_mirrors.front().buffer, nullptr); + mirror_spare::destroyed(sparse_ct, dead_mirrors.front().sparse_bytes, sparse_ms_since(destroy_t0)); // ((Triforce, 11-21, MEM-27)) for (VmaAllocation a : dead_mirrors.front().pieces) vma_piece_free(a); } dead_mirrors.pop_front(); @@ -6051,38 +6585,133 @@ void reap_mirrors() { // (pools_mu held) vkFreeMemory(dev.device, dead_imports.front().memory, nullptr); for (VkDeviceMemory m : dead_imports.front().pieces) vkFreeMemory(dev.device, m, nullptr); st_retired_imports_freed++, st_retired_import_bytes_freed += dead_imports.front().bytes; + dead_import_bytes -= std::min(dead_import_bytes, dead_imports.front().bytes); // ((Triforce, 11-22, MEM-21)) dead_imports.pop_front(); } } +bool system_heap_usage(uint64_t* used, uint64_t* budget); // (vk_vma.cpp) +/// ((Triforce, 11-22) whether RX_VK_KEEP_RETIRED is set at all, read once: the system heap's rule in +/// retired_imports_give_back is asked only where the mode keeps the imports by default (unset) +bool retired_keep_set() { + static const bool v = getenv("RX_VK_KEEP_RETIRED") != nullptr; + return v; +} +/// ((Triforce, 11-22, the performance audit's MEM-21, after Source A's) RX_VK_KEEP_RETIRED, read once: unset and the values it +/// took before decide as before (retired_imports_give_back says why); =wait: given back after a wait, on any driver +/// (retired_give_back.h) +retired_give_back::Mode retired_mode() { + static const retired_give_back::Mode m = [] { +#ifdef _WIN32 + const bool windows = true; +#else + const bool windows = false; +#endif + const retired_give_back::Mode v = retired_give_back::mode(getenv("RX_VK_KEEP_RETIRED"), windows, dev.props.vendorID == 0x10DE); + if (v == retired_give_back::Mode::Wait) + logf("vk: RX_VK_KEEP_RETIRED=wait: a retired pool's import is given back %lld s after it retired, %lld s while the system " + "heap is past 85%% of its budget", (long long)retired_give_back::secs(getenv("RX_VK_RETIRE_SECS"), 30), + (long long)retired_give_back::secs(getenv("RX_VK_RETIRE_MIN_SECS"), 2)); + return v; + }(); + return m; +} void retired_imports_give_back() { // (pools_mu held) // (hookshot) Kept by default on Windows: giving them back is for the Linux amdgpu submit above, and on Windows an Intel Arc // B390 lost the device entering Kokiri every time with it and 1 run in 5 without (issue 17, fuxau); the AMD device losses // on the Ally X (309) and RX 6600 (571) came right after retired pools went back. RX_VK_KEEP_RETIRED=0 or 1 chooses -#ifdef _WIN32 // (not on NVIDIA: kept there, ~20 areas of play left 6.8 GB of retired pools imported on an RTX 3070 (8 GB), and the next area // lost the device, twice; given back, the same tour finished every area (issue 1057). The losses above were Intel's and AMD's) - static const bool keep = getenv("RX_VK_KEEP_RETIRED") ? getenv("RX_VK_KEEP_RETIRED")[0] != '0' : dev.props.vendorID != 0x10DE; -#else - static const bool keep = getenv("RX_VK_KEEP_RETIRED") && getenv("RX_VK_KEEP_RETIRED")[0] != '0'; -#endif + namespace rg = retired_give_back; + const rg::Mode mode = retired_mode(); + // ((Triforce, 11-22, MEM-21) RX_VK_RETIRED_REFUSED: the imports of pools whose mirror was refused too (on with the wait) + static const bool refused = [&] { + const bool v = rg::refused_too(getenv("RX_VK_RETIRED_REFUSED"), mode); + if (v) logf("vk: RX_VK_RETIRED_REFUSED: retired pools whose mirror was refused give their imports back too%s", + mode == rg::Mode::Keep ? " (none is given back: RX_VK_KEEP_RETIRED keeps them)" : ""); + return v; + }(); + static const int64_t wait_s = rg::secs(getenv("RX_VK_RETIRE_SECS"), 30), min_s = rg::secs(getenv("RX_VK_RETIRE_MIN_SECS"), 2); + int64_t now_ms = 0; + int heap_short = -1; // (the wait only: the clock and the heap read once a call, and only when a pool waits on them) + const bool keep = mode == rg::Mode::Keep; // ((Triforce, 11-22, MEM-21) the old reading of RX_VK_KEEP_RETIRED: retired_give_back.h mode() + // ((Triforce, 11-22, after Casual Gamer's fix) Kept for good only while the system heap has room. On an integrated GPU (a + // Radeon 780M, 16 GB laptop) the retired imports filled the heap the driver counts every import in, new imports were refused + // (16 in a tour) and draws left out for want of their pools (24.9%). Past the import rule's high mark of the heap's budget + // (RX_VK_IMPORT_HIGH, 85) or for a minute after an import was refused for want of room (import_room_refused_ms), they are + // given back where they are kept otherwise: the draws skipped 0, refused imports 1. Asked every 16 submissions, as that rule. + // RX_VK_KEEP_RETIRED=1: kept always, as before. (MEM-21) Asked only where the mode keeps them: RX_VK_KEEP_RETIRED unset on + // Windows with AMD's or Intel's driver; with =wait or at once, retired_give_back.h decides below) + bool give_back = !keep; + if (keep && !retired_keep_set()) { + static const int high = + getenv("RX_VK_IMPORT_HIGH") && atoi(getenv("RX_VK_IMPORT_HIGH")) > 0 ? atoi(getenv("RX_VK_IMPORT_HIGH")) : 85; + static uint64_t calls = 0; + static bool pressed = false; + static uint64_t used = 0, budget = 0; + if (calls++ % 16 == 0) { + const bool was = pressed; + if (!system_heap_usage(&used, &budget)) used = budget = 0; + const int64_t refused = import_room_refused_ms.load(std::memory_order_relaxed), now = import_now_ms(); + pressed = mem_parts::retired_pressed(used, budget, high, now, refused); + static int told; + if (pressed && !was && told++ < 8) + logf("vk: retired imports given back after all: the system heap at %llu of %llu MB (past %d%%)%s " + "(RX_VK_KEEP_RETIRED=1: kept)", (unsigned long long)(used >> 20), (unsigned long long)(budget >> 20), high, + refused && now - refused < mem_parts::kRefusedHoldMs + ? (", an import refused for want of room " + std::to_string((now - refused) / 1000) + " s ago").c_str() : ""); + } + give_back = pressed; + } + // ((Triforce, 11-22, the performance audit's DESC-06, after Source A's) This runs at every part of a submission, and the + // sums are only for a line logged every 1200 epochs: the live pools are summed only when that line is due, and with the + // retired imports kept the retired pools are walked only then too (kept, the walk gives nothing back; (Triforce) kept is + // !give_back, the system heap's rule above). Where they are + // given back the walk starts past the retired pools that have no import left: the list only grows at its end + // (retire_pool) and a pool's memory is set only on a new Pool, never on a retired one, so those stay as they are and + // were skipped one by one at every part, in a list that grows all session. Same imports given back, same line) + static uint64_t logged_epoch; + const bool due = mirror_epoch - logged_epoch >= 1200; uint64_t live_bytes = 0, retired_bytes = 0; size_t retired_alive = 0; - for (auto& [lo, p] : imported) live_bytes += p.hi - p.lo; - for (auto& nh : retired_pools) { - Pool& p = nh.mapped(); - if (!p.memory) continue; - if (!keep && p.mstate == Pool::MIRROR_NONE && !p.inherit_pending && p.retired_value <= mem_completed && - p.last_value <= mem_completed) { - dead_imports.push_back({p.buffer, p.memory, std::max(mem_rec_value, p.last_value), p.hi - p.lo, std::move(p.pieces), p.lo, p.hi}); - p.pieces.clear(); - p.buffer = VK_NULL_HANDLE, p.memory = VK_NULL_HANDLE, p.address = 0; - pools_gen++; - continue; + if (due) + for (auto& [lo, p] : imported) live_bytes += p.hi - p.lo; + static size_t gone = 0; // (the retired pools before this one have no import) + if (gone > retired_pools.size()) gone = 0; // (never: the list does not shrink) + if (give_back || due) + for (size_t i = gone; i < retired_pools.size(); i++) { + Pool& p = retired_pools[i].mapped(); + if (!p.memory) { + if (i == gone) gone++; + continue; + } + const bool gpu_past = p.retired_value <= mem_completed && p.last_value <= mem_completed; + bool is_due = false; + if (mode == rg::Mode::Wait && gpu_past && !p.inherit_pending && + (p.mstate == Pool::MIRROR_NONE || (refused && p.mstate == Pool::MIRROR_REFUSED))) { + if (!now_ms) + now_ms = std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); + is_due = rg::due(now_ms, p.retired_ms, wait_s, min_s, false); + if (!is_due && rg::due(now_ms, p.retired_ms, wait_s, min_s, true)) { + if (heap_short < 0) { + uint64_t used = 0, budget = 0; + heap_short = system_heap_usage(&used, &budget) && rg::pressure(rg::settled(used, dead_import_bytes), budget); + } + is_due = heap_short > 0; + } + } + // ((Triforce) the system heap's rule above, where the mode keeps them: given back as at once, or MEM-21's rule) + const bool pressed_back = keep && give_back && p.mstate == Pool::MIRROR_NONE && !p.inherit_pending && gpu_past; + if (pressed_back || rg::want(mode, refused, p.mstate == Pool::MIRROR_NONE, p.mstate == Pool::MIRROR_REFUSED, p.inherit_pending, gpu_past, is_due)) { + dead_import_bytes += p.hi - p.lo; // ((Triforce, 11-22, MEM-21) taken off in reap_mirrors when it is freed) + dead_imports.push_back({p.buffer, p.memory, std::max(mem_rec_value, p.last_value), p.hi - p.lo, std::move(p.pieces), p.lo, p.hi}); + p.pieces.clear(); + p.buffer = VK_NULL_HANDLE, p.memory = VK_NULL_HANDLE, p.address = 0; + pools_gen++; + continue; + } + retired_alive++, retired_bytes += p.hi - p.lo; } - retired_alive++, retired_bytes += p.hi - p.lo; - } - static uint64_t logged_epoch; - if (mirror_epoch - logged_epoch >= 1200) { + if (due) { logged_epoch = mirror_epoch; mirror_use_report(); char touched[160] = ""; // ((Triforce) import_touch.h: Windows, the background thread's imports) @@ -6104,7 +6733,8 @@ void mirror_free(Pool& p) { if (live != imported.end() && &live->second == &p) guest_pages_set(p.lo, p.hi, p.address); } const uint64_t held = p.mpieces ? p.res_bytes : p.hi - p.lo; // (Triforce: in pieces, what its pieces hold) - dead_mirrors.push_back({p.mirror, p.mirror_alloc, std::max(mem_rec_value, p.last_value), std::move(p.res_allocs)}); + dead_mirrors.push_back({p.mirror, p.mirror_alloc, std::max(mem_rec_value, p.last_value), std::move(p.res_allocs), + p.mpieces ? p.hi - p.lo : 0}); shadow_free(p); p.mirror = VK_NULL_HANDLE, p.mirror_alloc = nullptr; if (p.mpieces) { // (the bits cleared in place: gpu_buffer_bind reads them without the lock) @@ -6419,6 +7049,16 @@ void demote_trials() { // were long unbound.) ---- bool system_heap_usage(uint64_t* used, uint64_t* budget); // (vk_vma.cpp) std::deque::node_type> evicted_pools; +/// (Triforce) Whether an import given back and not freed yet holds any of [lo, hi): an evicted pool (imports_give_back) or a +/// retired pool's import (retired_imports_give_back: dead_imports), freed once the GPU is past them; until then the driver +/// keeps their pages locked, as guest_release_unimported counts them. (pools_mu held) +bool imports_leaving_overlap(uint64_t lo, uint64_t hi) { + for (const auto& nh : evicted_pools) + if (nh.mapped().memory && nh.mapped().lo < hi && lo < nh.mapped().hi) return true; + for (const DeadImport& d : dead_imports) + if (d.lo < hi && lo < d.hi) return true; + return false; +} int64_t st_imports_evicted = 0; uint64_t st_import_bytes_evicted = 0; constexpr uint64_t kImportIdle = 600; @@ -6441,7 +7081,7 @@ void imports_give_back() { // (pools_mu held) // time, and on one the staging arenas refused ("no 64 MB arena could be made; the upload waits", 8,586 textures left for a // later frame: the flat white and blue ones). // And a second trigger beside the heap's budget: the PC's RAM itself running short (under 1.5 GB available, - // GlobalMemoryStatusEx; RX_VK_IMPORT_RAM=0: not), which then gives back every import unbound for kImportIdleRam submissions + // GlobalMemoryStatusEx; RX_VK_IMPORT_RAM=1 only), which then gives back every import unbound for kImportIdleRam submissions // (~10 s of play), up to 1 GB a pass. With RAM to spare nothing changes: an import given back costs 5-45 ms when its pool is // bound again. RX_VK_IMPORT_HIGH=0: off altogether. (lab: RX_VK_TEST_AVAIL_MB= plays that much RAM available) static const char* env_high = getenv("RX_VK_IMPORT_HIGH"); @@ -6455,14 +7095,33 @@ void imports_give_back() { // (pools_mu held) if (high <= 0 || ++calls % 16) return; // (the driver asked every 16 submissions) uint64_t used = 0, budget = 0; if (!system_heap_usage(&used, &budget)) return; + // ((Triforce, 11-22, the performance audit's MEM-21, after Source A's) with RX_VK_KEEP_RETIRED=wait, the imports given back and + // not freed yet are not counted: retired ones going back in a burst do not make this give back live ones too + if (retired_mode() == retired_give_back::Mode::Wait) used = retired_give_back::settled(used, dead_import_bytes); + // ((Triforce, 11-22, the performance audit's MEM-22, after Source A's) With Mesa's RADV and a budget of 12 GB or more the + // heap is measured against 4,500 MB, not the whole budget (import_budget.h): live imports piled up there for the session, + // and amdgpu pays for each one at every submission. RX_VK_IMPORT_CAP_MB=: that cap on any driver and budget; =0: none. + // On every other driver, at a smaller budget, or with RX_VK_IMPORT_HIGH or _LOW set, the rule is the one it was + static import_budget::Rule live; static const bool small = [&] { // (on by default where the budget is small: 16 GB of RAM and less) - const bool on = env_high || budget < (12ull << 30); - logf("vk: imports given back past %d%% of a %llu MB system heap budget: %s", high, (unsigned long long)(budget >> 20), - on ? "on" : "off (RX_VK_IMPORT_HIGH=85 turns it on)"); + static const char* env_cap = getenv("RX_VK_IMPORT_CAP_MB"); + live = import_budget::rule(budget, env_high != nullptr, getenv("RX_VK_IMPORT_LOW") != nullptr, env_cap ? atoll(env_cap) : -1, + dev.driver_id == VK_DRIVER_ID_MESA_RADV); + const bool on = live.on; + if (import_budget::measured(live, budget) != budget) + logf("vk: imports given back past %d%% of %llu MB, the live imports' cap on a %llu MB system heap budget: on " + "(RX_VK_IMPORT_CAP_MB=0: no cap)", high, (unsigned long long)(live.cap >> 20), (unsigned long long)(budget >> 20)); + else + logf("vk: imports given back past %d%% of a %llu MB system heap budget: %s", high, (unsigned long long)(budget >> 20), + on ? "on" : "off (RX_VK_IMPORT_HIGH=85 turns it on)"); return on; }(); + const uint64_t measure = import_budget::measured(live, budget); // (the budget itself wherever there is no cap) // (the RAM's trigger: asked every 256 submissions, ~0.7 s) - static const bool ram_rule = !(getenv("RX_VK_IMPORT_RAM") && !atoi(getenv("RX_VK_IMPORT_RAM"))); + // ((Triforce, 11-22, after Casual Gamer's fix) Off unless RX_VK_IMPORT_RAM=1. A 16 GB PC has under 1.5 GB available nearly + // all the time, so it fired every ~20 s and gave back every unbound import, which came straight back at its next binding + // (690 to 1216 imports in a 13-minute tour). The system heap's trigger above stays) + static const bool ram_rule = getenv("RX_VK_IMPORT_RAM") && atoi(getenv("RX_VK_IMPORT_RAM")) != 0; static const int64_t test_avail = getenv("RX_VK_TEST_AVAIL_MB") ? atoll(getenv("RX_VK_TEST_AVAIL_MB")) : -1; static bool ram_short = false; static uint64_t ram_avail = 0, ram_total = 0; @@ -6496,10 +7155,17 @@ void imports_give_back() { // (pools_mu held) ram_short = ram_avail < (1536ull << 20); } } - const bool by_heap = small && used > budget / 100 * (uint64_t)high; - if (!by_heap && !ram_short) return; - const uint64_t target = budget / 100 * (uint64_t)low; - const uint64_t want = by_heap ? (used > target ? used - target : 0) : (1ull << 30); + const bool by_heap = small && used > measure / 100 * (uint64_t)high; + // ((Triforce, 11-22, the performance audit's MEM-141, after Source A's) the swap hold (linux_pressure.h; off unless + // RX_VK_SWAP_RESERVE_MB is set, never on Windows) is a RAM trigger too, with a limit of its own: in all of one episode the + // idle imports given back are at most the bytes the PC was short at its worst, not up to 1 GB a pass, which is what made + // the thrash described below. The idle rule (kImportIdleRam) and the fresh-import rule stay + const bool by_hold = ram_rule && !by_heap && memory_swap_hold(); + if (!by_heap && !ram_short && !by_hold) return; + const uint64_t target = measure / 100 * (uint64_t)low; + // ((Triforce, 11-22, MEM-22) a pass against the cap gives back 1 GB at most; without a cap, down to the target as before) + const uint64_t want = by_heap ? import_budget::pass_want(live, budget, used, target) : by_hold ? memory_swap_episode_want() : (1ull << 30); + if (by_hold && !want) return; // (this episode has given back what it may) constexpr uint64_t kImportIdleRam = 3600; const uint64_t idle_for = by_heap ? kImportIdle : kImportIdleRam; std::vector::iterator> idle; @@ -6565,6 +7231,7 @@ void imports_give_back() { // (pools_mu held) std::fill(std::begin(pool_cache), std::end(pool_cache), nullptr); if (mirror_on()) publish_cap_ranges(); st_imports_evicted += n, st_import_bytes_evicted += freed; + if (by_hold) memory_swap_gave(freed); // ((Triforce, 11-22, MEM-141) counted against the episode's limit static int logged; if (logged++ < 40 || logged % 64 == 0) logf("vk: system heap %llu of %llu MB%s: %d idle imports given back (%llu MB); %lld (%llu MB) in all", @@ -6719,6 +7386,39 @@ void memory_begin_submission() { "piece runs bound in %lld flushes (%.1f ms in all); %lld bindings given guest memory instead", n_pieces, (unsigned long long)(held >> 20), (unsigned long long)(spans >> 20), n_whole, (unsigned long long)(whole >> 20), (long long)st_piece_binds, (long long)st_piece_flushes, st_piece_bind_ms, (long long)st_piece_direct); + { // ((Triforce, 11-22, the performance audit's MEM-26, its first step) what a release of pieces would find: piece_idle.h + static thread_local piece_idle::Ranges game; + game.clear(); + for (const GuestPool& g : guest_pools) game.push_back({g.lo, g.hi}); + piece_idle::merge(game); + piece_idle::Count idle; + const uint64_t idle_for = piece_idle_for(); + for (auto& [lo, p] : imported) { + if (p.mstate != Pool::MIRROR_ON || !p.mpieces) continue; + const Pool* q = &p; + piece_idle::count(p.lo, p.hi - p.lo, p.piece_shift, [q](uint64_t i) { return piece_resident(q, i); }, + [q](uint64_t i) { // (the submission of its last binding, from the 32 bits kept) + const uint32_t last = i < q->piece_last.size() ? __atomic_load_n(&q->piece_last[i], __ATOMIC_RELAXED) : (uint32_t)mirror_epoch; + return mirror_epoch - (uint32_t)((uint32_t)mirror_epoch - last); + }, + game, mirror_epoch, idle_for, idle); + } + char more[160] = ""; + if (idle_for) + snprintf(more, sizeof more, ", %llu MB of them not reached by a binding for %llu submissions", (unsigned long long)(idle.idle >> 20), + (unsigned long long)idle_for); + logf("vk: mirrors in pieces: %llu MB are pieces no game pool covers now%s (counted by piece, nothing is released; " + "RX_VK_MIRROR_PIECE_IDLE= counts the idle ones)", (unsigned long long)(idle.uncovered >> 20), more); + } + // ((Triforce, 11-21, the performance audit's MEM-27, its first step) the sparse buffers themselves: mirror_spare.h) + if (sparse_ct.creates || sparse_ct.destroys) + logf("vk: mirrors' sparse buffers so far: %llu made (%llu MB of address space, %.1f ms, the longest %.2f ms), %llu " + "destroyed (%llu MB, %.1f ms, the longest %.2f ms); %llu calls over %.0f ms; %llu of the creates had a freed " + "buffer of exactly their size before them", (unsigned long long)sparse_ct.creates, + (unsigned long long)(sparse_ct.create_bytes >> 20), sparse_ct.create_ms, sparse_ct.longest_create_ms, + (unsigned long long)sparse_ct.destroys, (unsigned long long)(sparse_ct.destroy_bytes >> 20), sparse_ct.destroy_ms, + sparse_ct.longest_destroy_ms, (unsigned long long)sparse_ct.over, mirror_spare::kLongMs, + (unsigned long long)sparse_ct.servable); } } if (mirror_wanted.empty()) return; @@ -6792,7 +7492,10 @@ bool mirror_make(Pool& pool) { bi.size = n; bi.usage = kUsage; VkBuffer sb = VK_NULL_HANDLE; - if (vkCreateBuffer(dev.device, &bi, nullptr, &sb) == VK_SUCCESS) { + const auto create_t0 = std::chrono::steady_clock::now(); + const VkResult sparse_made = vkCreateBuffer(dev.device, &bi, nullptr, &sb); + if (sparse_made == VK_SUCCESS) mirror_spare::created(sparse_ct, n, sparse_ms_since(create_t0)); // ((Triforce, 11-21, MEM-27)) + if (sparse_made == VK_SUCCESS) { VkMemoryRequirements req{}; vkGetBufferMemoryRequirements(dev.device, sb, &req); const uint64_t piece = std::max(piece_cfg, std::bit_ceil(std::max(req.alignment, 4096))); @@ -6803,6 +7506,8 @@ bool mirror_make(Pool& pool) { p->piece_align = req.alignment, p->piece_types = req.memoryTypeBits, p->mirror_size = req.size; const size_t words = (size_t)((((n + piece - 1) >> p->piece_shift) + 63) / 64); if (p->res_bits.size() != words) p->res_bits.assign(words, 0); // (once: read without the lock) + // ((Triforce, 11-22, MEM-26's count) and the pieces' last bindings, once too, when they are counted + if (piece_idle_for() && p->piece_last.size() != words * 64) p->piece_last.assign(words * 64, (uint32_t)mirror_epoch); if (p->direct_bits.size() != words) p->direct_bits.assign(words, 0); // (kept from its mirror before, if any) p->res_allocs.clear(), p->res_bytes = 0; VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO}; @@ -6868,7 +7573,7 @@ bool mirror_make(Pool& pool) { const VkBufferCopy all{0, 0, n}; GprofSpan gp(upload_command_buffer(), "mmake copy (a whole pool -> its new mirror)"); gp.bytes = n; - vkCmdCopyBuffer(upload_command_buffer(), p->buffer, p->mirror, 1, &all); // (all of it, at this generation) + mirror_up_copy(*p), vkCmdCopyBuffer(upload_command_buffer(), p->buffer, p->mirror, 1, &all); // (all of it, at this generation) } { // (EvilOOT) then the mirrors of the pools it took in, over their part: what the GPU wrote there and only there // (retire_pool). The captures' pages go over it after, as before: the CPU's newer writes win @@ -6914,7 +7619,7 @@ bool mirror_make(Pool& pool) { clip_resident(r, parts, true); clip_resident(*p, parts); if (!parts.empty()) - vkCmdCopyBuffer(upload_command_buffer(), r.mirror, p->mirror, (uint32_t)parts.size(), parts.data()); + mirror_up_copy(*p), vkCmdCopyBuffer(upload_command_buffer(), r.mirror, p->mirror, (uint32_t)parts.size(), parts.data()); if (from_guest) { static int told; if (told++ < 40) @@ -6980,6 +7685,7 @@ std::vector shadow_written_collected_list; // (the same, of the coll std::vector shadow_written_by_replay; // (the same, of the replay's own CPU writes: guest_cpu_wrote) bool shadow_written_all = false; // (more than those lists hold: every page unknown at the next write-back) int64_t st_wbd_known_game = 0, st_wbd_known_collected = 0, st_wbd_known_replay = 0; // (pages the shadows knew when reported) +int64_t st_wbd_forget_ns = 0, st_wbd_forget_calls = 0; // ((Triforce, 11-21, MEM-61) shadows_forget_written's time and calls: pools_mu) struct DeadShadow { VkBuffer buffer; VmaAllocation alloc; @@ -7105,7 +7811,14 @@ void shadows_forget_written() { std::vector& pages = which == 0 ? game : which == 1 ? collected : replay; int64_t& known = which == 0 ? st_wbd_known_game : which == 1 ? st_wbd_known_collected : st_wbd_known_replay; if (pages.empty()) continue; - std::sort(pages.begin(), pages.end()); + // ((Triforce, 11-21, the performance audit's MEM-61, after Source A's) not sorted first. The sort was only so that the + // pool found for one page serves the next; the bits cleared and the three counts do not depend on the order (a page + // counts under the first list that has it, and one met again finds its bit clear). The lists hold ascending runs as + // they come (a scan reports a range's pages in order, a replay's write its pages in order), so the pool found still + // serves a run, and thousands of pages a part are no longer sorted three times at each part's end, under pools_mu. + // RX_VK_WB_FORGET_SORT=1: sorted, as before) + static const bool forget_sort = getenv("RX_VK_WB_FORGET_SORT") && atoi(getenv("RX_VK_WB_FORGET_SORT")) != 0; + if (forget_sort) std::sort(pages.begin(), pages.end()); Pool* p = nullptr; for (uint32_t pg : pages) { const uint64_t a = (uint64_t)pg << kPageShift; @@ -7169,7 +7882,7 @@ bool wbd_pipe_ready() { return ok; } void shadow_written_to(std::vector& list, const uint32_t* pages, int64_t n) { - if (n <= 0 || !wb_diff_on()) return; + if (n <= 0 || !wb_diff_on() || wb_notes_off()) return; // ((Triforce, 11-22, MEM-51) no mirrors: no shadow forgets these pages) std::lock_guard lk(shadow_written_mu); if (shadow_written_pages.size() + shadow_written_collected_list.size() + shadow_written_by_replay.size() > (8u << 20)) { shadow_written_all = true; // (no write-back has read them for a long while: all of them, then) @@ -7190,7 +7903,7 @@ int guest_cpu_write(uint64_t gpu, const void* src, uint64_t n) { return ok; } void guest_cpu_wrote(uint64_t gpu, uint64_t n) { - if (!n || !wb_diff_on()) return; + if (!n || !wb_diff_on() || wb_notes_off()) return; // ((Triforce, 11-22, MEM-51) no mirrors: no page is looked up for the shadows) uint32_t pages[64]; int k = 0; for (uint64_t a = gpu & ~4095ull; a < gpu + n; a += 4096) { // (GPU pages are mapped to guest memory page by page) @@ -7253,11 +7966,138 @@ WbdTable* wbd_table_get(uint64_t bytes) { return &t; } +// ---- ((Triforce, 11-21, the performance audit's MEM-48, after Source A's) The mirror syncs' pages from the captures by a +// compute pass (RX_VK_MSYNC_PASS; msync_pass.h has the rules and what keeps it right) ---- +// A capture's buffer is host-cached system memory, and a sync copied its pages into the mirror as copy regions, a page or a +// few each. The write-back diff's pass above is a copy when its shadow is the destination itself: it reads the source, +// compares it with the destination and writes it there where they differ. So a sync's regions go as its chunks, one workgroup +// a chunk, in one dispatch where their copies were. The tables are blocks of host memory of their own (not the diff's ring: +// one table of 128 KB or more a dispatch there, and a serial replay makes about 30 of these dispatches a frame) +namespace { +std::vector msync_chunks; // (pools_mu: a sync's, from msync_pass_add to msync_pass_flush) +struct MsyncCopy { + VkBuffer src, dst; + std::vector rs; +}; +std::vector msync_copies; // (the same as copies: for a sync whose table finds no room) +struct MsyncBlock { + VkBuffer buf = VK_NULL_HANDLE; + VmaAllocation alloc = nullptr; + uint8_t* map = nullptr; + VkDeviceAddress address = 0; +}; +MsyncBlock msync_blocks[msync_pass::kBlocks]; +msync_pass::Ring msync_ring; + +/// (pools_mu held) Room for a table of `bytes`: its block and its offset there (msync_pass.h Ring: each table from a cache +/// atom of its own, a block taken again once the GPU is done with the recording that last read it); null: none +MsyncBlock* msync_block_get(uint64_t bytes, uint64_t& at) { + const uint64_t align = std::max(sizeof(WbdChunk), dev.props.limits.nonCoherentAtomSize); + const int i = msync_ring.get(bytes, align, mem_rec_value, mem_completed, &at); + if (i < 0) return nullptr; + MsyncBlock& b = msync_blocks[i]; + if (!msync_ring.b[i].made) { + void* map = nullptr; + if (!create_buffer(msync_pass::kBlockBytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT, true, + b.buf, b.alloc, &map) || !map) { + if (b.buf) destroy_buffer(b.buf, b.alloc); + b = {}; + return nullptr; + } + VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO}; + da.buffer = b.buf; + b.address = vkGetBufferDeviceAddress(dev.device, &da); + if (!b.address) { + destroy_buffer(b.buf, b.alloc); + b = {}; + return nullptr; + } + b.map = (uint8_t*)map; + msync_ring.b[i].made = true; + } + return &b; +} + +/// (pools_mu held) A capture buffer's runs into p's mirror (offsets: the buffer's, the pool's) as the pass's chunks, for +/// msync_pass_flush; false: none taken and the caller copies them (no pipeline, no address, or a run not in whole words) +bool msync_pass_add(Pool& p, VkBuffer buf, const std::vector& rs) { + if (rs.empty() || !p.mirror || !buf || !wbd_pipe_ready()) return false; + VkBufferDeviceAddressInfo da{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO}; + da.buffer = buf; + const VkDeviceAddress src = vkGetBufferDeviceAddress(dev.device, &da); + da.buffer = p.mirror; + const VkDeviceAddress dst = p.mirror_address ? p.mirror_address : vkGetBufferDeviceAddress(dev.device, &da); + if (!src || !dst) return false; + for (const VkBufferCopy& r : rs) + if (!msync_pass::words(src + r.srcOffset, dst + r.dstOffset, r.size)) return false; + for (const VkBufferCopy& r : rs) // (source, shadow, destination: the shadow is the destination) + msync_pass::run_chunks(src + r.srcOffset, dst + r.dstOffset, r.size, + [](uint64_t at, uint64_t size, auto&& e) { wbd_split(at, size, e); }, + [&](uint64_t s, uint64_t d, uint32_t count, bool vecs) { + msync_chunks.push_back({s, d, d, count, vecs ? 1u : 0u}); + }); + msync_copies.push_back({buf, p.mirror, rs}); + return true; +} + +/// (pools_mu held) The chunks a sync took as one dispatch in the upload command buffer, or as their copies when no table +/// has room; up: the sync's writes, for the barrier a mirror written before in that command buffer needs +void msync_pass_flush(UpWrites& up) { + if (msync_chunks.empty()) { + msync_copies.clear(); + return; + } + VkCommandBuffer cmd = upload_command_buffer(); + const uint64_t bytes = msync_chunks.size() * sizeof(WbdChunk); + uint64_t at = 0; + MsyncBlock* b = msync_block_get(bytes, at); + if (!b) { + up.write(msync_pass::kCopies); + for (const MsyncCopy& c : msync_copies) { + GprofSpan gp(cmd, "msync copies (CPU pages from a capture -> mirror)"); + gp.items = (int)c.rs.size(); + vkCmdCopyBuffer(cmd, c.src, c.dst, (uint32_t)c.rs.size(), c.rs.data()); + for (const VkBufferCopy& r : c.rs) gp.bytes += r.size; + } + st_msync_fallbacks++; + static int logged; + if (logged++ < 8) logf("vk: mirror syncs' pass: no table room for %zu chunks: this sync as copies", msync_chunks.size()); + } else { + memcpy(b->map + at, msync_chunks.data(), bytes); + flush_mapped_range(b->alloc, at, bytes); + const uint32_t n = (uint32_t)msync_chunks.size(); + up.write(msync_pass::kPass); + GprofSpan gp(cmd, "msync pass (CPU pages from a capture -> mirror, one dispatch)"); + gp.items = (int)n; + for (const WbdChunk& c : msync_chunks) gp.bytes += (uint64_t)c.count * (c.vecs ? 16 : 4); + // (a marker of its own, as the write-back diff's: a fault in it would read as whatever was marked before it) + checkpoint_mark(cmd, n, 's'); + vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, wbd_pipe.pipe); + const struct { uint64_t table; uint32_t count, pad; } pc{b->address + at, n, 0}; + vkCmdPushConstants(cmd, wbd_pipe.layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, &pc); + vkCmdDispatch(cmd, std::min(n, 65535u), (n + 65534u) / 65535u, 1); + st_msync_dispatches++, st_msync_chunks += (int64_t)n; + } + msync_chunks.clear(); + msync_copies.clear(); +} + +/// (pools_mu held, every 600 submissions: memory_end_submission) How the pass's dispatches went +void msync_pass_report() { + if (!msync_pass_on()) return; + logf("vk: mirror syncs' pass (600 submits): %lld dispatches, %lld chunks; %lld syncs as copies (no table room); %lld full " + "barriers for a mirror written twice in one upload command buffer (RX_VK_MSYNC_PASS=0: copy regions, as before)", + (long long)st_msync_dispatches, (long long)st_msync_chunks, (long long)st_msync_fallbacks, (long long)st_msync_barriers); + st_msync_dispatches = st_msync_chunks = st_msync_fallbacks = st_msync_barriers = 0; +} +} // namespace + void memory_end_submission(VkCommandBuffer cmd) { std::lock_guard lk(pools_mu); { // (every 600 submissions: how the mirror syncs' dirty pages were read) static int64_t n, l_live, l_cap; if (++n % 600 == 0) { + msync_pass_report(); // ((Triforce, 11-21, MEM-48) how the syncs' pass went) static int64_t l_old; logf("vk: mirror syncs (600 submits): %lld dirty pages from their submits' captures, %lld from earlier ones', " "%lld from guest memory as it was at the replay; %zu captures kept; %lld pages the GPU wrote copied over " @@ -7278,6 +8118,10 @@ void memory_end_submission(VkCommandBuffer cmd) { st_wbw_bytes[2] / 1048576.0, (long long)st_wbw_pools[2], st_wbw_bytes[3] / 1048576.0, (long long)st_wbw_pools[3]); for (int k = 0; k < 4; k++) st_wbw_bytes[k] = st_wbw_pools[k] = 0; st_wbd_runs = st_wbd_bytes = st_wbd_fill_pages = 0; + if (st_wbd_forget_calls) // ((Triforce, 11-21, MEM-61)) + logf("vk: write-back diff (600 submits): forgetting the written pages took %.2f ms in %lld calls " + "(RX_VK_WB_FORGET_SORT=1: their lists sorted first, as before)", st_wbd_forget_ns / 1e6, (long long)st_wbd_forget_calls); + st_wbd_forget_ns = st_wbd_forget_calls = 0; st_wbd_known_game = st_wbd_known_collected = st_wbd_known_replay = 0; lazy_report(); // (Triforce: the lazy write-back's line, with the row on) if (n % 6000 == 0) { // (Triforce: what the 64-bit write generations cost here, and how fast a run uses them) @@ -7387,7 +8231,14 @@ void memory_end_submission(VkCommandBuffer cmd) { std::vector runs; // (the write-back's diff: the runs of pools with a shadow, as chunks of 64 KB at most, for one pass after the loop) const bool diff = wb_diff_on() && wbd_pipe_ready(); - if (diff) shadows_forget_written(), shadows_reap(), shadows_check_room(); + if (diff) { + // ((Triforce, 11-21, MEM-61) what the forgetting costs, for the 600-submit line) + const auto f0 = std::chrono::steady_clock::now(); + shadows_forget_written(); + st_wbd_forget_ns += std::chrono::duration_cast(std::chrono::steady_clock::now() - f0).count(); + st_wbd_forget_calls++; + shadows_reap(), shadows_check_room(); + } static thread_local std::vector chunks; struct ChunkSrc { Pool* p; uint64_t at, len; }; static thread_local std::vector chunk_src; // (for the whole copy, when no table is free) @@ -7814,7 +8665,73 @@ namespace { std::atomic ram_short_looks{0}; // (the looks in a row that found the RAM short) } bool memory_ram_short() { return ram_short_looks.load(std::memory_order_relaxed) >= 2; } + +// ---- (Triforce, 11-22, the performance audit's MEM-141, after Source A's) The swap-aware pressure hold (linux_pressure.h has +// the rule and the numbers). Off unless RX_VK_SWAP_RESERVE_MB is set; stepped off Windows only, every 30 presents, from +// memory_ram_watch. Its valves ask memory_swap_hold(): the capture spares (cap_buf_free), the idle imports +// (imports_give_back, by memory_swap_episode_want), and in vk_texture.cpp the staging arenas, malloc_trim and an integrated +// GPU's texture budget (memory_swap_cut). Windows keeps its own rules: the hold is never on there ---- +namespace { +std::mutex swap_hold_mu; +linux_pressure::Hold swap_hold; // (swap_hold_mu held) +std::atomic swap_held{false}; // (swap_hold.held, for the valves: read without the lock) +std::atomic swap_cut{0}; // (swap_hold.cut) +std::atomic swap_valve_bytes[3]; // (this episode's: imports given back, staging arenas freed, capture spares not kept) +/// One look at /proc/meminfo and one step of the hold; its beginning and its end logged, the end with what each valve gave +void swap_hold_step(uint64_t presents) { + static const uint64_t reserve = linux_pressure::reserve_of(getenv("RX_VK_SWAP_RESERVE_MB")); + if (!reserve || presents % linux_pressure::kSampleEvery) return; + static const long long test_avail = getenv("RX_VK_TEST_AVAIL_MB") ? atoll(getenv("RX_VK_TEST_AVAIL_MB")) : -1; + static const long long test_swap = getenv("RX_VK_TEST_SWAP_MB") ? atoll(getenv("RX_VK_TEST_SWAP_MB")) : -1; + char text[8192]; + size_t n = 0; + if (FILE* f = fopen("/proc/meminfo", "r")) { + n = fread(text, 1, sizeof text - 1, f); + fclose(f); + } + text[n] = 0; + const linux_pressure::Sample s = linux_pressure::sample_of(linux_pressure::parse(text), test_avail, test_swap); + char swap[64] = ", swap not known"; + if (s.swap_known) snprintf(swap, sizeof swap, ", %llu MB of swap free", (unsigned long long)(s.swap_free >> 20)); + std::lock_guard lk(swap_hold_mu); + const linux_pressure::Hold::Step step = swap_hold.step(s, reserve); + swap_held.store(swap_hold.held, std::memory_order_relaxed), swap_cut.store(swap_hold.cut, std::memory_order_relaxed); + static std::chrono::steady_clock::time_point since; + if (step == linux_pressure::Hold::Entered) { + for (auto& v : swap_valve_bytes) v.store(0, std::memory_order_relaxed); + since = std::chrono::steady_clock::now(); + logf("vk: swap hold: on (%llu MB available%s: %llu MB short of 1536 MB available or %llu MB with the swap); capture " + "spares at most 32 MB, idle staging arenas freed, malloc_trim sooner, idle imports given back up to the shortfall " + "(RX_VK_SWAP_RESERVE_MB=0: no hold)", (unsigned long long)(s.avail >> 20), swap, (unsigned long long)(swap_hold.now >> 20), + (unsigned long long)(reserve >> 20)); + } else if (step == linux_pressure::Hold::Left) { + logf("vk: swap hold: off after %.0f s (%llu MB available%s; %llu MB short at its worst); given back: imports %llu MB, " + "staging arenas %llu MB, capture spares not kept %llu MB", + std::chrono::duration(std::chrono::steady_clock::now() - since).count(), (unsigned long long)(s.avail >> 20), swap, + (unsigned long long)(swap_hold.deepest >> 20), (unsigned long long)(swap_valve_bytes[0].load() >> 20), + (unsigned long long)(swap_valve_bytes[1].load() >> 20), (unsigned long long)(swap_valve_bytes[2].load() >> 20)); + } +} +} // namespace +bool memory_swap_hold() { return swap_held.load(std::memory_order_relaxed); } +uint64_t memory_swap_cut() { return swap_cut.load(std::memory_order_relaxed); } +void memory_swap_valve(int valve, uint64_t bytes) { + if (valve >= 0 && valve < 3) swap_valve_bytes[valve].fetch_add(bytes, std::memory_order_relaxed); +} +uint64_t memory_swap_episode_want() { + std::lock_guard lk(swap_hold_mu); + return swap_hold.episode_want(); +} +void memory_swap_gave(uint64_t bytes) { + std::lock_guard lk(swap_hold_mu); + swap_hold.gave(bytes); + swap_valve_bytes[0].fetch_add(bytes, std::memory_order_relaxed); +} + void memory_ram_watch(uint64_t presents) { +#ifndef _WIN32 + swap_hold_step(presents); // ((Triforce, 11-22, MEM-141) every 30 presents, and only with RX_VK_SWAP_RESERVE_MB set +#endif if (presents % 60) return; uint64_t avail = 0, total = 0; if (!ram_now(&avail, &total)) return; diff --git a/gpu/vk/vk_menu.cpp b/gpu/vk/vk_menu.cpp index ad66ce0..3944161 100644 --- a/gpu/vk/vk_menu.cpp +++ b/gpu/vk/vk_menu.cpp @@ -43,6 +43,7 @@ #include #include "vk_dlssg.h" +#include "cpu_count.h" // ((Triforce, 11-22, OS-49) the threads to size by: HOS_SIM_CPU #include "vk_pace.h" // (Triforce: who holds the frames, for the rows that hand them to another limiter) #include "vk_bugshot.h" // (Triforce: a bug capture's picture, taken before the overlays are drawn) #include "vk_paths.h" @@ -70,6 +71,7 @@ #include "vk_reflex.h" #include "vk_antilag.h" #include "upscaler_rows.h" +#include "first_start.h" // ((Triforce, 11-22, PRESENT-46) the Render resolution of a first start on a weak card: menu_first_start) #include "vk_present_rect.h" // ((Triforce, options_version 7) where the picture goes in the window: the filter it takes) #include "text_spread.h" // ((Triforce, 11-19) the title entry's shadow and glow as the game's entries' #include "game_grid.h" // ((Triforce, 11-19) where the game's layouts are in the window: the Mods entry on its list) @@ -83,9 +85,11 @@ #include "game_language.h" // (the Game language row: what language.txt chooses and offers) #include "menu_restart.h" #include "mod_cheat_rows.h" // (Triforce: the System tab's rows for the cheats of the mods' cheat files) +#include "achievement_rows.h" // (Triforce: the achievements the host names, their notice and their page) #include "menu_sound_events.h" // (Triforce: the options' own sounds: which one a tick asks for, and their queue to the host) #include "mods_entry_anim.h" // ((Triforce, 11-19) the Mods entry under the title's list: the game's entries' animations) #include "focus_anim.h" // ((Triforce, 11-19) Mods and Quit under the System list, Quit's dialog: the game's buttons' and dialogs') +#include "mods_page_anim.h" // ((Triforce, 11-20) the Mods screen as it opens: its In) #include "vk_mods.h" // (Triforce: the Mods page's engine, GameBanana's list and the installs) #include "vk_salesman.h" // (Triforce: the Happy Mask Salesman beside the cards) #include "imgui_internal.h" // (RegisterUserTexture: the mods' pictures) @@ -392,6 +396,9 @@ const char* tx_dlss_model(int v) { else snprintf(b, sizeof b, "NVIDIA default (RX_VK_DLSS_PRESET)"); return b; } + // ((Triforce, 11-22, the performance audit's PRESENT-36, after Source A's) with RX_VK_DLSS_CNN_VRAM=1 the K row on a card + // under 7 GB draws with NVIDIA's CNN model, preset E, and says so: what DLSS asks for, vk_dlss.cpp wanted_preset) + if (ur::cnn_for_vram(v, dlss_cnn_vram())) return "CNN (E)"; return t[std::clamp(v, 0, ur::kModels - 1)]; } /// The DLSS model: the value chosen; variant 1: set for this run by RX_VK_DLSS_PRESET. ((Triforce, options_version 7) The row @@ -404,8 +411,11 @@ const char* hn_dlss_model(int v, int variant) { "Lets DLSS's runtime pick for each mode: what it picks can change with a runtime update.", }; static const char* const env = "Set for this run by RX_VK_DLSS_PRESET."; - if (variant < 0) variant = dlss_preset_env() ? 1 : 0; + // ((Triforce, 11-22, PRESENT-36) variant 2, the K row's alone: preset E for this card (RX_VK_DLSS_CNN_VRAM=1)) + static const char* const cnn = "Chosen for this card's video memory (under 7 GB): less GPU time than K."; + if (variant < 0) variant = dlss_preset_env() ? 1 : ur::cnn_for_vram(v, dlss_cnn_vram()) ? 2 : 0; if (variant == 1) return env; + if (variant == 2) return v == ur::MODEL_K ? cnn : nullptr; return variant == 0 ? t[std::clamp(v, 0, ur::kModels - 1)] : nullptr; } const char* tx_aniso(int v) { // (Triforce: the Texture filtering row) @@ -964,6 +974,20 @@ bool is_merged(const int* v) { return v == &merged.hdr || v == &merged.framegen // the group's title under it (ui::heading). The selection steps over it, the mouse has nothing on it, and it is hidden while // every row of its group is (row_shown). The Debug tab has its own layout and its one heading, Developer (debug_rows) #define MENU_HEADING(tab, title) {tab, title, nullptr, 0, 0, 0, nullptr, nullptr, nullptr, false, true, -1, true} +// (Triforce) Achievements (host/achievements.c keeps them, on unless HOS_ACHIEVEMENTS=0, and names them for the menu: +// menu_option_set, HOS_OPT_ACH_*; achievement_rows.h has the list). The System tab's Achievements row opens their page, +// kAchPage: every one with its icon, earned or not. The row is not shown until the host has named them (row_shown). One +// earned while the game runs has a notice at the top of the screen for a moment, draw_ach_notice +achievement_rows::List ach_list; +achievement_rows::View ach_view; +constexpr int kAchPage = 10; // (page: see kModsPage) +constexpr char kAchRow[] = "Achievements"; // (the row's name in the table below: is_ach_row tells it by it) +constexpr int64_t kAchNoticeMs = 4000; // (how long a notice stays: host/achievements.c waits a little longer between two) +int ach_columns = 2, ach_capacity = 8; // (the page's cards as last drawn: the keys move by them) +int ach_notice = -1; // (the achievement the notice shows, -1 none) +int64_t ach_notice_at = 0; // (and since when, prof_now) +/// The System tab's Achievements row: confirm opens their page +bool is_ach_row(const Row& r) { return r.tab == TAB_SYSTEM && !r.value && !strcmp(r.name, kAchRow); } Row rows[] = { // ---- Display ---- @@ -1153,6 +1177,9 @@ Row rows[] = { {TAB_SYSTEM, "Bug capture (F8)", nullptr, 0, 0, 0, nullptr, "Writes what a problem report needs into a folder of its own under captures in the game folder: a picture, your " "settings, the end of the log and your save. F8 does the same while you play."}, + // (Triforce: host/achievements.c's list, on a page of its own; shown once the host has named it) + {TAB_SYSTEM, "Achievements", nullptr, 0, 0, 0, nullptr, // (kAchRow) + "Every achievement, and the ones you have earned in this game folder. They are earned as you play; nothing is set here."}, MENU_HEADING(TAB_SYSTEM, "Online"), {TAB_SYSTEM, "Discord status", &opts.discord, 0, 1, 1, tx_onoff, "Shows \"Playing Triforce\" and the time played in your Discord status, while Discord runs on this PC.", nullptr, false, @@ -1408,6 +1435,7 @@ int mods_btn = 0; // (a dialog's button chosen: 0 the u // over the game's list (the game's input is ours then, and the host lets its cursor go); A opens kModsPage constexpr int kModsRowPage = 9; std::atomic mods_list_bits{0}; // (HOS_MODS_LIST_*, as the last report had them) +std::atomic mods_list_alpha{-1}; // ((Triforce, 11-20) HOS_OPT_MODS_ALPHA: the game's own buttons' alpha, thousandths; -1 not told) std::atomic mods_box[5]; // (the last button's box and the pitch, the layout's units: x0 x1 y0 y1 pitch) std::atomic mods_list_ns{0}; // (prof_now at the last report) std::atomic mods_list_frame{0}; // ((Triforce, 11-19) and game_frames then: a slow game frame is not a list gone) @@ -1428,6 +1456,9 @@ int quit_sel = 0; // (its choice: 0 Quit Game, 1 Restar // the System list's own buttons do (focus_anim.h: PaCommonBtnThin_00's Select and UnSelect), and Quit's dialog opens and // closes as the game's SystemWindow_00 (In, Out), its buttons as the same part; it takes input once its In has played) focus_anim::Focus mods_focus[2]; // (Mods, Quit) +// ((Triforce, 11-20) the owner: "Mod menu fade in". The Mods screen came whole on the frame after the press; it fades in as +// the game's lists and windows do (mods_page_anim.h), stepped by menu_tick, and takes input once that has played) +mods_page::Page mods_screen; focus_anim::Focus quit_focus[3]; // (the dialog's buttons) int64_t quit_open_ns = 0; // (prof_now when the dialog opened: its In; 0 shut) int64_t quit_shut_ns = 0; // (when B or Cancel closed it: its Out; 0 none playing) @@ -1455,6 +1486,14 @@ int64_t mods_list_age() { return frames >= 0 && frames <= 2 ? 0 : prof_now() - t; } bool mods_list_fresh() { return mods_list_age() < 150000000LL; } +/// ((Triforce, 11-20) the owner: "the mods button ... should be synced with the timing of the other button ui so it +/// seamlessly moves and disappears with them". How much of its own list's buttons the game shows this frame, 0 to 1, as +/// the host reads it from the game (host/oot_menu.c, HOS_OPT_MODS_ALPHA); under 0 while it has not said (an older host) +float mods_list_shown() { + const int a = mods_list_alpha.load(std::memory_order_relaxed); + // (as the overlay has to draw it to look as the game's own buttons at that alpha: mods_entry_anim.h list_amount) + return a < 0 ? -1.0f : mods_entry::list_amount((float)a / 1000.0f, dev.hdr); +} // ((Triforce, 11-19) the title's list, the owner: "make it the same as the other options, with the green filigree, the same // animations: when you select it, it just pops in". The entry is drawn as the game's own entries (draw_title_mods, with // mods_entry_anim.h's times): it fades with the list, and A on it plays the game's Decide before the Mods screen opens) @@ -1902,6 +1941,8 @@ bool row_shown(int i) { (rows[i].value != &opts.language || lang_count > 1) && // (Triforce: the mods' cheats: as many rows as the host named, and their heading with them) (!rows[i].heading || strcmp(rows[i].name, kModCheatHeading) || mod_cheats.shown() > 0) && + // (Triforce: the Achievements row once the host has named them: off, or none, and there is no such row) + (strcmp(rows[i].name, kAchRow) || rows[i].tab != TAB_SYSTEM || ach_list.shown() > 0) && (mod_cheat_slot(rows[i].name) < 0 || mod_cheat_slot(rows[i].name) < mod_cheats.shown()) && (!rows[i].heading || rows[i].tab == TAB_DEBUG || group_shown(i)); } @@ -2602,6 +2643,13 @@ void dim_page(ui::Ctx& c, ImU32 col); // (the caption's texts: PaGuideKey's 48 units for the help, a little smaller for the line about the value) // (the game's caption is 54 units, 65 as drawn; ours say more than its one line, so a little smaller: 57 and 50) constexpr float kHelpPt = 19.0f, kHintPt = 16.6f; +constexpr float kModsDescPt = 15.2f; // ((Triforce, 11-18) the Mods description: a step under the hints, more of it in the panel) +// ((Triforce, 11-18) a page's floor: the caption's plate stops this far above the prompt bar, for a page with something of +// its own at the foot (the Mods page's Salesman). Set while that page draws, 0 for every other +float caption_floor = 0; +// (...and its margin: the plate's right end stays this far inside the window's edge while the Mods page draws; 0, every +// other page: the plate runs off the screen as the game's own caption plate does) +float caption_edge = 0; /// The tallest help a row of the options can have, in pixels at width: its help and the tallest line any of its values can /// have: the help box keeps that height whichever row is selected and whatever its value @@ -2710,7 +2758,12 @@ void help_box(ui::Ctx& c, float x0, float x1, float y0, const char* title, const float tall = std::max(min_h, content + 2 * pad_y); if (y0 + tall > y_max) y0 = std::max(y_min, y_max - tall); const float y1 = std::min(y0 + tall, y_max); - ui::box(c, ImVec2(x0, y0), ImVec2(x1 + bleed, y1)); + // ((Triforce, 11-18) on the Mods page (caption_edge) the plate's bleed stops short of the window's edge: at a large menu + // unit hx1 is 3770 of a 3840 column and the bleed ran off the screen; flush with the edge read as on it (his "fix the + // search bar being on the edge of the 16:9 window"), so it keeps a margin there. Never short of the text's own end, x1. + // The options' own pages keep the plate off the screen, as the game's caption plate is) + const float plate_x1 = caption_edge > 0 ? std::max(x1, std::min(x1 + bleed, (float)c.w - caption_edge)) : x1 + bleed; + ui::box(c, ImVec2(x0, y0), ImVec2(plate_x1, y1)); c.d->PushClipRect(ImVec2(x0, y0), ImVec2(x1, y1), true); float y = std::max(y0 + pad_y * 0.5f, (y0 + y1 - content) * 0.5f); if (title) ui::text(c, c.f.strong, 21.6f * s, ImVec2(x0 + pad_x, y), ui::kInk, title), y += 34 * s; @@ -2759,7 +2812,8 @@ void caption(ui::Ctx& c, float hx0, float hx1, const char* title, const char* he const float y0 = std::max(hh + 14 * s, mid - tall * 0.5f); // ((Triforce, 11-17) the plate's right end where the game's own ends on a window wider than 16:9 (measured at 3440 x 1440: // 51 pixels short of ours), off a 16:9 screen as before. Only the plate: the text keeps its width, hx0 to hx1) - help_box(c, hx0, hx1, y0, title, help, hint, gold_line, tall, after_restart, hh + 14 * s, bar_top - 8 * s, ui::lu(c, 252)); + help_box(c, hx0, hx1, y0, title, help, hint, gold_line, tall, after_restart, hh + 14 * s, bar_top - 8 * s - caption_floor, + ui::lu(c, 252)); } /// A row's name on its plate; pending: a change that applies after a restart, the game's corner mark at its top left in /// gold (the help says it in words). The selected row's plate is as the others': the game's cursor alone marks it @@ -3445,6 +3499,8 @@ void draw_inventory(ui::Ctx& c) { const int first = begin + inventory_page * inventory_capacity; const float width = (right - left + gap) / inventory_columns; c.d->PushClipRect(ImVec2(left - 12 * s, top - 12 * s), ImVec2(right + 12 * s, grid_bottom + 12 * s)); + ImVec2 glow_a{}, glow_b{}; // ((Triforce, 11-18) the chosen one's cursor, drawn after every item: the next one's + bool glow_have = false; // plate covered part of it, which read as the highlight clipped) for (int i = first; i < std::min(end, first + inventory_capacity); ++i) { const int cell = i - first; const ImVec2 a(left + (cell % inventory_columns) * width, top + (cell / inventory_columns) * pitch); @@ -3467,9 +3523,10 @@ void draw_inventory(ui::Ctx& c) { ui::text_mid(c, c.f.body, 15 * s, a.x + 13 * s, ly, owned ? ui::kAccent : kCardOff, amount); } c.d->PopClipRect(); - if (selected) ui::glow(c, a, b); + if (selected) glow_a = a, glow_b = b, glow_have = true; if (a.y < grid_bottom) hits.add(a.x, a.y, b.x, std::min(b.y, grid_bottom), menu_hit::kListLine, i); } + if (glow_have) ui::glow(c, glow_a, glow_b); c.d->PopClipRect(); if (detail >= top) { // (the page: its arrows, Page n of m and the range; the selected item's action at the right) c.d->AddLine(ImVec2(left, detail), ImVec2(right, detail), ui::alpha(ui::kEdge, 0.8f)); @@ -3582,6 +3639,8 @@ void draw_teleport(ui::Ctx& c) { c.d->AddLine(ImVec2(left + tw + 14 * s, hy + s), ImVec2(right - nw - 12 * s, hy + s), ui::alpha(ui::kAccent, 0.45f), std::max(1.0f, s)); } + ImVec2 glow_a{}, glow_b{}; // ((Triforce, 11-18) the chosen one's cursor, drawn after every item: the next one's + bool glow_have = false; // plate covered part of it, which read as the highlight clipped) for (int i = 0; i < kTeleports; ++i) { const int k = teleport_section_of(i), cell = i - teleport_sections[k].first; const ImVec2 a(left + cell % cols * width, top + card_y(i) - at); @@ -3594,9 +3653,10 @@ void draw_teleport(ui::Ctx& c) { c.d->PushClipRect(a, ImVec2(name_end, b.y), true); ui::text_mid(c, c.f.body, 17 * s, a.x + 16 * s, (a.y + b.y) * 0.5f, ui::kInk, teleport_names[i]); c.d->PopClipRect(); - if (selected) ui::glow(c, a, b); + if (selected) glow_a = a, glow_b = b, glow_have = true; hits.add(a.x, std::max(a.y, top), b.x, std::min(b.y, bottom), menu_hit::kListLine, i); } + if (glow_have) ui::glow(c, glow_a, glow_b); c.d->PopClipRect(); if (detail >= top) { // (the selection, its section and note, and Travel: what Enter does) c.d->AddLine(ImVec2(left, detail), ImVec2(right, detail), ui::alpha(ui::kEdge, 0.8f)); @@ -3705,6 +3765,8 @@ void draw_names(ui::Ctx& c) { const int first = p.first(), last = p.last(); const float width = (right - left + gap) / p.columns; c.d->PushClipRect(ImVec2(left - 12 * s, top - 12 * s), ImVec2(right + 12 * s, grid_bottom + 12 * s)); + ImVec2 glow_a{}, glow_b{}; // ((Triforce, 11-18) the chosen one's cursor, drawn after every item: the next one's + bool glow_have = false; // plate covered part of it, which read as the highlight clipped) for (int i = first; i < last; ++i) { const int cell = i - first; const ImVec2 a(left + (cell % p.columns) * width, top + (cell / p.columns) * pitch); @@ -3730,9 +3792,10 @@ void draw_names(ui::Ctx& c) { ui::text_mid(c, c.f.body, 15 * s, a.x + 13 * s, a.y + 37 * s, st == '1' ? ui::kAccent : st == '0' ? ui::kSoft : kCardOff, st == '1' ? "Set" : st == '0' ? "Not set" : st == '-' ? "Not in this save" : "Not known"); c.d->PopClipRect(); - if (selected) ui::glow(c, a, b); + if (selected) glow_a = a, glow_b = b, glow_have = true; if (a.y < grid_bottom) hits.add(a.x, a.y, b.x, std::min(b.y, grid_bottom), menu_hit::kListLine, i); } + if (glow_have) ui::glow(c, glow_a, glow_b); c.d->PopClipRect(); if (detail >= top) { // (the page: its arrows, Page n of m and the range; the chosen name's action at the right) c.d->AddLine(ImVec2(left, detail), ImVec2(right, detail), ui::alpha(ui::kEdge, 0.8f)); @@ -3954,11 +4017,14 @@ void salesman_says(ui::Ctx& c, float x0, float x1, float y1, const char* line) { /// ((Triforce) 11-17) The search's text in a..b at px, its end kept in view when it is longer; a caret after it while sel void mods_query_text(ui::Ctx& c, ImVec2 a, ImVec2 b, float px, bool sel, const char* empty_hint) { const float my = (a.y + b.y) * 0.5f, room = b.x - a.x; - c.d->PushClipRect(a, b, true); + // ((Triforce, 11-18) the clip keeps the caret's whole line: drawn on the clip's first pixel, half of it was cut, which + // read as the highlight clipped (the hint starts clear of it too)) + c.d->PushClipRect(ImVec2(a.x - 2 * c.s, a.y), ImVec2(b.x + 2 * c.s, b.y), true); + const float caret_x = a.x + 1.5f * c.s; if (mods_query.empty()) { - ui::text_mid(c, c.f.body, px * 0.92f, a.x, my, ui::alpha(ui::kSoft, 0.55f), empty_hint); + ui::text_mid(c, c.f.body, px * 0.92f, sel ? caret_x + 7 * c.s : a.x, my, ui::alpha(ui::kSoft, 0.55f), empty_hint); if (sel && (prof_now() / 500000000LL) % 2 == 0) - c.d->AddLine(ImVec2(a.x, my - px * 0.55f), ImVec2(a.x, my + px * 0.55f), ui::kInk, std::max(1.5f, 1.6f * c.s)); + c.d->AddLine(ImVec2(caret_x, my - px * 0.55f), ImVec2(caret_x, my + px * 0.55f), ui::kInk, std::max(1.5f, 1.6f * c.s)); } else { const float w = ui::size_of(c.f.body, px, mods_query.c_str()).x; const float x = w + 6 * c.s > room ? b.x - w - 6 * c.s : a.x; @@ -3984,8 +4050,11 @@ void mods_search_bar(ui::Ctx& c, ImVec2 a, ImVec2 b, const std::string& searched else if (feed == mods::Feed::Loading && !n) found = "Searching..."; else found = (n == 1 ? std::string("1 found") : std::to_string(n) + (more ? "+ found" : " found")); } - const float found_w = found.empty() ? 0 : ui::size_of(c.f.body, 15 * s, found.c_str()).x + 16 * s; - if (!found.empty()) ui::text_mid_right(c, c.f.body, 15 * s, b.x - 14 * s, my, ui::kAccent, found.c_str()); + // ((Triforce, 11-18) the count capped at two fifths of the bar, so the field keeps its room) + const float found_max = std::max(0.0f, (b.x - a.x) * 0.4f); + std::string found_fit; + const float found_w = found.empty() ? 0 : ui::fit_text(found.c_str(), c.f.body, 15 * s, found_max, &found_fit) + 16 * s; + if (!found_fit.empty()) ui::text_mid_right(c, c.f.body, 15 * s, b.x - 14 * s, my, ui::kAccent, found_fit.c_str()); mods_query_text(c, ImVec2(a.x + 46 * s, a.y), ImVec2(b.x - 14 * s - found_w, b.y), 18 * s, sel && !mods_kbd, sel ? "Type to search, or press Confirm for a keyboard" : "Search mods"); if (sel && !mods_kbd) ui::glow(c, a, b); @@ -4006,6 +4075,8 @@ void mods_keyboard(ui::Ctx& c) { ui::value_plate(c, fa, fb, false); mods_query_text(c, ImVec2(fa.x + 14 * s, fa.y), ImVec2(fb.x - 14 * s, fb.y), 20 * s, true, "Type a word to search for"); const float ky = fb.y + 16 * s; + ImVec2 glow_a{}, glow_b{}; // ((Triforce, 11-18) the chosen one's cursor, drawn after every item: the next one's + bool glow_have = false; // plate covered part of it, which read as the highlight clipped) for (int row = 0; row < 5; row++) { const int keys = row < 4 ? 10 : 4; for (int k = 0; k < keys; k++) { @@ -4018,10 +4089,23 @@ void mods_keyboard(ui::Ctx& c) { const char* t = row < 4 ? label : kKbdWide[k]; const ImU32 ink = on ? ui::kValueOnInk : row == 4 && k == 3 ? ui::kGoldHi : ui::kInk; ui::text_mid_centre(c, row < 4 ? c.f.strong : c.f.body, (row < 4 ? 21 : 17) * s, (ka.x + kb.x) * 0.5f, (ka.y + kb.y) * 0.5f, ink, t); - if (on) ui::glow(c, ka, kb); + if (on) glow_a = ka, glow_b = kb, glow_have = true; hits.add(ka.x, ka.y, kb.x, kb.y, menu_hit::kKey, row * 10 + col0); } } + if (glow_have) { + // ((Triforce, 11-18) his "the highlight left and right inside showing black bars": the cursor's opening stands a + // few units off the key's plate and the dark box showed between them. The chosen key's own plate is drawn a + // little larger under the cursor, its letter with it, so the gold runs to the frame on every side) + const float g = ui::lu(c, 13); + ui::value_plate(c, ImVec2(glow_a.x - g, glow_a.y - g), ImVec2(glow_b.x + g, glow_b.y + g), true); + const int row = mods_kbd_row, col = mods_kbd_col; + char label[2] = {row < 4 ? (char)toupper((unsigned char)kKbdRows[row][col]) : 0, 0}; + const char* t = row < 4 ? label : kKbdWide[kbd_wide(col)]; + ui::text_mid_centre(c, row < 4 ? c.f.strong : c.f.body, (row < 4 ? 21 : 17) * s, + (glow_a.x + glow_b.x) * 0.5f, (glow_a.y + glow_b.y) * 0.5f, ui::kValueOnInk, t); + ui::glow(c, glow_a, glow_b); + } } void draw_mods(ui::Ctx& c) { @@ -4035,11 +4119,20 @@ void draw_mods(ui::Ctx& c) { // (the Salesman's corner: the right of the cards' column, by the details; the cards narrower while he is there) if (salesman::state() == salesman::State::Idle && host && host->romfs_dir) salesman::start(host->romfs_dir()); const bool salesman_on = salesman::state() != salesman::State::Failed && mods::available(); - // ((Triforce) 11-17, layout A: the list at the left with the category tabs over it, the details at the right, the - // Salesman small at the list's foot with his line beside him) + // ((Triforce) 11-17, layout A: the list at the left with the category tabs over it, the details at the right) const float tabs_h = mods::available() && mods::accepted() ? 44 * s : 0; - const float sales_h = salesman_on && mods::accepted() ? 214 * s : 0; - const float left = cx0 + 34 * s, right = cx1 - 34 * s, top = hh + 18 * s + tabs_h, bottom = bar - 10 * s - sales_h; + // ((Triforce, 11-18) the mods expanded out, more of them on screen: the cards run to the prompt bar, and the Salesman has + // a band of his own at the right column's foot, under its plate, with his line beside him, over nothing) + // ((Triforce, 11-18) his band, but never over a quarter of the page: at a large menu unit 258 * s took almost all of a + // 2160-high window's room and the details panel had none left) + const float sales_h = salesman_on && mods::accepted() ? std::max(0.0f, std::min(300 * s, (bar - hh) * 0.26f)) : 0; + // (the caption plates stop above his band, and short of the window's edge, while this page draws, wherever it returns from) + struct CaptionFor { + float floor_was, edge_was; + CaptionFor(float floor, float edge) : floor_was(caption_floor), edge_was(caption_edge) { caption_floor = floor, caption_edge = edge; } + ~CaptionFor() { caption_floor = floor_was, caption_edge = edge_was; } + } const caption_for(sales_h, 16 * s); + const float left = cx0 + 34 * s, right = cx1 - 34 * s, top = hh + 18 * s + tabs_h, bottom = bar - 10 * s; const float search_h = 50 * s, list_top = top + search_h + 12 * s; // ((Triforce) the search bar over the cards) if (tabs_h > 0) { // (the categories: a strip of tabs over the list, the chosen one light on dark as the game's own tabs) const float ty = hh + 14 * s, th = 32 * s, gap = 8 * s, px = 15 * s; @@ -4120,6 +4213,8 @@ void draw_mods(ui::Ctx& c) { list[i].texture = nullptr, list[i].thumb_state = 0; } } + ImVec2 glow_a{}, glow_b{}; // ((Triforce, 11-18) the chosen one's cursor, drawn after every item: the next one's + bool glow_have = false; // plate covered part of it, which read as the highlight clipped) for (int k = 0; k < n; k++) { const int i = vis[k]; const ImVec2 a(left, list_top + k * pitch - at), b(right, a.y + card_h); @@ -4141,17 +4236,19 @@ void draw_mods(ui::Ctx& c) { float tag_w = 0; if (m.installed) tag_w = card_tag(c, b.x - 12 * s, a.y + 20 * s, m.update ? "Update" : "Installed") + 10 * s; c.d->PushClipRect(ImVec2(tx, a.y), ImVec2(b.x - 12 * s - tag_w, b.y), true); - ui::text_mid(c, c.f.strong, 19 * s, tx, a.y + 22 * s, ui::kInkBright, m.name.c_str()); + // ((Triforce, 11-18) each line cut on whole characters with an ellipsis where it does not fit, not by the clip alone) + ui::text_mid_fit(c, c.f.strong, 19 * s, tx, a.y + 22 * s, ui::kInkBright, m.name.c_str(), b.x - 12 * s - tag_w - tx); c.d->PopClipRect(); c.d->PushClipRect(ImVec2(tx, a.y), ImVec2(b.x - 12 * s, b.y), true); const std::string by = "by " + m.author + (m.category.empty() ? "" : " \xC2\xB7 " + m.category) + mods_kind(m); - ui::text_mid(c, c.f.body, 15 * s, tx, a.y + 48 * s, ui::kSoft, by.c_str()); + ui::text_mid_fit(c, c.f.body, 15 * s, tx, a.y + 48 * s, ui::kSoft, by.c_str(), b.x - 12 * s - tx); const std::string counts = mods_count(m.likes) + " likes \xC2\xB7 " + mods_count(m.views) + " views"; - ui::text_mid(c, c.f.body, 13.5f * s, tx, a.y + 70 * s, ui::alpha(ui::kSoft, 0.72f), counts.c_str()); + ui::text_mid_fit(c, c.f.body, 13.5f * s, tx, a.y + 70 * s, ui::alpha(ui::kSoft, 0.72f), counts.c_str(), b.x - 12 * s - tx); c.d->PopClipRect(); - if (sel) ui::glow(c, a, b); + if (sel) glow_a = a, glow_b = b, glow_have = true; hits.add(a.x, std::max(a.y, list_top), b.x, std::min(b.y, bottom), menu_hit::kListLine, i); } + if (glow_have) ui::glow(c, glow_a, glow_b); if (n && (more || feed == mods::Feed::Loading)) { // ((Triforce) 11-18, the foot of the list: the pages come by themselves) const float y = list_top + n * pitch - at + 16 * s; if (y < bottom) ui::text_mid_centre(c, c.f.body, 15 * s, (left + right) * 0.5f, y, feed == mods::Feed::Failed ? ui::kAccent : ui::kDim, @@ -4189,50 +4286,88 @@ void draw_mods(ui::Ctx& c) { can_install = m.details_state == 2 && files > 0; install_label = m.update ? "Update" : m.installed ? "Reinstall" : "Install"; if (m.kit) install_label = m.update ? "Update and start" : m.installed ? "Start" : "Install and start"; - const float x0 = hx0, x1 = hx1, pad = ui::lu(c, 64), y0 = top, y1 = bar - 8 * s; + const float x0 = hx0, x1 = hx1, pad = ui::lu(c, 64), y0 = top, y1 = bar - 8 * s - sales_h; // ((Triforce, 11-18) his band kept clear) ui::box(c, ImVec2(x0, y0), ImVec2(x1, y1)); c.d->PushClipRect(ImVec2(x0, y0), ImVec2(x1, y1), true); const float w = x1 - x0 - 2 * pad; float y = y0 + pad * 0.6f; - const ImVec2 pa(x0 + pad, y), pb(x0 + pad + w, y + w * 9 / 16); - mods_picture(c, m, pa, pb, &made); - y = pb.y + 16 * s; - y += ui::para(c, c.f.strong, 20 * s, ImVec2(x0 + pad, y), ui::kInk, m.name.c_str(), w) + 4 * s; - const std::string by = "by " + m.author + (m.category.empty() ? "" : " \xC2\xB7 " + m.category) + mods_kind(m); - ui::text(c, c.f.body, 15 * s, ImVec2(x0 + pad, y), ui::kAccent, by.c_str()); + // (the file chosen, its line, and a kit's note: measured for the header before they are drawn) + static const char* const kit_note = "A program of its own: it goes into the kits folder beside the game and starts in a window of " + "its own, where it shows its steps. No restart."; + const mods::File* file = m.details_state == 2 && files ? &m.files[std::clamp(m.file_choice, 0, files - 1)] : nullptr; + const std::string file_line = !file ? std::string() + : (files > 1 ? "File " + std::to_string(m.file_choice + 1) + " of " + std::to_string(files) + ": " : std::string("File: ")) + + file->name + " (" + mods_size(file->bytes) + ")"; + float file_h = 0; + if (file) { + file_h = ui::para(c, c.f.strong, 14 * s, ImVec2(), 0, file_line.c_str(), w - 16 * s, false) + 2 * s; + if (!file->description.empty()) file_h += ui::para(c, c.f.body, 13.5f * s, ImVec2(), 0, file->description.c_str(), w - 16 * s, false); + if (m.kit) file_h += ui::para(c, c.f.body, 13.5f * s, ImVec2(), 0, kit_note, w - 16 * s, false) + 2 * s; + file_h += 8 * s; + } + // ((Triforce, 11-18) his "the actual mod detail preview windows information to be compacted and actually be + // readable in that window size": the header's own height is measured at this menu's unit, the description keeps + // three lines, and the picture is a 16:9 box of what is left over (none when that is nothing: the words matter + // more), so nothing the panel holds is cut whatever the window and the unit do) + const float panel_h = y1 - y0; + const float title_max = 2 * 1.3f * 20 * s; // ((Triforce, 11-18) a long name is two lines at most) + const float title_h = std::min(ui::para(c, c.f.strong, 20 * s, ImVec2(), 0, m.name.c_str(), w, false), title_max) + 4 * s; + // (under the picture: 16, the name, the author's and the counts' lines (24 each), the file, the rule's 12) + const float header_h = 16 * s + title_h + 48 * s + file_h + 12 * s; + const float desc_need = 3 * kModsDescPt * s * 1.5f; + const float pic_room = panel_h - pad * 1.2f - header_h - desc_need; + const float pic_h = pic_room > 60 * s ? std::min(pic_room, std::min(panel_h * 0.42f, 420 * s)) : 0.0f; + const float pic_w = pic_h > 0 ? std::min(w, pic_h * 16.0f / 9.0f) : 0.0f; + const ImVec2 pa(x0 + pad, y), pb(x0 + pad + pic_w, y + pic_h); + { // (what the panel measured: once for each mod chosen, for a run's log) + static int64_t told_for = -2; + if (told_for != mods_choice) + told_for = mods_choice, logf("menu: mods details: panel %.0f..%.0f (%.0f tall), picture %.0fx%.0f at %.0f, " + "files %d, details %d, window %ux%u, unit %.3f, header %.0f, description %.0f, search %.0f..%.0f x %.0f..%.0f", + y0, y1, panel_h, pic_w, pic_h, pa.x, files, m.details_state, (unsigned)c.w, (unsigned)c.h, s, + header_h, panel_h - pad * 1.2f - header_h - pic_h, left, right, top, top + search_h); + } + if (pic_h > 0) mods_picture(c, m, pa, pb, &made); + y = (pic_h > 0 ? pb.y : y) + 16 * s; + { // ((Triforce, 11-18) drawn in the room the header was measured with: the rest is cut, not pushed down) + c.d->PushClipRect(ImVec2(x0 + pad, y), ImVec2(x1 - pad, y + title_max), true); + ui::para(c, c.f.strong, 20 * s, ImVec2(x0 + pad, y), ui::kInk, m.name.c_str(), w); + c.d->PopClipRect(); + y += title_h; + } + // ((Triforce, 11-18) no blank "by " line: the category alone without an author, the kind alone without either) + std::string by = m.author.empty() ? m.category : "by " + m.author + (m.category.empty() ? "" : " \xC2\xB7 " + m.category); + if (const std::string kind = mods_kind(m); !kind.empty()) by += by.empty() ? kind.substr(6) : kind; // (6: its " · " in front) + if (!by.empty()) ui::text_fit(c, c.f.body, 15 * s, ImVec2(x0 + pad, y), ui::kAccent, by.c_str(), w); y += 24 * s; std::string counts = mods_count(m.likes) + " likes \xC2\xB7 " + mods_count(m.views) + " views"; if (m.downloads >= 0) counts += " \xC2\xB7 " + mods_count(m.downloads) + " downloads"; - ui::text(c, c.f.body, 14 * s, ImVec2(x0 + pad, y), ui::kSoft, counts.c_str()); + ui::text_fit(c, c.f.body, 14 * s, ImVec2(x0 + pad, y), ui::kSoft, counts.c_str(), w); y += 24 * s; - if (m.details_state == 2 && files) { - const mods::File& f = m.files[std::clamp(m.file_choice, 0, files - 1)]; - std::string line = (files > 1 ? "File " + std::to_string(m.file_choice + 1) + " of " + std::to_string(files) + ": " : std::string("File: ")) + - f.name + " (" + mods_size(f.bytes) + ")"; + if (file) { ui::diamond(c, ImVec2(x0 + pad + 5 * s, y + 9 * s), 4 * s, ui::kGoldHi); - y += ui::para(c, c.f.strong, 14 * s, ImVec2(x0 + pad + 16 * s, y), ui::kGoldHi, line.c_str(), w - 16 * s) + 2 * s; - if (!f.description.empty()) y += ui::para(c, c.f.body, 13.5f * s, ImVec2(x0 + pad + 16 * s, y), ui::kSoft, f.description.c_str(), w - 16 * s); + y += ui::para(c, c.f.strong, 14 * s, ImVec2(x0 + pad + 16 * s, y), ui::kGoldHi, file_line.c_str(), w - 16 * s) + 2 * s; + if (!file->description.empty()) y += ui::para(c, c.f.body, 13.5f * s, ImVec2(x0 + pad + 16 * s, y), ui::kSoft, file->description.c_str(), w - 16 * s); if (m.kit) // ((Triforce) 11-18) a program of its own, not a mod in the game's mods folder - y += ui::para(c, c.f.body, 13.5f * s, ImVec2(x0 + pad + 16 * s, y + 2 * s), ui::kAccent, - "A program of its own: it goes into the kits folder beside the game and starts in a window of its own, " - "where it shows its steps. No restart.", w - 16 * s) + 2 * s; + y += ui::para(c, c.f.body, 13.5f * s, ImVec2(x0 + pad + 16 * s, y + 2 * s), ui::kAccent, kit_note, w - 16 * s) + 2 * s; y += 8 * s; } c.d->AddLine(ImVec2(x0 + pad, y), ImVec2(x1 - pad, y), ui::alpha(ui::kEdge, 0.8f), std::max(1.0f, s)); y += 12 * s; const char* text = m.details_state == 2 ? m.text.c_str() : m.details_state == 3 ? "The details could not be read from GameBanana." : "Loading the details..."; if (m.details_state == 2 && !files) text = "This mod has no file to download on GameBanana."; + if (m.details_state == 2 && files && m.text.empty()) text = "No description on GameBanana."; // ((Triforce, 11-18)) { // ((Triforce) 11-17) the description in its own window down to the panel's foot, scrolled by the right stick or the wheel const float dtop = y, dbot = y1 - pad * 0.6f; if (mods_desc_for != mods_choice) mods_desc_for = mods_choice, mods_desc_px = 0; - const float th = ui::para(c, c.f.body, kHintPt * s, ImVec2(x0 + pad, dtop), ui::kInk, text, w, false); + const float th = ui::para(c, c.f.body, kModsDescPt * s, ImVec2(x0 + pad, dtop), ui::kInk, text, w, false); // ((Triforce, 11-18) smaller) const float room = std::max(0.0f, dbot - dtop), most = std::max(0.0f, th - room); if (menu_pad_raw >> HOS_PAD_RS_DOWN & 1) mods_desc_px += 10 * s; if (menu_pad_raw >> HOS_PAD_RS_UP & 1) mods_desc_px -= 10 * s; if (mods_desc_wheel) mods_desc_px -= mods_desc_wheel * 60 * s, mods_desc_wheel = 0; mods_desc_px = std::clamp(mods_desc_px, 0.0f, most); c.d->PushClipRect(ImVec2(x0, dtop), ImVec2(x1, dbot), true); - ui::para(c, c.f.body, kHintPt * s, ImVec2(x0 + pad, dtop - mods_desc_px), ui::kInk, text, w); + ui::para(c, c.f.body, kModsDescPt * s, ImVec2(x0 + pad, dtop - mods_desc_px), ui::kInk, text, w); c.d->PopClipRect(); if (most > 0) scroll_bar(c, x1 - pad * 0.45f, dtop, dbot, (int)th, (int)room, mods_desc_px); mods_desc_x0 = x0; @@ -4250,8 +4385,12 @@ void draw_mods(ui::Ctx& c) { if (ask_details >= 0) mods::want_details(ask_details); for (int i : ask_thumbs) mods::want_thumb(i); if (salesman_on) { // (him, and a line now and then: a new one when the choice moves, not more than every 9 s) - const float sy0 = bottom + 10 * s, sy1 = bar - 8 * s; - salesman_picture(c, ImVec2(left, sy0), ImVec2(left + 170 * s, sy1)); + // ((Triforce, 11-18) his "move the text box ... not covering the mods and make the model ... not cover mods either": + // in the details column's foot, under its plate, where nothing is drawn, with his line beside him to the left: the + // cards keep the list's whole height and neither he nor his box is over them) + const float sy1 = bar - 8 * s, sy0 = sy1 - sales_h; + const float sx1 = hx1, sx0 = hx1 - sales_h * 0.82f; // ((Triforce, 11-18) his width follows the band: he keeps his shape) + salesman_picture(c, ImVec2(sx0, sy0), ImVec2(sx1, sy1)); const int64_t now = prof_now(); static int last_choice = -1; if (last_choice != mods_choice && now - salesman_line_at > 9000000000LL) { @@ -4259,7 +4398,7 @@ void draw_mods(ui::Ctx& c) { salesman_line = (salesman_line + 1) % (int)(sizeof salesman_lines / sizeof *salesman_lines); } if (salesman::state() == salesman::State::Ready && inst.job == mods::Job::None) - salesman_says(c, left + 178 * s, right, bar - 14 * s, salesman_lines[salesman_line]); + salesman_says(c, hx0, sx0 - 10 * s, bar - 14 * s, salesman_lines[salesman_line]); } // (the install under way, or its end: over the page) if (inst.job == mods::Job::Downloading || inst.job == mods::Job::Unpacking) { @@ -4430,7 +4569,7 @@ constexpr ImU32 kTitleGlow = ui::rgba(120, 140, 255); mods_entry::Anim title_anim; void draw_title_mods(ui::Ctx& c, bool sel, float cx, float cy, float k, float pitch) { const int64_t now = prof_now(), age = mods_list_age(); - title_anim.step(now, sel, std::min(age, mods_entry::kListShownNs)); + title_anim.step(now, sel, std::min(age, mods_entry::kListShownNs), mods_list_shown()); const float show = title_anim.shown, f = title_anim.focus; const ImVec2 m = ui::at(c, cx, cy); const char* const label = "Mods"; @@ -4527,6 +4666,7 @@ void draw_mods_row(ui::Ctx& c, bool sel) { static const char* const names[2] = {"Mods", "Quit"}; static const bool still = getenv("RX_VK_UI_STILL") != nullptr; // (pictures that compare: as it stands) const int64_t now = prof_now(); + const int vtx_row = c.d->VtxBuffer.Size; for (int i = 0; i < 2; i++) { const float y = cy - pitch * i; const ImVec2 a = ui::at(c, cx - 336, y + 72), b = ui::at(c, cx + 336, y - 72); @@ -4535,6 +4675,10 @@ void draw_mods_row(ui::Ctx& c, bool sel) { hits.add(a.x, a.y, b.x, b.y, menu_hit::kButton, i); if (i == 0) mods_entry_px[0] = a.x, mods_entry_px[1] = a.y, mods_entry_px[2] = b.x, mods_entry_px[3] = b.y; } + // ((Triforce, 11-20) Mods and Quit as much there as the game's own buttons above them are (mods_list_shown): they + // come and go with the System page, not a frame before or after) + if (const float list = mods_list_shown(); list >= 0 && list < 1) + for (int v = vtx_row; v < c.d->VtxBuffer.Size; v++) c.d->VtxBuffer[v].col = ui::alpha(c.d->VtxBuffer[v].col, list); mods_entry_px[4] = ui::lu(c, pitch); return; } @@ -4609,6 +4753,194 @@ void end_game(bool restart) { if (const HWND window = app_window()) PostMessageA(window, restart ? WM_APP + 9 : WM_CLOSE, 0, 0); } +/// (Triforce) 11-18, his "if you can draw these in game style can you not do that for the achievements too?": the game's own +/// picture for an achievement's icon, where it has one (vk_game_tex.h takes them from his game's files when they are there): +/// a Spiritual Stone and a medallion by their colours, as the load list's plate has them (DataFileImportantIcon_00 to _08: +/// the stones of the forest, of fire, of water; the medallions of light, forest, fire, water, spirit, shadow); a song's icon +/// for a note; a Gold Skulltula's mark for a token, and a rupee for the green one. nullptr: none, the drawn icon +const char* ach_art_name(int icon) { + namespace ar = achievement_rows; + const int tone = ar::tone_of(icon); + switch (ar::shape_of(icon)) { + case ar::kGem: + return tone == ar::kGreen ? "DataFileImportantIcon_00" : tone == ar::kRed ? "DataFileImportantIcon_01" : tone == ar::kBlue ? "DataFileImportantIcon_02" : nullptr; + case ar::kMedal: + return tone == ar::kYellow ? "DataFileImportantIcon_03" : tone == ar::kGreen ? "DataFileImportantIcon_04" : tone == ar::kRed ? "DataFileImportantIcon_05" : + tone == ar::kBlue ? "DataFileImportantIcon_06" : tone == ar::kOrange ? "DataFileImportantIcon_07" : tone == ar::kPurple ? "DataFileImportantIcon_08" : nullptr; + case ar::kNote: return "OcarinaMelodyIcon_00"; + case ar::kToken: return tone == ar::kGreen ? "RupeeIconNew_00" : "MapIconCollectSkullSign_00"; + } + return nullptr; +} +/// (Triforce) An achievement's icon: the game's own picture of what it is about where there is one (ach_art_name), else +/// drawn: a dark disc with a ring, and on it one of achievement_rows.h's shapes in one of its tones. One not earned in grey. +/// m its middle, r its radius +void ach_icon(ui::Ctx& c, ImVec2 m, float r, int icon, bool earned) { + namespace ar = achievement_rows; + if (const char* const name = ach_art_name(icon)) { + if (const ui::Art* t = ui::art(name)) { + static const ImU32 tints[ar::kTones] = {ui::rgba(232, 184, 74), ui::rgba(96, 196, 104), ui::rgba(226, 92, 76), ui::rgba(92, 158, 236), + ui::rgba(168, 112, 226), ui::rgba(238, 146, 62), ui::rgba(246, 228, 128), ui::rgba(200, 208, 218)}; + // (a picture as it is, a mask in the icon's tone; one not earned dark and half there) + const ImU32 col = !earned ? ui::rgba(96, 90, 78, 150) : t->mask ? tints[ar::tone_of(icon)] : IM_COL32_WHITE; + const float hw = r * 1.2f, hh = hw * t->h / t->w; + ui::image(c, t, ImVec2(m.x - hw, m.y - hh), ImVec2(m.x + hw, m.y + hh), col); + return; + } + } + static const ImU32 tones[ar::kTones] = {ui::rgba(232, 184, 74), ui::rgba(96, 196, 104), ui::rgba(226, 92, 76), ui::rgba(92, 158, 236), + ui::rgba(168, 112, 226), ui::rgba(238, 146, 62), ui::rgba(246, 228, 128), ui::rgba(200, 208, 218)}; + const ImU32 tone = earned ? tones[ar::tone_of(icon)] : ui::rgba(112, 106, 92), dark = ui::rgba(20, 15, 9, 245); + const ImU32 cut = ui::alpha(dark, 0.6f); // (a line drawn on a shape: its facets) + const float line = std::max(1.5f, r * 0.09f), g = r * 0.56f; + c.d->AddCircleFilled(m, r, dark, 48); + c.d->AddCircle(m, r - line * 0.5f, earned ? tone : ui::alpha(tone, 0.7f), 48, line); + const auto p = [&](float x, float y) { return ImVec2(m.x + x * g, m.y + y * g); }; // (a place of the shape: -1 to 1 each way, y down) + switch (ar::shape_of(icon)) { + case ar::kSword: { // (point up and to the right: the blade and its point, the guard across, the grip and its pommel) + c.d->AddLine(p(-0.28f, 0.28f), p(0.62f, -0.62f), tone, g * 0.30f); + c.d->AddTriangleFilled(p(0.50f, -0.74f), p(0.74f, -0.50f), p(0.96f, -0.96f), tone); + c.d->AddLine(p(-0.72f, -0.10f), p(0.10f, 0.72f), tone, g * 0.20f); + c.d->AddLine(p(-0.31f, 0.31f), p(-0.78f, 0.78f), tone, g * 0.18f); + c.d->AddCircleFilled(p(-0.84f, 0.84f), g * 0.15f, tone, 16); + break; + } + case ar::kGem: { + const ImVec2 pts[] = {p(-0.55f, -0.80f), p(0.55f, -0.80f), p(0.95f, -0.22f), p(0, 0.95f), p(-0.95f, -0.22f)}; + c.d->AddConvexPolyFilled(pts, 5, tone); + c.d->AddLine(p(-0.95f, -0.22f), p(0.95f, -0.22f), cut, line * 0.8f); + c.d->AddLine(p(-0.38f, -0.22f), p(0, 0.95f), cut, line * 0.8f), c.d->AddLine(p(0.38f, -0.22f), p(0, 0.95f), cut, line * 0.8f); + c.d->AddLine(p(-0.38f, -0.22f), p(-0.20f, -0.80f), cut, line * 0.8f), c.d->AddLine(p(0.38f, -0.22f), p(0.20f, -0.80f), cut, line * 0.8f); + break; + } + case ar::kMedal: + c.d->AddCircleFilled(m, g * 0.92f, tone, 32); + c.d->AddCircle(m, g * 0.62f, cut, 32, line); + ui::diamond(c, m, g * 0.26f, cut); + break; + case ar::kNote: // (its head, its stem, its flag) + c.d->AddCircleFilled(p(-0.28f, 0.55f), g * 0.36f, tone, 24); + c.d->AddLine(p(0, 0.55f), p(0, -0.85f), tone, g * 0.17f); + c.d->AddLine(p(0, -0.80f), p(0.62f, -0.42f), tone, g * 0.22f); + c.d->AddLine(p(0.62f, -0.42f), p(0.46f, 0.02f), tone, g * 0.14f); + break; + case ar::kToken: + c.d->AddCircle(m, g * 0.72f, tone, 32, g * 0.30f); + c.d->AddCircleFilled(m, g * 0.22f, tone, 16); + break; + case ar::kShield: { + const ImVec2 pts[] = {p(-0.78f, -0.85f), p(0.78f, -0.85f), p(0.78f, 0.05f), p(0, 0.95f), p(-0.78f, 0.05f)}; + c.d->AddConvexPolyFilled(pts, 5, tone); + c.d->AddLine(p(0, -0.85f), p(0, 0.95f), cut, line), c.d->AddLine(p(-0.78f, -0.25f), p(0.78f, -0.25f), cut, line); + break; + } + default: { // (a star: its five points about a pentagon) + ImVec2 in[5], out[5]; + for (int i = 0; i < 5; i++) { + const float a = -1.5707963f + i * 1.2566371f, h = a + 0.6283185f; + out[i] = p(cosf(a), sinf(a)), in[i] = p(0.42f * cosf(h), 0.42f * sinf(h)); + } + c.d->AddConvexPolyFilled(in, 5, tone); + for (int i = 0; i < 5; i++) c.d->AddTriangleFilled(in[(i + 4) % 5], out[i], in[i], tone); + } + } +} + +/// (Triforce) The Achievements page (kAchPage): a card for each the host named, its icon, its name and whether it is earned, +/// in pages; the chosen one's line about it in the caption. Nothing is set here: they are earned in the game +void draw_achievements(ui::Ctx& c) { + const float s = c.s; + const int n = ach_list.shown(); + char count_text[48]; + snprintf(count_text, sizeof count_text, "%d of %d earned", ach_list.earned_count(), n); + const float hh = title_banner(c, "Achievements", count_text); + const ui::Prompt prompts[] = {{ui::kUpDown, "Move"}, {ui::kLeftRight, "Move"}, {ui::kTabs, "Page"}, {ui::kBack, "Back"}}; + float cx0, cx1, hx0, hx1; + bool narrow; + columns(c, &cx0, &cx1, &hx0, &hx1, &narrow); + column(c, cx0, cx1, hh); + const float bar = ui::prompt_bar_top(c); + const float left = cx0 + 24 * s, right = cx1 - 24 * s, gap = 10 * s; + const float top = hh + 22 * s, pitch = 74 * s; + const float strip = 58 * s, detail = bar - 14 * s - strip, grid_bottom = std::max(top, detail - 8 * s); + ach_columns = right - left >= 620 * s ? 2 : 1; + ach_capacity = ach_columns * std::max(1, (int)((grid_bottom - top + gap) / pitch)); + ach_view.keep(n, ach_capacity); + const int choice = ach_view.choice; + if (n > 0) { + const bool got = ach_list.is_earned(choice); + char when[48] = "Earned"; + if (got && ach_list.days[choice][0]) snprintf(when, sizeof when, "Earned on %s", ach_list.days[choice]); + caption(c, hx0, hx1, ach_list.names[choice], ach_list.abouts[choice][0] ? ach_list.abouts[choice] : nullptr, + got ? nullptr : "Not earned yet.", got ? when : nullptr); + } + const int first = ach_view.page * ach_capacity, last = std::min(n, first + ach_capacity); + const float width = (right - left + gap) / ach_columns; + c.d->PushClipRect(ImVec2(left - 12 * s, top - 12 * s), ImVec2(right + 12 * s, grid_bottom + 12 * s)); + for (int i = first; i < last; ++i) { + const int cell = i - first; + const ImVec2 a(left + (cell % ach_columns) * width, top + (cell / ach_columns) * pitch); + const ImVec2 b(a.x + width - gap, a.y + pitch - gap); + const bool earned = ach_list.is_earned(i); + ui::plate(c, a, b, earned); + const float r = 23 * s, tx = a.x + 14 * s + 2 * r + 13 * s; + ach_icon(c, ImVec2(a.x + 14 * s + r, (a.y + b.y) * 0.5f), r, ach_list.icons[i], earned); + c.d->PushClipRect(ImVec2(tx, a.y), b - ImVec2(8 * s, 0), true); + ui::text_mid(c, c.f.strong, 15.5f * s, tx, a.y + 21 * s, earned ? ui::kInkBright : ui::kSoft, ach_list.names[i]); + char line[40] = "Locked"; + if (earned && ach_list.days[i][0]) snprintf(line, sizeof line, "Earned %s", ach_list.days[i]); + else if (earned) snprintf(line, sizeof line, "Earned"); + ui::text_mid(c, c.f.body, 14 * s, tx, a.y + 43 * s, earned ? ui::kAccent : kCardOff, line); + c.d->PopClipRect(); + if (i == choice) ui::glow(c, a, b); + if (a.y < grid_bottom) hits.add(a.x, a.y, b.x, std::min(b.y, grid_bottom), menu_hit::kListLine, i); + } + c.d->PopClipRect(); + if (detail >= top && n > 0) { // (the page: its arrows, Page n of m and the range, as the Inventory's) + c.d->AddLine(ImVec2(left, detail), ImVec2(right, detail), ui::alpha(ui::kEdge, 0.8f)); + const int pages = achievement_rows::View::pages(n, ach_capacity), at = ach_view.page; + const float my = detail + strip * 0.5f + 2 * s, side = 32 * s; + float px0 = left; + if (pages > 1) { + page_arrow(c, ImVec2(px0, my - side * 0.5f), side, 0, at > 0); + hits.add(px0, my - side * 0.5f, px0 + side, my + side * 0.5f, menu_hit::kButton, 2); + char page_text[32]; + snprintf(page_text, sizeof page_text, "Page %d of %d", at + 1, pages); + const float tw = ui::size_of(c.f.strong, 17 * s, page_text).x; + ui::text_mid(c, c.f.strong, 17 * s, px0 + side + 12 * s, my, ui::kInkBright, page_text); + const float nx = px0 + side + 24 * s + tw; + page_arrow(c, ImVec2(nx, my - side * 0.5f), side, 1, at + 1 < pages); + hits.add(nx, my - side * 0.5f, nx + side, my + side * 0.5f, menu_hit::kButton, 3); + px0 = nx + side + 18 * s; + } + char range[64]; + snprintf(range, sizeof range, "%d\xe2\x80\x93%d of %d", first + 1, last, n); + ui::text_mid(c, c.f.body, 15 * s, px0, my, ui::kSoft, range); + } + ui::prompt_bar(c, prompts, (int)(sizeof prompts / sizeof prompts[0])); +} + +/// (Triforce) The notice of an achievement earned: a plate at the top of the screen, its icon, "Achievement earned" and its +/// name, slid in from above and out again (achievement_rows.h notice_in). Over the game and over the options alike +void draw_ach_notice(ui::Ctx& c) { + const int slot = ach_notice; + if (slot < 0 || slot >= ach_list.shown()) return; + const float k = achievement_rows::notice_in((prof_now() - ach_notice_at) / 1000000, kAchNoticeMs); + if (k <= 0) return; + const float s = c.s, in = 1 - (1 - k) * (1 - k), r = 26 * s, pad = 16 * s; + const float text_w = std::max(ui::size_of(c.f.strong, 21 * s, ach_list.names[slot]).x, ui::size_of(c.f.strong, 13.5f * s, "Achievement earned").x); + const float w = pad + 2 * r + 16 * s + text_w + 28 * s, h = 76 * s; + const ImVec2 a((c.w - w) * 0.5f, -h + (h + 34 * s) * in), b(a.x + w, a.y + h); + // ((Triforce) 11-18: the game's own plate, as the file select's cards and the Achievements page's, with a gold rim; it was a + // dark rounded box of the menu's own) + ui::plate(c, a, b, true); + c.d->AddRect(a, b, ui::alpha(ui::kGoldHi, 0.9f), 0, 0, std::max(1.5f, 1.6f * s)); + ach_icon(c, ImVec2(a.x + pad + r, (a.y + b.y) * 0.5f), r, ach_list.icons[slot], true); + const float tx = a.x + pad + 2 * r + 16 * s; + ui::text_mid(c, c.f.strong, 13.5f * s, tx, a.y + 25 * s, ui::kGoldHi, "Achievement earned"); + ui::text_mid(c, c.f.strong, 21 * s, tx, a.y + 51 * s, ui::kInkBright, ach_list.names[slot]); +} + void draw_ask(ui::Ctx& c) { const float s = c.s; dim_page(c, ui::rgba(8, 5, 2, 110)); // (the dialog over the page: darker still; the whole window, the Full panel with it) @@ -4727,6 +5059,13 @@ void prebake(const ui::Ctx& c) { memcpy(scrolled_to, saved_to, sizeof scrolled_to); } +/// ((Triforce, 11-20) how much of the Mods screen shows now, as its In plays (mods_page_anim.h); 1 on every other page, and +/// in a still (no window's frames stepping the easing, or RX_VK_UI_STILL for pictures that compare): as it stands +float mods_screen_alpha() { + static const bool still = getenv("RX_VK_UI_STILL") != nullptr; + return page == kModsPage ? mods_screen.alpha(prof_now(), !ease_live || still) : 1.0f; +} + /// The menu: the game dimmed across the whole window, the pages in a column as wide as a 16:9 screen of this height, in the /// middle (a window of 16:9 or narrower: the whole of it). The pages draw from 0 as on 16:9; their vertices and clip /// rectangles are moved over after, and menu_tick moves the mouse back through menu_map before it tests the hits. (Triforce) @@ -4735,6 +5074,16 @@ void prebake(const ui::Ctx& c) { /// its list is: game_grid.h) void draw_menu(ui::Ctx& c, bool perf_under) { const bool row = page == kModsRowPage; // ((Triforce) the Mods entry has the cursor: over the game's list, nothing dimmed) + // ((Triforce, 11-20) the Mods screen's In: while it plays, the entries that opened it (Mods, and Quit under it) are still + // drawn under it, on the game's list, and go as the screen comes; without them the list stood a frame without its Mods + // entry before the screen covered it) + const float mods_in = mods_screen_alpha(); + if (mods_in < 1 && (mods_list_fresh() || mods_title_fading())) { + const int vtx_under = c.d->VtxBuffer.Size; + draw_mods_row_shut(c); + for (int i = vtx_under; i < c.d->VtxBuffer.Size; i++) c.d->VtxBuffer[i].col = ui::alpha(c.d->VtxBuffer[i].col, 1 - mods_in); + } + const int vtx_page = c.d->VtxBuffer.Size; // (from here the page: its background, its column and all it draws) // (the game behind: inside its own Options screen its own background, untouched; over play the System screens' own // (ui::system_bg), or dimmed flat without its textures) if (!embed_open && !row && !ui::system_bg(c)) c.d->AddRectFilled(ImVec2(0, 0), ImVec2(c.w, c.h), ui::rgba(8, 5, 2, ui::kDimAlpha)); @@ -4791,6 +5140,7 @@ void draw_menu(ui::Ctx& c, bool perf_under) { else if (page == 4) draw_inventory(k); else if (page == 6) draw_names(k); // ((Triforce) Game flags, Cutscenes, Scan amiibo) else if (page == kModsPage) draw_mods(k); + else if (page == kAchPage) draw_achievements(k); // ((Triforce) the System tab's Achievements) else draw_options(k); if (on_game) c.d->PopClipRect(); int perf0 = 0, perf1 = 0; // (the Full panel's vertices: in the window's pixels already) @@ -4798,6 +5148,10 @@ void draw_menu(ui::Ctx& c, bool perf_under) { panel_between(c.d); perf0 = panel_wait.v0, perf1 = panel_wait.v1; } + // ((Triforce, 11-20) the Mods screen fades in whole: the background, the column, the cards, the details, the Salesman's + // picture and the prompts; not the Full panel, which was there before it) + for (int i = vtx_page; mods_in < 1 && i < c.d->VtxBuffer.Size; i++) + if (i < perf0 || i >= perf1) c.d->VtxBuffer[i].col = ui::alpha(c.d->VtxBuffer[i].col, mods_in); if (menu_map.ox <= 0 && menu_map.oy <= 0 && menu_map.kx == 1) return; // (a clip rectangle of the column's: moved with it; the whole window's (the dim's, and the draw list's own) stays, as it // holds the column wherever it is) @@ -5130,7 +5484,10 @@ void startup_screen(const StartupView* view) { int hdr_at_start = 0; int renodx_at_start = 0, upscale_pre_at_start = 0; -int tex_bc1_at_start = 0; +// ((Triforce, 11-21, the performance audit's TEX-50) the row's own default, not 0: with no options file (a first start) +// nothing set it, and the textures stayed BC3 although the row said On +int tex_bc1_at_start = Options().tex_bc1; +static_assert(Options().tex_bc1 == 1, "Opaque textures as BC1 is on by default: a first start takes it"); int tex_detail_at_start = 0; int replay_threads_at_start = replay_threads::kDefaultRow; static_assert(Options().replay_threads == replay_threads::kDefaultRow, "the row's default is replay_threads.h's"); @@ -5139,6 +5496,9 @@ bool renodx_chosen() { return e && *e ? atoi(e) != 0 : renodx_at_start != 0; } +// ((Triforce, 11-22, the performance audit's PRESENT-46) menu_init found no options file: a first start, menu_first_start) +static bool first_start_no_file = false; + void menu_init() { static bool done; if (done) return; @@ -5233,6 +5593,7 @@ void menu_init() { // low-VRAM hardware without requiring the menu to render first. ((Triforce, options_version 7) What the Quality preset's // Minimum set, without the preset: the 3D at 50%, the handheld profile's output, the smallest textures, frames shown // evenly; the memory budgets are the card's own) + first_start_no_file = !has_options; // ((Triforce, 11-22, PRESENT-46) for menu_first_start, once the card is known) if (!has_options && getenv("RX_VK_LOW_SPEC") && atoi(getenv("RX_VK_LOW_SPEC")) != 0) { opts.resolution = ur::snap(upscaler(), 4), opts.output_res = merged_rows::kHandheldOutput, opts.tex_detail = 3; opts.pace = 1, opts.realtime = 1, opts.sharpness = 0; @@ -5254,6 +5615,41 @@ void menu_init() { restart_init(); // (what this run started with: the rows that take a restart differ from it once changed) } +/// (Triforce, 11-22, the performance audit's PRESENT-46, after Source A's) A first start sized to the card, with +/// RX_VK_FIRST_START_AUTO (off unless set: first_start.h): with no options file, RX_VK_LOW_SPEC not asked and no switch +/// setting the Render resolution, a weak card (an integrated GPU, no mesh shaders, 4.5 GB of video memory or less) starts +/// with the Render resolution row at 50%, or at 33% for an integrated GPU beside 4 threads or fewer, in place of Auto. +/// Only that row: the upscaler, the textures and the output stay as they are, and nothing is saved, so every start chooses +/// again until the player saves options of their own. Called once the device is made (vk_main.cpp mkwvk_init: menu_init +/// runs before the card is known), before the host's one read of the row +void menu_first_start() { + static bool done; + if (done) return; + done = true; + const int64_t asked = first_start::switch_mb(getenv("RX_VK_FIRST_START_AUTO")); + const bool low_spec = getenv("RX_VK_LOW_SPEC") && atoi(getenv("RX_VK_LOW_SPEC")) != 0; + if (!first_start::applies(!first_start_no_file, low_spec, getenv("HOS_DYNRES") != nullptr, asked)) return; + first_start::Card card; + card.integrated = dev.props.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU; + card.mesh_shaders = dev.mesh_shader; + card.vram_mb = asked > 0 ? (uint64_t)asked : device_vram_bytes() >> 20; + card.threads = cpu_count(); + const first_start::Choice choice = first_start::choose(card); + if (!choice.row) { + logf("menu: first start: no change (%s: %llu MB of video memory%s, mesh shaders %s; RX_VK_FIRST_START_AUTO)", + first_start::why_text(choice.why), (unsigned long long)card.vram_mb, asked > 0 ? " asked" : "", + card.mesh_shaders ? "yes" : "no"); + return; + } + opts.resolution = ur::snap(upscaler(), choice.row); + // (the row takes a restart: what this run starts with is this, not menu_init's) + for (const Row& r : rows) + if (r.value == &opts.resolution) restart_state.restart_value(restart_slot_of(r), restart_value_of(r)); + logf("menu: first start: tier 1 (%s, %llu MB of video memory%s, %u threads): Render resolution %s " + "(RX_VK_FIRST_START_AUTO; nothing is saved)", first_start::why_text(choice.why), (unsigned long long)card.vram_mb, + asked > 0 ? " asked" : "", card.threads, tx_resolution(opts.resolution)); +} + bool resolution_auto() { return opts.resolution == 0; } int resolution_scale() { const float s = upscaler_rows::percent(opts.resolution) / 100.0f; @@ -5361,12 +5757,24 @@ static menu_sound::State sound_state() { } else if (page == 6) { s.cursor = names_page[names_kind].choice, s.group = names_page[names_kind].group; s.mode = names_edit ? menu_sound::kModeEdit : menu_sound::kModeNone; + } else if (page == kAchPage) { + s.cursor = ach_view.choice, s.group = ach_view.page; } return s; } int menu_take_sound() { return sound_queue.take(prof_now()); } // (the options' next sound, for the host's sound thread) void mods_sound(int id) { if (opts.menu_sounds > 0) sound_queue.put(id, prof_now()); } // ((Triforce) the Mods entry's and screen's own sounds int menu_mods_selected() { return open_now.load(std::memory_order_relaxed) && (page == kModsRowPage || page == kModsPage); } +/// ((Triforce, 11-20) the owner: "have the in game mods menu in the title screen play the shop theme quietly in the +/// background ... it mutes the title screen music's audio while in the mod menu and fades in/out". What the host's sound +/// thread asks each block (host/mods_music.c): 2 while the Mods screen is open from the title's list, 1 while that list is +/// there with its entry (the tune is read then, ahead of the screen), 0 otherwise: the pause menu's Mods screen keeps the +/// music of where the player is +int menu_mods_music() { + if (!mods_title_list()) return 0; + if (open_now.load(std::memory_order_relaxed) && page == kModsPage) return 2; + return mods_list_fresh() || mods_title_fading() ? 1 : 0; +} int menu_text_wanted() { return mods_text_wanted.load(std::memory_order_relaxed) ? 1 : 0; } // ((Triforce) the Mods screen's search) int menu_bug_shot() { return bugshot_state(); } @@ -5633,7 +6041,15 @@ void menu_option_set(int id, int value) { mods_box[id - HOS_OPT_MODS_X0].store(value, std::memory_order_relaxed); return; } + if (id == HOS_OPT_MODS_ALPHA) { // ((Triforce, 11-20) how much of its own buttons the game shows: the entry's too) + mods_list_alpha.store(std::clamp(value, 0, 1000), std::memory_order_relaxed); + return; + } if (id == HOS_OPT_MODS_LIST) { + if (!(value & HOS_MODS_LIST_SHOWN)) { // ((Triforce, 11-20) the list is no longer the entry's: gone at once, not after 150 ms) + mods_list_ns.store(0, std::memory_order_release); + return; + } mods_list_bits.store(value, std::memory_order_relaxed); mods_list_frame.store(game_frames.load(std::memory_order_relaxed), std::memory_order_relaxed); mods_list_ns.store(prof_now(), std::memory_order_release); @@ -5661,6 +6077,19 @@ void menu_option_set(int id, int value) { game_options_from_pause.store(value == 2, std::memory_order_relaxed); return; } + // (Triforce: the achievements, named by the host once this renderer answers, and each one earned as it is: + // host/achievements.c, one call at a time) + if (id == HOS_OPT_ACH_TEXT) return ach_list.text(value); + if (id == HOS_OPT_ACH_ADD) return ach_list.add(value); + if (id == HOS_OPT_ACH_COUNT) { + ach_list.done(value); + menu_refresh = true; + return; + } + if (id == HOS_OPT_ACH_EARNED) { + if (ach_list.earn(value)) menu_refresh = true; + return; + } // (Triforce: the cheats of the mods' cheat files, named by the host at the first frame: host/mod_cheats.c. A cheat may not // have the name of a row that acts by its name, share_row's) if (id == HOS_OPT_MODCHEAT_TEXT) return mod_cheats.text(value); @@ -5912,15 +6341,20 @@ void mods_tick(bool* press, bool tab_left, bool tab_right, bool mouse, bool move if (!dialog && mods_kbd) { // ((Triforce) the on-screen keyboard: a key pointed at is chosen, a click types it; a click off it shuts it) if (hit.part == menu_hit::kKey && (moved || ptr.pressed)) { const int r = hit.index / 10, col = hit.index % 10; - if (r != mods_kbd_row || col != mods_kbd_col) mods_kbd_row = r, mods_kbd_col = col, changed = true; + if (r != mods_kbd_row || col != mods_kbd_col) { + mods_kbd_row = r, mods_kbd_col = col, changed = true; + if (ptr.pressed & HOS_MOUSE_LEFT) mods_sound(HOS_MENU_SOUND_CURSOR); // (a click; hover alone is quiet) + } } const bool inside = mx >= mods_kbd_box[0] && mx <= mods_kbd_box[2] && my >= mods_kbd_box[1] && my <= mods_kbd_box[3]; if ((ptr.pressed & HOS_MOUSE_LEFT) && hit.part == menu_hit::kKey) press[7] = true, kbd_click = true; else if ((ptr.pressed & HOS_MOUSE_LEFT) && hit.part != menu_hit::kSearch && !inside) mods_kbd = false, changed = true; } else if (!dialog) { - if (hit.part == menu_hit::kListLine && (moved || ptr.pressed) && hit.index != mods_choice) mods_choice = hit.index, mods_follow = true, changed = true; + if (hit.part == menu_hit::kListLine && (moved || ptr.pressed) && hit.index != mods_choice) + mods_choice = hit.index, mods_follow = true, changed = true, mods_sound(HOS_MENU_SOUND_CURSOR); if (hit.part == menu_hit::kSearch && (moved || ptr.pressed) && mods_choice != -1) mods_choice = -1, mods_follow = true, changed = true; - if ((ptr.pressed & HOS_MOUSE_LEFT) && hit.part == menu_hit::kSearch) mods_kbd = true, mods_kbd_row = 1, mods_kbd_col = 0, changed = true; + if ((ptr.pressed & HOS_MOUSE_LEFT) && hit.part == menu_hit::kSearch) + mods_kbd = true, mods_kbd_row = 1, mods_kbd_col = 0, changed = true, mods_sound(HOS_MENU_SOUND_DECIDE); if (ptr.pressed & HOS_MOUSE_LEFT) { if (hit.part == menu_hit::kTabPrev) tab_left = true; else if (hit.part == menu_hit::kTabNext) tab_right = true; @@ -5937,39 +6371,41 @@ void mods_tick(bool* press, bool tab_left, bool tab_right, bool mouse, bool move return; } if (!mods::accepted()) { // (the warning: I understand, or Not now) - if (press[1] || press[2] || press[3] || press[4]) mods_btn ^= 1, changed = true; + if (press[1] || press[2] || press[3] || press[4]) mods_btn ^= 1, changed = true, mods_sound(HOS_MENU_SOUND_CURSOR); if (press[7] && mods_btn == 0) { mods::accept(); logf("mods: the warning accepted"); mods_open_page(); - changed = true; + changed = true, mods_sound(HOS_MENU_SOUND_DECIDE); } else if (press[5] || press[7]) { // (Not now: back to the game's list) + mods_sound(HOS_MENU_SOUND_CANCEL); mods_back(changed); } return; } if (busy) { // (only Cancel) - if (press[5] || press[7]) mods::cancel(), changed = true; + if (press[5] || press[7]) mods::cancel(), changed = true, mods_sound(HOS_MENU_SOUND_CANCEL); return; } if (ended) { if (inst.job == mods::Job::Done && inst.kit) { // ((Triforce) 11-18) a kit started: OK alone - if (press[5] || press[7]) mods::clear_done(), mods_btn = 0, changed = true; + if (press[5] || press[7]) mods::clear_done(), mods_btn = 0, changed = true, mods_sound(HOS_MENU_SOUND_DECIDE); } else if (inst.job == mods::Job::Done) { - if (press[1] || press[2] || press[3] || press[4]) mods_btn ^= 1, changed = true; + if (press[1] || press[2] || press[3] || press[4]) mods_btn ^= 1, changed = true, mods_sound(HOS_MENU_SOUND_CURSOR); if (press[7] && mods_btn == 0) { // (Restart Now: as the restart question's, everything saved first) logf("mods: restart now, to load %s", inst.mod_name.c_str()); mods::clear_done(); + mods_sound(HOS_MENU_SOUND_DECIDE); save(); pipeline_cache_save(true); memory_trial_orderly_end(); open_now = false, page = 0, changed = true; PostMessageA(app_window(), WM_APP + 9, 0, 0); } else if (press[5] || press[7]) { - mods::clear_done(), mods_btn = 0, changed = true; + mods::clear_done(), mods_btn = 0, changed = true, mods_sound(HOS_MENU_SOUND_CANCEL); } } else if (press[5] || press[7]) { - mods::clear_done(), mods_btn = 0, changed = true; + mods::clear_done(), mods_btn = 0, changed = true, mods_sound(HOS_MENU_SOUND_CANCEL); } return; } @@ -5982,6 +6418,8 @@ void mods_tick(bool* press, bool tab_left, bool tab_right, bool mouse, bool move else if (cp == 127) edited |= !mods_query.empty(), mods_query.clear(); else if (cp >= 0x20 && mods_query.size() < 100) utf8_add(mods_query, cp), edited = true; } + if (edited && !mods_kbd) mods_sound(HOS_MENU_SOUND_CURSOR); // ((Triforce, 11-18) a character typed; the host's + // cursor gap keeps fast typing from stacking) if (edited && !mods_kbd && mods_choice != -1) mods_choice = -1, mods_follow = true; // (typing: the cursor on the bar) if (mods_kbd && !pad_last && !kbd_click && (press[7] || press[5])) { if (press[7]) search_now = true; @@ -5998,7 +6436,7 @@ void mods_tick(bool* press, bool tab_left, bool tab_right, bool mouse, bool move if (mods_kbd_row == 4) mods_kbd_col = kKbdWideAt[(kbd_wide(mods_kbd_col) + (press[4] ? 1 : 3)) % 4]; else mods_kbd_col = (mods_kbd_col + (press[4] ? 1 : 9)) % 10; } - if (mods_kbd_row != was_row || mods_kbd_col != was_col) changed = true; + if (mods_kbd_row != was_row || mods_kbd_col != was_col) changed = true, mods_sound(HOS_MENU_SOUND_CURSOR); if (press[7]) { if (mods_kbd_row < 4) { if (mods_query.size() < 100) utf8_add(mods_query, kKbdRows[mods_kbd_row][mods_kbd_col]), edited = true; @@ -6009,12 +6447,12 @@ void mods_tick(bool* press, bool tab_left, bool tab_right, bool mouse, bool move else if (k == 2) edited |= !mods_query.empty(), mods_query.clear(); else if (k == 3) search_now = true, mods_kbd = false; } - changed = true; + changed = true, mods_sound(search_now ? HOS_MENU_SOUND_DECIDE : HOS_MENU_SOUND_CURSOR); // (a key pressed) } if (press[5]) { // (Back: a character off; with none left the keyboard shut) if (!mods_query.empty()) edited |= utf8_pop(mods_query); else mods_kbd = false; - changed = true; + changed = true, mods_sound(mods_kbd ? HOS_MENU_SOUND_CURSOR : HOS_MENU_SOUND_CANCEL); } press[7] = press[5] = false; for (int i = 1; i <= 4; i++) press[i] = false; @@ -6067,11 +6505,12 @@ void mods_tick(bool* press, bool tab_left, bool tab_right, bool mouse, bool move else if (press[16]) at = n - 1; else at = std::clamp(at + (press[2] - press[1]) + (press[14] - press[13]) * mods_page_rows, -1, std::max(n - 1, -1)); mods_choice = at >= 0 ? vis[at] : -1; - if (mods_choice != was) mods_follow = true, changed = true; + if (mods_choice != was) mods_follow = true, changed = true, mods_sound(HOS_MENU_SOUND_CURSOR); if (mods_choice >= 0) { mods::Mod& m = mods::list()[mods_choice]; const int files = (int)m.files.size(); - if ((press[3] || press[4]) && files > 1) m.file_choice = (m.file_choice + files + (press[4] ? 1 : -1)) % files, changed = true; + if ((press[3] || press[4]) && files > 1) + m.file_choice = (m.file_choice + files + (press[4] ? 1 : -1)) % files, changed = true, mods_sound(HOS_MENU_SOUND_CURSOR); if (press[7] && m.details_state == 2 && files) install = mods_choice; } else if (press[7]) { // (Confirm on the bar: tried again after a failure; else the on-screen keyboard (a pad), or the results) if (feed == mods::Feed::Failed) retry = true; // ((Triforce) 11-18: also a list that stopped with cards in it) @@ -6082,14 +6521,14 @@ void mods_tick(bool* press, bool tab_left, bool tab_right, bool mouse, bool move mods::unlock(); if (install >= 0) { mods::install(install); - mods_btn = 0, changed = true; + mods_btn = 0, changed = true, mods_sound(HOS_MENU_SOUND_DECIDE); } else if (retry) { // (try again: the search, or the list; (Triforce) 11-18: a list that stopped on a page goes on from it) if (searching) mods_retire(mods::search(mods::search_text())); else if (n_all) mods::load_more(); else mods_retire(mods::open()); - changed = true; + changed = true, mods_sound(HOS_MENU_SOUND_CURSOR); } else if (keyboard) { - mods_kbd = true, mods_kbd_row = 1, mods_kbd_col = 0, changed = true; + mods_kbd = true, mods_kbd_row = 1, mods_kbd_col = 0, changed = true, mods_sound(HOS_MENU_SOUND_DECIDE); } // ((Triforce) 11-18) the next pages: asked for by the engine one after another (vk_mods.cpp fetch_page), not by the cursor if (feed == mods::Feed::Loading) changed = true; // (drawn again as it comes) @@ -6316,6 +6755,8 @@ void menu_tick() { snprintf(notice_text, sizeof notice_text, "%s", share_lines[HOS_SHARE_NOTICE].c_str()); notice_until = prof_now() + 4000000000LL; } + // ((Triforce) an achievement earned: its notice, from now for kAchNoticeMs) + if (const int earned = ach_list.take_fresh(); earned >= 0) ach_notice = earned, ach_notice_at = prof_now(); if (cur_tab == TAB_DEBUG && !debug_shown()) cur_tab = TAB_DISPLAY; { // (Triforce) presses the host passed on while the Mods entry or its screen has the cursor (a test's scripted ones) const int mp = mods_press.exchange(0, std::memory_order_relaxed); @@ -6678,6 +7119,19 @@ void menu_tick() { else teleport_choice = std::clamp(teleport_step_rows(teleport_choice, rows) + (press[4] - press[3]), 0, kTeleports - 1); if (teleport_choice != was) teleport_follow = true, changed = true; } + } else if (open_now.load() && page == kAchPage) { // (Triforce) the Achievements page: nothing to set, the choice moves + const int n = ach_list.shown(); + if (mouse) { // (a card chosen by a click, the page arrows, the wheel a page; a right click goes back) + const menu_hit::Hit hit = hits.at((float)mx, (float)my); + if ((ptr.pressed & HOS_MOUSE_LEFT) && hit.part == menu_hit::kListLine) changed |= ach_view.choose(hit.index, n); + else if ((ptr.pressed & HOS_MOUSE_LEFT) && hit.part == menu_hit::kButton && (hit.index == 2 || hit.index == 3)) + press[hit.index == 2 ? 13 : 14] = true; + if (ptr.pressed & HOS_MOUSE_RIGHT) press[5] = true; + if (ptr.wheel) changed |= ach_view.turn(-ptr.wheel, n, ach_capacity); + } + if (press[5] || !n) page = 0, changed = true; // (to the System tab, Achievements still selected) + else changed |= ach_view.move(press[4] - press[3], press[2] - press[1], (press[14] - press[13]) + (tab_right ? 1 : 0) - (tab_left ? 1 : 0), + press[15], press[16], n, ach_columns, ach_capacity); } else if (open_now.load() && page == kModsRowPage && mods_decide_ns) { // ((Triforce, 11-19) the title's entry's Decide plays // (its sound with the press; no input meanwhile, as the game takes none during its own; the Mods screen at its end) if (!mods_list_fresh()) mods_decide_ns = 0, open_now = false, page = 0, changed = true; @@ -6738,7 +7192,9 @@ void menu_tick() { open_now = false, page = 0, mods_row_sel = 0, changed = true; } } else if (open_now.load() && page == kModsPage) { // (Triforce) the Mods page - mods_tick(press, tab_left, tab_right, mouse, moved, ptr, mx, my, changed); + // ((Triforce, 11-20) it takes input once its In has played, as the game's screens and Quit's dialog: what comes + // before is not kept) + if (mods_screen.ready(prof_now())) mods_tick(press, tab_left, tab_right, mouse, moved, ptr, mx, my, changed); } else if (open_now.load()) { if (mouse) { // (the mouse: see the top of this file) const menu_hit::Hit hit = hits.at((float)mx, (float)my); @@ -6836,6 +7292,7 @@ void menu_tick() { // rows that act by their name or open a page: a cheat's name is the file's) else if (mod_cheat_slot(r.name) >= 0) changed |= mod_cheats.press(mod_cheat_slot(r.name), press[3], press[4], press[7]); else if (press[7] && share_row(r)) share_press(share_row(r)), press[7] = false, changed = true; // (Triforce: the host does it) + else if (press[7] && is_ach_row(r)) page = kAchPage, ach_view.reveal(ach_capacity), press[7] = false, changed = true; else if (press[7] && is_motion_row(r)) cur_tab = TAB_MOTION, tab_selected[TAB_MOTION] = 0, press[7] = false, changed = true; else if (press[7] && !r.value && can_bind()) page = 1, csel = 0, ccol = 0, changed = true; if ((press[3] || press[4] || press[7]) && r.value) { // (Enter, or A, as Right) @@ -6883,6 +7340,7 @@ void menu_tick() { // (Triforce) the Salesman's thread draws only while the Mods page shows him salesman::show(open_now.load() && page == kModsPage && mods::available() && mods::accepted(), salesman_w, salesman_h); if (open_now.load() && page == kModsPage) changed = true; // (he moves: drawn again) + mods_screen.step(prof_now(), open_now.load() && page == kModsPage); // ((Triforce, 11-20) its In from the tick it came at) { // (hookshot) the window lets the mouse go while the menu is open, and takes it back after (host/display.c) static bool told = false; const bool now_open = open_now.load(); @@ -7017,7 +7475,8 @@ void menu_record(VkCommandBuffer cmd, uint32_t index, uint32_t w, uint32_t h) { const int64_t game_seen = game_options_ns.load(std::memory_order_relaxed); const bool game_tab = !open && !start && embed_allowed() && game_seen && prof_now() - game_seen < 250000000LL && embed_since; const bool mods_row = !open && !start && (mods_list_fresh() || mods_title_fading()); // ((Triforce) the Mods entry under the game's list) - if (!start && !fps && !open && !note && !game_tab && !mods_row && !bench_on) { // (nothing shows: no ImGui frame, no render pass) + const bool earned = !start && ach_notice >= 0 && prof_now() - ach_notice_at < kAchNoticeMs * 1000000; // ((Triforce) an achievement's notice) + if (!start && !fps && !open && !note && !game_tab && !mods_row && !bench_on && !earned) { // (nothing shows: no ImGui frame, no render pass) // (but a new window size or Menu size since the last bake: one frame with nothing drawn that bakes the menu's glyphs // and measures its page for the new size, so the next open has no hitch whatever was resized while nothing showed) if (prebake_needed(w, h) && ui::begin(w, h)) { @@ -7059,6 +7518,7 @@ void menu_record(VkCommandBuffer cmd, uint32_t index, uint32_t w, uint32_t h) { if (full && over_row) draw_perf(c); if (fps && !full) draw_fps(c, (open && page != kModsRowPage) || game_tab); // ((Triforce) the Mods entry: the counter stays where the game list has it) if (note) draw_notice(c); + if (earned) draw_ach_notice(c); if (bench_on) { // ((Triforce) benchmark mode's panel, in the overlay's own fonts, over the rest ui::Ctx bc = ui_ctx(w, h); draw_bench(bc, bench); @@ -7075,7 +7535,8 @@ void menu_record_image(VkCommandBuffer cmd, VkImage image, VkImageView view, uin const bool mods_row = !open && (mods_list_fresh() || mods_title_fading()); static MkwVkBench bench; // ((Triforce) benchmark mode's panel, in the shots too) const bool bench_on = bench_now(&bench); - if (!open && !fps && !note && !game_tab && !mods_row && !bench_on) return; + const bool earned = ach_notice >= 0 && prof_now() - ach_notice_at < kAchNoticeMs * 1000000; // ((Triforce) as menu_record) + if (!open && !fps && !note && !game_tab && !mods_row && !bench_on && !earned) return; if (!ui::begin(w, h)) return; // ((Triforce) a still: with no window's frames stepping the easing (menu_record), the lists stand where the selection asks. // Before, a headless shot, and tests/ui_preview.cpp's pictures, showed every list at its top whatever was selected) @@ -7092,6 +7553,7 @@ void menu_record_image(VkCommandBuffer cmd, VkImage image, VkImageView view, uin if (full && over_row) draw_perf(c); if (fps && !full) draw_fps(c, (open && page != kModsRowPage) || game_tab); // ((Triforce) the Mods entry: the counter stays where the game list has it) if (note) draw_notice(c); + if (earned) draw_ach_notice(c); if (bench_on) { ui::Ctx bc = ui_ctx(w, h); draw_bench(bc, bench); diff --git a/gpu/vk/vk_menu.h b/gpu/vk/vk_menu.h index 8d0332e..50537dd 100644 --- a/gpu/vk/vk_menu.h +++ b/gpu/vk/vk_menu.h @@ -176,6 +176,9 @@ extern int tex_bc1_at_start; bool preset_caps_memory(); void menu_init(); // (the options from the file, once) +/// (Triforce, 11-22, the performance audit's PRESENT-46) With RX_VK_FIRST_START_AUTO, a first start (no options file) on a +/// weak card sets the Render resolution row; once, after the device is made (first_start.h) +void menu_first_start(); void menu_set_gpu_count(int count); /// ((Triforce) 11-20) the GPU device row's card by its name, as triforce_options.ini's gpu_device_name had it (empty: none /// saved): vk_main.cpp pick_device finds the card by it first (gpu_choice.h pick) @@ -205,6 +208,9 @@ int menu_take_sound(); /// (Triforce) 1 while the Mods entry under a game list has the cursor, or its screen is open (HOS_OPT_MODS_SELECTED: the /// host lets the game's own cursor go then, host/oot_menu.c) int menu_mods_selected(); +/// ((Triforce, 11-20) the Mods screen's music: 0 none, 1 the title's list is there (the tune read ahead), 2 the Mods screen +/// is open from the title's list (HOS_OPT_MODS_MUSIC, host/mods_music.c) +int menu_mods_music(); /// (Triforce) 1 while the Mods screen's search takes typed text (HOS_OPT_TEXT_WANTED: host/display.c sends it then) int menu_text_wanted(); /// (Triforce) What became of the picture a bug capture asked for (vk_bugshot.h bugshot_state): 0 none asked, 1 not there yet, diff --git a/gpu/vk/vk_predict.cpp b/gpu/vk/vk_predict.cpp index bacbabd..689678f 100644 --- a/gpu/vk/vk_predict.cpp +++ b/gpu/vk/vk_predict.cpp @@ -11,6 +11,7 @@ #include #include +#include "predict_types.h" // ((Triforce, 11-22, PIPE-44) the prediction's checks: the vertex inputs' types) #include "vk_draw.h" #include "vk_shader.h" @@ -24,6 +25,7 @@ uint64_t fnv(const void* p, size_t n, uint64_t h = 0xcbf29ce484222325ull) { struct Face { uint64_t vs = 0, fs = 0; // the vertex (or mesh) and fragment modules, by their SPIR-V (0: no fragment stage) uint32_t vs_locs = 0, fs_locs = 0; // generic inputs read, colour outputs written + predict_types::Locations vs_in; // ((Triforce, 11-22, PIPE-44) the vertex inputs' base types, RX_VK_PREDICT_TYPES=1) }; /// The locations of a module's Input (storage 1) or Output (3) variables uint32_t spirv_locations(const std::vector& w, uint32_t storage) { @@ -78,28 +80,65 @@ struct Seen { it->second.insert(pv); } }; -/// Every class of an exact module, else the one most program variants of the signature had -std::vector candidates(const Seen* exact, const Seen* sig) { - if (exact && !exact->order.empty()) return exact->order; +/// Every class of an exact module, else the one most program variants of the signature had. ((Triforce, 11-22, the +/// performance audit's PIPE-44, after Source A's) Of those that fit: fits is always true but with RX_VK_PREDICT_TYPES=1, +/// where a class whose attributes definitely differ from the program's vertex inputs' types is dropped and counted in +/// *misfits, and an exact module whose classes all differ falls back to the signature's) +template +std::vector candidates(const Seen* exact, const Seen* sig, Fits fits, size_t* misfits) { + if (exact && !exact->order.empty()) { + std::vector kept; + for (const Class& c : exact->order) + if (fits(c)) kept.push_back(c); + else (*misfits)++; + if (!kept.empty()) return kept; + } if (!sig || sig->order.empty()) return {}; const Class* best = nullptr; size_t most = 0; for (const Class& c : sig->order) - if (sig->by.at(c).size() > most) most = sig->by.at(c).size(), best = &c; + if (sig->by.at(c).size() > most) { + if (!fits(c)) { + (*misfits)++; + continue; + } + most = sig->by.at(c).size(), best = &c; + } + if (!best) return {}; return {*best}; } +/// ((Triforce, 11-22, PIPE-44) What a note's vertex layout hands each location: unset (it reads zeros, as in play) or the +/// attribute's kind as the translator reads it (attribute_type_of; scaled formats as integers) +void layout_kinds(const PipeNote& n, predict_types::Kind out[16]) { + for (int i = 0; i < 16; i++) { + out[i] = predict_types::Kind::Unknown; + const uint32_t a = n.s.vattr[i]; + if (i >= n.s.nvattr || (a >> 27) >= 15) continue; + switch (attribute_type_of(a)) { + case Shader::AttributeType::Float: out[i] = predict_types::Kind::Float; break; + case Shader::AttributeType::UnsignedInt: case Shader::AttributeType::UnsignedScaled: out[i] = predict_types::Kind::Uint; break; + case Shader::AttributeType::SignedInt: case Shader::AttributeType::SignedScaled: out[i] = predict_types::Kind::Sint; break; + default: break; + } + } +} } // namespace +/// ((Triforce, 11-21, PIPE-83) into the caller's note: predict_background makes one at a time of the two notes it reads) +void predicted_into(PipeNote& n, const PipeNote& V, const PipeNote& O, uint64_t prog, uint64_t variant) { + n = O; + std::memcpy(n.s.vattr, V.s.vattr, sizeof n.s.vattr); + n.s.nvattr = V.s.nvattr; + std::memcpy(n.s.vstream, V.s.vstream, sizeof n.s.vstream); + n.topo = V.topo; + n.prog = prog, n.variant = variant; + n.key = 0; +} namespace { std::unique_ptr predicted_note(const PipeNote& V, const PipeNote& O, uint64_t prog, uint64_t variant) { - auto n = std::make_unique(O); - std::memcpy(n->s.vattr, V.s.vattr, sizeof n->s.vattr); - n->s.nvattr = V.s.nvattr; - std::memcpy(n->s.vstream, V.s.vstream, sizeof n->s.vstream); - n->topo = V.topo; - n->prog = prog, n->variant = variant; - n->key = 0; + auto n = std::make_unique(); + predicted_into(*n, V, O, prog, variant); return n; } } // namespace @@ -118,14 +157,19 @@ std::vector> predict_rebuild(const std::vector> predict_pipelines(const std::vector>& notes, size_t* programs, - std::vector* plan) { + std::vector* plan, PredictCounts* counts) { + // ((Triforce, 11-22, PIPE-44) RX_VK_PREDICT_TYPES=1: the type check, predict_types.h) + static const bool typed = predict_types::typed(predict_types::mode(getenv("RX_VK_PREDICT_TYPES"))); + PredictCounts tally; std::unordered_map, Face, PairHash> faces; program_cache_each([&](const KeptProgram& k) { Face f; for (const auto& [stage, spirv] : k.stages) { const uint64_t h = fnv(spirv.data(), spirv.size() * 4); - if (stage == VK_SHADER_STAGE_VERTEX_BIT) f.vs = h, f.vs_locs = spirv_locations(spirv, 1); - else if (stage == VK_SHADER_STAGE_TASK_BIT_EXT || stage == VK_SHADER_STAGE_MESH_BIT_EXT) f.vs = h, f.vs_locs = 0; + if (stage == VK_SHADER_STAGE_VERTEX_BIT) { + f.vs = h, f.vs_locs = spirv_locations(spirv, 1); + if (typed) f.vs_in = predict_types::locations(spirv); + } else if (stage == VK_SHADER_STAGE_TASK_BIT_EXT || stage == VK_SHADER_STAGE_MESH_BIT_EXT) f.vs = h, f.vs_locs = 0; else if (stage == VK_SHADER_STAGE_FRAGMENT_BIT) f.fs = h, f.fs_locs = spirv_locations(spirv, 3); } faces[{k.key, k.variant}] = f; @@ -136,7 +180,10 @@ std::vector> predict_pipelines(const std::vector pv{n->prog, n->variant}; const auto it = faces.find(pv); - if (it == faces.end()) continue; + if (it == faces.end()) { // ((Triforce, 11-22, PIPE-44) counted: a noted variant the programs' file has no face for) + tally.no_face++; + continue; + } const Face& f = it->second; const Class v = vertex_class(*n, f.vs_locs), o = output_class(*n); vh[f.vs].add(v, pv), vsig[f.vs_locs].add(v, pv), fh[f.fs].add(o, pv), fsig[f.fs_locs].add(o, pv); @@ -151,8 +198,15 @@ std::vector> predict_pipelines(const std::vector vc = candidates(find(vh, f.vs), find(vsig, f.vs_locs)); - const std::vector oc = candidates(find(fh, f.fs), find(fsig, f.fs_locs)); + const auto vertex_fits = [&](const Class& c) { + if (!typed || !f.vs_in.mask) return true; + predict_types::Kind kinds[16]; + layout_kinds(*vdonor.at(c), kinds); + return predict_types::fits(f.vs_in, kinds); + }; + const auto any = [](const Class&) { return true; }; + const std::vector vc = candidates(find(vh, f.vs), find(vsig, f.vs_locs), vertex_fits, &tally.misfits); + const std::vector oc = candidates(find(fh, f.fs), find(fsig, f.fs_locs), any, &tally.misfits); if (vc.empty() || oc.empty()) continue; (*programs)++; for (const Class& v : vc) @@ -163,6 +217,7 @@ std::vector> predict_pipelines(const std::vector kChunk ? n + align : kChunk; if (nchunks == kMaxChunks || cap > g_hard_budget || capacity > g_hard_budget - cap) return nullptr; + new_wait::NoWait at_once; // ((Triforce, OS-04) no chunk: the call is made directly; not a wait at the commit limit) void* p = ::operator new(cap, std::align_val_t(64), std::nothrow); if (!p) return nullptr; chunks[nchunks++] = {static_cast(p), cap}; @@ -311,6 +314,9 @@ inline uint64_t now_ns() { inline void (*g_thread_start_hook)() = nullptr; /// (set by vk_replay.cpp: its end_process, which on Windows ends the process without running every library's detach) inline void (*g_end_process_hook)(int) = nullptr; +/// (set by vk_replay.cpp, lab: RX_VK_LAB_AB's knob norec) true: this hold of the replay's lock makes its calls directly, with no +/// producer (DrainingMutex::enter), so one run measures the recorder thread against none in its phases +inline bool (*g_direct_hook)() = nullptr; #ifdef RX_REC_TEST_HOOKS inline void (*g_test_idle_locked)() = nullptr; // (tests: called in idle() with sleep_mu held) #endif @@ -463,6 +469,7 @@ inline bool Recorder::start() { // (the producer, its first packet) if (!base) { size_t c = 4096; while (c < g_ring_bytes && c < (size_t(1) << 30)) c <<= 1; + new_wait::NoWait at_once; // ((Triforce, OS-04) no ring: the recorder is off; not a wait at the commit limit) base = static_cast(::operator new(c, std::align_val_t(64), std::nothrow)); if (!base) { broken.store(true); @@ -655,22 +662,26 @@ inline void segments_begin() { boundary(B_PAR); } /// producer; the unlock waits for the recorder to run all it published, so the recorder is idle whenever no thread holds it struct DrainingMutex { std::mutex m; + std::atomic owner{nullptr}; // ((Triforce, THREAD-68) the holder, for the lock census; unset when it is off) void lock() { - m.lock(); + lock_census::lock(m, owner, "replay_mu"); // (m.lock(), as before, unless RX_VK_LOCK_CENSUS counts) enter(); } bool try_lock() { - if (!m.try_lock()) return false; + if (!lock_census::try_lock(m, owner)) return false; enter(); return true; } void unlock() { boundary(B_REPLAY_UNLOCK); t_prod = nullptr; + lock_census::unlocking(owner); // (after the drain: a waiter is behind this thread until here) m.unlock(); } static void enter() { - if (mode() == THREAD) t_prod = &g_rec; + // (the previous holder's unlock waited until the recorder had run all it published: the ring is empty here, so a hold + // with no producer makes its calls directly, in order after them) + if (mode() == THREAD && !(g_direct_hook && g_direct_hook())) t_prod = &g_rec; } }; diff --git a/gpu/vk/vk_reflex.cpp b/gpu/vk/vk_reflex.cpp index 51538c7..c7fbe9f 100644 --- a/gpu/vk/vk_reflex.cpp +++ b/gpu/vk/vk_reflex.cpp @@ -297,7 +297,11 @@ bool sleep_now() { } VkSemaphoreWaitInfo w{VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO}; w.semaphoreCount = 1, w.pSemaphores = &r.wake, w.pValues = &value; - vkWaitSemaphores(dev.device, &w, 100000000ull); // (100 ms at most: a driver that never wakes it does not stop the game) + { + // ((Triforce, 11-22, SYNC-27) the driver's sleep for latency is pacing, not a wait for the GPU's work: not counted) + const ::gpu_wait_kinds::Exclude pacing; + vkWaitSemaphores(dev.device, &w, 100000000ull); // (100 ms at most: a driver that never wakes it does not stop the game) + } std::lock_guard lk(r.mu); if (!on() || generation != r.swapchain_generation || value != r.wake_value) return true; mark(VK_LATENCY_MARKER_SIMULATION_START_NV); diff --git a/gpu/vk/vk_replay.cpp b/gpu/vk/vk_replay.cpp index 053f7ad..7f90776 100644 --- a/gpu/vk/vk_replay.cpp +++ b/gpu/vk/vk_replay.cpp @@ -31,11 +31,16 @@ #include // (RX_VK_SEG_EARLY: ub_grow_mu) #include "vk_dlssg.h" +#include "cpu_count.h" // ((Triforce, 11-22, OS-49) the threads to size by: HOS_SIM_CPU +#include "sim_gpu.h" // ((Triforce, 11-22, OS-49) RX_VK_SIM_GPU_SLOW, a lab switch: a slower GPU, sim_burn +#include "occ_lean.h" // ((Triforce, 11-22, DESC-82) the occlusion proxies' census's line, status_tick +#include "shaders/sim_gpu_burn_comp_spv.h" #include "vk_temporal.h" #include "vk_fsr3_inputs.h" // (the camera jitter while AMD FSR 3.1 upscales) #include "status_line.h" #include "stall_watch.h" // (Triforce: the frame times and the stall lines, status_tick) #include "lab_ab.h" // (Triforce, lab: in-run A/B phases and delay sites, RX_VK_LAB_AB) +#include "ub_batch.h" // ((Triforce, 11-22, SYNC-21) a recording's uniform updates as copies of many regions, RX_VK_UB_BATCH) #include "occ_answer.h" // (Triforce: samples-passed reports from occlusion queries, RX_VK_OCC_QUERY) #include #include "present_hold.h" // (Triforce: the hold of the last picture, do_present) @@ -66,10 +71,16 @@ #include "tail_early.h" // (Triforce: the early tail's rules, Start next frame early) #include "pass_barrier.h" // (Triforce: the pass barrier's game mode, which RX_VK_EVENT_BARRIER follows when unset) #include "replay_prio.h" // (Triforce: the replay's priority while it records, RX_VK_REPLAY_PRIO: ReplayPin) +#include "replay_pin.h" // (Triforce: the soft pin where the cores are of more than one kind, RX_VK_REPLAY_PIN_SOFT: ReplayPin) #include "mesh_ahead.h" // (Triforce: a run of indirect mesh draws, RX_VK_MESH_AHEAD: mesh_ahead_note) #include "gate_wait.h" // (Triforce: a submission that begins with a wait for the CPU's signal, RX_VK_GATE_WAIT_MS: submit_q) +#include "wait_hold.h" // ((Triforce, 11-22, THREAD-11) a fence wait's hold for stopped work: counted, RX_VK_WAIT_HOLD_PAUSE_US: wait) +#include "page_marks.h" // ((Triforce, 11-22, THREAD-25) an invalidate with nothing to wait for, RX_VK_INVALIDATE_QUICK: invalidate_quick) #include "async_rules.h" // (Triforce: the replay thread's rules, RX_VK_ASYNC=1: the capture at a signal, the present's drain) #include "wb_rec_wait.h" // (Triforce, wp20: a wait for a write-back, by its writer's recording: vk_timeline_wait) +#include "readback_late.h" // ((Triforce, 11-22, SYNC-19) ranges read back without their write-back's wait, RX_VK_READBACK_LATE) +#include "settle_gate.h" // ((Triforce, 11-22, SYNC-29) a read looks at the stopped queues only after a new CPU signal, RX_VK_SETTLE_GATE) +#include "release_queue.h" // ((Triforce, 11-22, MEM-85) texture and program releases queued for the replaying thread, RX_VK_RELEASE_QUEUE) #include "frame_trace.h" // (Triforce, wp20: the frame trace's split of each frame, status_tick) #include "segment_plan.h" // (Triforce: the early segments' count and boundaries, RX_VK_SEG_EARLY) @@ -196,12 +207,43 @@ const bool rec_thread_hook = (::mkwvk::rec::g_thread_start_hook = [] { ::mkwvk::logf("vk: RX_VK_RECORDER_PIN=%s: the recorder %s %#llx (reserved %#llx)", pin, ok ? "pinned to" : "NOT pinned, mask", (unsigned long long)m, (unsigned long long)reserved); }, true); +// (Triforce, lab) RX_VK_LAB_AB's knob norec: in its phases the replay makes its calls directly with the recorder thread on, what the +// recorder gives the frame measured in one run (vk_recorder.h DrainingMutex::enter) +const bool rec_direct_hook = (::mkwvk::rec::g_direct_hook = [] { return lab_ab::on("norec"); }, true); +// ((Triforce, 11-22, the performance audit's THREAD-68) the lock census's two calls that need windows.h (lock_census.h has none): a +// thread's own description, which is how the census names waiters and holders, and the name of its reporting thread +const bool lock_census_hooks = (lock_census::g_thread_name = [](char* out, size_t cap) { +#if defined(_WIN32) + PWSTR w = nullptr; + if (SUCCEEDED(GetThreadDescription(GetCurrentThread(), &w)) && w) { + WideCharToMultiByte(CP_UTF8, 0, w, -1, out, (int)cap - 1, nullptr, nullptr); + out[cap - 1] = 0; + LocalFree(w); + } +#else + (void)out, (void)cap; +#endif +}, lock_census::g_report_start = [] { SetThreadDescription(GetCurrentThread(), L"mkw_vk lock census"); }, true); // (RX_VK_BATCH: see queue_flush_locked) VkSemaphore batch_tl = VK_NULL_HANDLE; uint64_t batch_next = 0, batch_flushed = 0; struct { std::vector cbs; uint64_t signal = 0; bool any_signal = false; uint64_t rec = 0; size_t recs = 0; } pend; // (rec: the recordings' timeline value the held ones reach; recs: how many recordings, RX_VK_BATCH=parts) +// ((Triforce) RX_VK_PEND_HELD, on by default: whether pend holds anything, raised under dev.present_mu after a submit is held and +// lowered by queue_flush_locked once the batch went out. queue_flush_for and queue_flush_poll look at it before the lock and +// return when nothing is held (always, with batching off): every fence wait and progress poll took dev.present_mu to find +// nothing there, behind a present or a late page's fix waiting on the GPU. And a held submit's recording is registered for the +// waiters (value_rec, fin_rec) only once it is in pend with the flag raised: a waiter that finds it then finds it held and +// flushes it. Before, the registration came first, and a waiter could flush nothing and wait for the held signal until the +// next poll, present or cap. RX_VK_PEND_HELD=0: the lock every time and the registration before it, as before; in-run A/B: +// RX_VK_LAB_AB's knob nopendheld) +std::atomic pend_held{false}; +std::atomic st_flush_lockfree{0}; // (flushes that found nothing held without the lock: the batched submits' line) +bool pend_held_on() { + static const bool env = !(getenv("RX_VK_PEND_HELD") && atoi(getenv("RX_VK_PEND_HELD")) == 0); + return env && !lab_ab::on("nopendheld"); +} int64_t st_flushes, st_batched_cbs; -enum { FR_OTHER, FR_POLL, FR_WAIT, FR_SLOT, FR_PRESENT, FR_CAP, FR_VALUE, FR_COUNT }; +enum { FR_OTHER, FR_POLL, FR_WAIT, FR_SLOT, FR_PRESENT, FR_CAP, FR_VALUE, FR_IDLE, FR_COUNT }; // (FR_IDLE: (Triforce, 11-22, SYNC-09)) int64_t st_reason[FR_COUNT]; int flush_reason = FR_OTHER; std::chrono::steady_clock::time_point pend_since; @@ -214,6 +256,9 @@ struct StallCounts { std::atomic wb_unsent{0}, wb_unsent_ns{0}; // (those whose value was not on the queue yet when the wait began) std::atomic wb_held{0}, wb_held_ns{0}; // ((Triforce, wp20) those that went on after the writer's recording was done, // and that part of them: the timeline held, wb_rec_wait.h) + // ((Triforce, 11-22, the performance audit's SYNC-122) that part by caller (wb_rec_wait.h Caller): a capture's take, a read + // of a page put off, a binding of one, and a readback's fallback, which is in none of the counts above) + std::atomic wb_held_by_ns[::mkwvk::wb_rec_wait::kCallers]{}; std::atomic qs_bg{0}, qs_bg_ns{0}; // ((Triforce) queue submits made while the background thread imported, their time) std::atomic wb_ahead_max{0}; // (since the stall line's last look: taken with exchange) std::atomic queue_top{0}; // (dev.present_mu: written; read anywhere) @@ -308,6 +353,22 @@ barrier_rules::Batch batch_way() { bool batch_on() { return batch_way() != barrier_rules::Batch::Off; } +/// ((Triforce, 11-22, the performance audit's SYNC-09, after Source A's; barrier_rules.h batch_idle) RX_VK_BATCH_IDLE=1: a +/// submit that joins the held batch sends it at once while the GPU is idle. OFF unless set, on every driver +bool batch_idle_on() { + static const bool on = [] { + const bool v = barrier_rules::batch_idle_on(getenv("RX_VK_BATCH_IDLE")); + if (v) logf("vk: RX_VK_BATCH_IDLE=1: a submit that joins the held batch sends it at once when the GPU has finished every " + "batch sent (with batched submits, RX_VK_BATCH; nothing is held without them)"); + return v; + }(); + return on; +} +/// The GPU has finished every batch sent: the batches' timeline has reached the last one flushed (dev.present_mu held) +bool batch_gpu_idle() { + uint64_t done = 0; + return vkGetSemaphoreCounterValue(dev.device, batch_tl, &done) == VK_SUCCESS && done >= batch_flushed; +} thread_local int submit_depth = 0; // ((Triforce) RX_VK_BATCH=parts: inside submit_job, the replay of a game's submit) // ((Triforce) RX_VK_SPLIT_BARRIER, maybe_split: how the parts' splits went, logged every 3,600 splits; and the command buffer of the // last part a pass went on into, for whether its uploads (which have full barriers of their own) ordered it after the part before) @@ -404,6 +465,7 @@ void queue_flush_locked() { // (dev.present_mu held) st_reason[flush_reason]++; flush_reason = FR_OTHER; pend.cbs.clear(), pend.any_signal = false, pend.signal = 0, pend.rec = 0, pend.recs = 0; + pend_held.store(false, std::memory_order_release); // ((Triforce) RX_VK_PEND_HELD: nothing held now) } void queue_flush() { std::lock_guard lk(dev.present_mu); @@ -416,6 +478,10 @@ void device_wait_idle_locked() { // (dev.present_mu held) } namespace { void queue_flush_for(int why) { + if (!pend_held.load(std::memory_order_acquire) && pend_held_on()) { // ((Triforce) RX_VK_PEND_HELD: nothing held, no lock) + st_flush_lockfree.fetch_add(1, std::memory_order_relaxed); + return; + } std::lock_guard lk(dev.present_mu); if (pend.cbs.empty()) return; flush_reason = why; @@ -427,6 +493,10 @@ void queue_flush_for(int why) { // 413, 528) void queue_flush_poll() { static const int64_t us = getenv("RX_VK_BATCH_POLL_US") ? atoll(getenv("RX_VK_BATCH_POLL_US")) : 2000; + if (!pend_held.load(std::memory_order_acquire) && pend_held_on()) { // ((Triforce) RX_VK_PEND_HELD: nothing held, no lock) + st_flush_lockfree.fetch_add(1, std::memory_order_relaxed); + return; + } std::lock_guard lk(dev.present_mu); if (pend.cbs.empty()) return; if (us > 0 && std::chrono::steady_clock::now() - pend_since < std::chrono::microseconds(us)) return; @@ -459,16 +529,34 @@ bool rb_fix(int which) { } // namespace uint64_t vk_timeline_completed() { return timeline ? completed() : 0; } /// ((Triforce, wp20, wb_rec_wait.h) RX_VK_WB_WAIT_REC=1: a wait for a write-back ends once its writer's recording is done -static bool wb_wait_rec_on() { - static const bool v = [] { - const bool on = ::mkwvk::wb_rec_wait::on(getenv("RX_VK_WB_WAIT_REC")); - if (on) logf("vk: RX_VK_WB_WAIT_REC=1: a wait for a write-back still on the GPU ends once the recording its submission was " - "finished in is done, not when the submissions' timeline reaches it"); - return on; +/// ((Triforce, 11-22, the performance audit's SYNC-122, after Source B's) a mode: 2 is the check, which ends no wait earlier +/// and has a readback's fallback count what 1 would not have waited (readback_fallback_wait); any other text is off) +static int wb_wait_rec_mode() { + static const int v = [] { + namespace wr = ::mkwvk::wb_rec_wait; + const char* e = getenv("RX_VK_WB_WAIT_REC"); + const int m = wr::mode(e); + if (m == wr::kOn) + logf("vk: RX_VK_WB_WAIT_REC=1: a wait for a write-back still on the GPU ends once the recording its submission was " + "finished in is done, not when the submissions' timeline reaches it"); + else if (m == wr::kCheck) + logf("vk: RX_VK_WB_WAIT_REC=2: the check: no wait ends earlier; a readback whose write-back is not recorded yet waits " + "in 1 ms slices and counts its part after the writer's recording was done"); + else if (e && *e && strcmp(e, "0")) + logf("vk: RX_VK_WB_WAIT_REC=%s is not 0, 1 or 2: off", e); + return m; }(); return v; } +/// ((Triforce, 11-22, SYNC-122) RX_VK_TL_REC=2: the old lookup of RX_VK_TL_REC, value_rec's recording (a submission's first +/// part unless RX_VK_REC_LAST: before its burst's write-backs are recorded). Any other number that turns RX_VK_TL_REC on, and +/// the lab knob tlrec, end the wait at the recording the submission was finished in, as RX_VK_WB_WAIT_REC=1 does +static bool tl_rec_first_part_on() { + static const bool v = ::mkwvk::wb_rec_wait::tl_rec_first_part(getenv("RX_VK_TL_REC")); + return v; +} static thread_local int64_t wb_held_last_ns = 0; // ((Triforce, wp20) this thread's last wait's held part: vk_timeline_wait_held_last) +static thread_local int wb_wait_who = ::mkwvk::wb_rec_wait::kTake; // ((Triforce, 11-22, SYNC-122) the wait's caller: vk_timeline_wait_from) int64_t vk_timeline_wait_held_last() { return wb_held_last_ns; } void vk_timeline_wait(uint64_t v) { // (at most 250 ms: a submission held back behind the caller's thread goes on without it) wb_held_last_ns = 0; @@ -479,7 +567,8 @@ void vk_timeline_wait(uint64_t v) { // (at most 250 ms: a submission held back // (Triforce, the stall line: a value past the queue's top was not handed to the GPU yet (a submission the event ordering // holds back, or one not recorded yet): its wait is the replay's, not the GPU's work) const bool unsent = v > stall_ct.queue_top.load(std::memory_order_relaxed); - if (rb_fix(1)) { // (upstream #2458: where the recording that carries the value is known, its own timeline) + // ((Triforce, 11-22, SYNC-122) only with RX_VK_TL_REC=2 now: value_rec's recording is a first part) + if (rb_fix(1) && tl_rec_first_part_on()) { // (upstream #2458: where the recording that carries the value is known, its own timeline) int ok = 0; if (vk_ft_on()) vk_ft("gpuwait> value %llu on the recordings timeline (done %llu)", (unsigned long long)v, (unsigned long long)done0); if (wait_recorded(v, 250000000ull, &ok)) { @@ -507,6 +596,9 @@ void vk_timeline_wait(uint64_t v) { // (at most 250 ms: a submission held back std::lock_guard mk(rec_map_mu); seq = fin_rec.of(v); } + // ((Triforce) RX_VK_PEND_HELD: a held submit's recording is registered after the flush above can have looked: found now, it + // may be held, so flush again (nothing held: no lock) + if (seq && pend_held_on()) queue_flush_for(FR_VALUE); bool rec_done = false; int64_t rec_ns = 0; if (seq) { @@ -517,7 +609,9 @@ void vk_timeline_wait(uint64_t v) { // (at most 250 ms: a submission held back rec_done = vkWaitSemaphores(dev.device, &ri, wr::kBoundNs) == VK_SUCCESS; rec_ns = std::chrono::duration_cast(std::chrono::steady_clock::now() - t0).count(); } - if (!wr::ends_at_recording(wb_wait_rec_on(), seq != 0, rec_done)) { + // ((Triforce, 11-22, SYNC-122) RX_VK_TL_REC and its knob end the wait here too, at the recording that finished v) + const bool ends_here = wr::ends_by_switch(wb_wait_rec_mode(), rb_fix(1), tl_rec_first_part_on()); + if (!wr::ends_at_recording(ends_here, seq != 0, rec_done)) { VkSemaphoreWaitInfo wi{VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO}; wi.semaphoreCount = 1; wi.pSemaphores = &timeline; @@ -536,6 +630,7 @@ void vk_timeline_wait(uint64_t v) { // (at most 250 ms: a submission held back if (unsent) stall_ct.wb_unsent.fetch_add(1, std::memory_order_relaxed), stall_ct.wb_unsent_ns.fetch_add(waited_ns, std::memory_order_relaxed); if (const int64_t held = wr::held_ns(seq != 0, rec_done, rec_ns, waited_ns)) { // ((Triforce, wp20) the timeline held) stall_ct.wb_held.fetch_add(1, std::memory_order_relaxed), stall_ct.wb_held_ns.fetch_add(held, std::memory_order_relaxed); + stall_ct.wb_held_by_ns[wb_wait_who].fetch_add(held, std::memory_order_relaxed); // ((Triforce, 11-22, SYNC-122) by caller) wb_held_last_ns = held; } const uint64_t ahead = v - done0; @@ -549,6 +644,13 @@ void vk_timeline_wait(uint64_t v) { // (at most 250 ms: a submission held back } } } +/// ((Triforce, 11-22, SYNC-122) vk_timeline_wait for a caller other than a capture's take (wb_rec_wait.h Caller: a read or a +/// binding of a page put off): the same wait, its held part counted under that caller +void vk_timeline_wait_from(uint64_t v, int who) { + wb_wait_who = who >= 0 && who < ::mkwvk::wb_rec_wait::kCallers ? who : ::mkwvk::wb_rec_wait::kTake; + vk_timeline_wait(v); + wb_wait_who = ::mkwvk::wb_rec_wait::kTake; +} namespace { int64_t st_wait_ns; // (time the replay waited for a recording slot: the GPU behind) @@ -568,6 +670,129 @@ int64_t st_gpu_up_ns, st_gpu_up_max_ns; // (Triforce, 11-15) the parallel replay's segments have their timestamps too (they had none: in the owner's test8 the GPU // time line read 2.2 to 2.8 ms a frame while the helpers recorded, 6.4 without them), and their share is counted apart int64_t st_gpu_seg_ns; + +// ---- (Triforce, 11-22, the performance audit's OS-49, after Source B's) RX_VK_SIM_GPU_SLOW=, a lab switch: a slower +// GPU (sim_gpu.h). With it the timestamp pool has three a recording: its start, its work's end, its end; the replay's own +// recordings (not a parallel replay's segments) end with a burn between the last two, planned from the time owed, and the +// collection carries what it measured. A line every 10 s. Unset: two a recording, nothing else of it runs ---- +struct SimGpu { + bool tried = false, ready = false; + VkPipeline pipe = VK_NULL_HANDLE; + VkPipelineLayout layout = VK_NULL_HANDLE; + VkDescriptorSetLayout set_layout = VK_NULL_HANDLE; + VkDescriptorPool pool = VK_NULL_HANDLE; + VkDescriptorSet set = VK_NULL_HANDLE; + VkBuffer sink = VK_NULL_HANDLE; + VmaAllocation sink_alloc = nullptr; + int64_t debt = 0, in_flight = 0; + double ns_per_loop = sim_gpu::kFirstNsPerLoop; + int64_t plan[kRecordings] = {}; + uint32_t loops[kRecordings] = {}; + double st_work = 0, st_added = 0, st_dropped = 0; // (ns since the last line) + int64_t st_burns = 0; + std::chrono::steady_clock::time_point told{}; +} slow; +std::mutex slow_mu; +double sim_gpu_factor() { + static const double f = [] { + const double v = sim_gpu::factor_of(getenv("RX_VK_SIM_GPU_SLOW")); + if (v) + logf("vk: a slower GPU simulated (RX_VK_SIM_GPU_SLOW=%.2f, a lab switch): %.2f ms of burn for each ms of the recordings' " + "work, inside their measured GPU time", v, v - 1.0); + return v; + }(); + return f; +} +int ts_count() { // (timestamps a recording in ts_pool: 2, or 3 with the switch; the same for the run) + static const int n = sim_gpu_factor() ? 3 : 2; + return n; +} +/// (slow_mu held) the burn's pipeline, its sink and its descriptor set, made once +bool sim_ready() { + if (slow.tried) return slow.ready; + slow.tried = true; + const VkPushConstantRange pr{VK_SHADER_STAGE_COMPUTE_BIT, 0, 8}; + VkDescriptorSetLayoutBinding b{0, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr}; + VkDescriptorSetLayoutCreateInfo sli{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO}; + sli.bindingCount = 1, sli.pBindings = &b; + VkDescriptorPoolSize ps{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1}; + VkDescriptorPoolCreateInfo dpi{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; + dpi.maxSets = 1, dpi.poolSizeCount = 1, dpi.pPoolSizes = &ps; + if (vkCreateDescriptorSetLayout(dev.device, &sli, nullptr, &slow.set_layout) == VK_SUCCESS && + vkCreateDescriptorPool(dev.device, &dpi, nullptr, &slow.pool) == VK_SUCCESS && + create_buffer(16, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, false, slow.sink, slow.sink_alloc, nullptr)) { + VkDescriptorSetAllocateInfo ai{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO}; + ai.descriptorPool = slow.pool, ai.descriptorSetCount = 1, ai.pSetLayouts = &slow.set_layout; + VkPipelineLayoutCreateInfo li{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; + li.setLayoutCount = 1, li.pSetLayouts = &slow.set_layout, li.pushConstantRangeCount = 1, li.pPushConstantRanges = ≺ + VkShaderModule m = VK_NULL_HANDLE; + VkShaderModuleCreateInfo mi{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO}; + mi.codeSize = sizeof SIM_GPU_BURN_COMP_SPV, mi.pCode = SIM_GPU_BURN_COMP_SPV; + if (vkAllocateDescriptorSets(dev.device, &ai, &slow.set) == VK_SUCCESS && + vkCreatePipelineLayout(dev.device, &li, nullptr, &slow.layout) == VK_SUCCESS && + vkCreateShaderModule(dev.device, &mi, nullptr, &m) == VK_SUCCESS) { + VkDescriptorBufferInfo bi{slow.sink, 0, 16}; + VkWriteDescriptorSet w{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET}; + w.dstSet = slow.set, w.dstBinding = 0, w.descriptorCount = 1, w.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + w.pBufferInfo = &bi; + vkUpdateDescriptorSets(dev.device, 1, &w, 0, nullptr); + VkComputePipelineCreateInfo ci{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO}; + ci.stage.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO; + ci.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT; + ci.stage.module = m; + ci.stage.pName = "main"; + ci.layout = slow.layout; + if (vkCreateComputePipelines(dev.device, VK_NULL_HANDLE, 1, &ci, nullptr, &slow.pipe) != VK_SUCCESS) slow.pipe = VK_NULL_HANDLE; + vkDestroyShaderModule(dev.device, m, nullptr); + } + } + slow.ready = slow.pipe != VK_NULL_HANDLE; + if (!slow.ready) logf("vk: RX_VK_SIM_GPU_SLOW: the burn's pipeline was not made: no GPU time is added"); + return slow.ready; +} +/// (gpu_time_end, with the switch) the recording's work ends here: its middle timestamp, then, in the replay's own recordings, +/// the burn the time owed asks for, between full barriers +void sim_burn(Recording& r) { + vkCmdWriteTimestamp2(r.cmd, VK_PIPELINE_STAGE_2_BOTTOM_OF_PIPE_BIT, ts_pool, 3 * r.slot + 1); + std::lock_guard lk(slow_mu); + slow.plan[r.slot] = 0, slow.loops[r.slot] = 0; + if (r.segment || !sim_ready()) return; + const int64_t ns = sim_gpu::plan_ns(slow.debt, slow.in_flight); + if (!ns) return; + const uint32_t loops = sim_gpu::loops_for(ns, slow.ns_per_loop); + end_pass(r.cmd); + full_barrier(r.cmd, "barrier (RX_VK_SIM_GPU_SLOW's burn)"); + vkCmdBindPipeline(r.cmd, VK_PIPELINE_BIND_POINT_COMPUTE, slow.pipe); + vkCmdBindDescriptorSets(r.cmd, VK_PIPELINE_BIND_POINT_COMPUTE, slow.layout, 0, 1, &slow.set, 0, nullptr); + const struct { uint32_t loops; float seed; } pc{loops, 1.0f + (float)r.slot}; + vkCmdPushConstants(r.cmd, slow.layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, &pc); + vkCmdDispatch(r.cmd, 1, 1, 1); + full_barrier(r.cmd, "barrier (RX_VK_SIM_GPU_SLOW's burn)"); + slow.plan[r.slot] = ns, slow.loops[r.slot] = loops, slow.in_flight += ns, slow.st_burns++; +} +/// (gpu_time_collect, with the switch) a recording's three timestamps (null: not read): its work and its burn into the debt +void sim_collect(const Recording& r, const uint64_t* t) { + std::lock_guard lk(slow_mu); + const int64_t plan = slow.plan[r.slot]; + slow.in_flight -= plan, slow.plan[r.slot] = 0; + if (!t || t[1] < t[0] || t[2] < t[1]) return; + const double period = dev.props.limits.timestampPeriod; + const int64_t work = (int64_t)((t[1] - t[0]) * period), added = (int64_t)((t[2] - t[1]) * period); + const double o = sim_gpu::owed(slow.debt, work, added, sim_gpu_factor()); + slow.debt = sim_gpu::held(o); + if (plan) slow.ns_per_loop = sim_gpu::calibrated(slow.ns_per_loop, added, slow.loops[r.slot]); + slow.st_work += (double)work, slow.st_added += (double)added, slow.st_dropped += o - (double)slow.debt; + const auto now = std::chrono::steady_clock::now(); + if (slow.told.time_since_epoch().count() == 0) slow.told = now; + if (now - slow.told >= std::chrono::seconds(10)) { + logf("vk: RX_VK_SIM_GPU_SLOW (10 s): %.0f ms of work, %.0f ms added in %lld burns: x%.2f of the asked x%.2f; %.0f ms " + "dropped at the debt's bounds, %.1f ms owed; a loop %.2f ns", slow.st_work / 1e6, slow.st_added / 1e6, + (long long)slow.st_burns, slow.st_work > 0 ? (slow.st_work + slow.st_added) / slow.st_work : 0.0, sim_gpu_factor(), + slow.st_dropped / 1e6, slow.debt / 1e6, slow.ns_per_loop); + slow.st_work = slow.st_added = slow.st_dropped = 0, slow.st_burns = 0, slow.told = now; + } +} + void gpu_time_collect(Recording& r) { if (ts_up_pool && ts_up_pending[r.slot]) { ts_up_pending[r.slot] = false; @@ -582,11 +807,14 @@ void gpu_time_collect(Recording& r) { } if (!ts_pool || !ts_pending[r.slot]) return; ts_pending[r.slot] = false; - uint64_t t[2] = {0, 0}; - if (vkGetQueryPoolResults(dev.device, ts_pool, 2 * r.slot, 2, sizeof t, t, 8, VK_QUERY_RESULT_64_BIT) != VK_SUCCESS) - return; - if (t[1] <= t[0]) return; - const int64_t ns = (int64_t)((t[1] - t[0]) * (double)dev.props.limits.timestampPeriod); + // ((Triforce, 11-22, OS-49) ts_count: three with RX_VK_SIM_GPU_SLOW, the last the recording's end: sim_collect) + const int tn = ts_count(); + uint64_t t[3] = {0, 0, 0}; + const bool got = vkGetQueryPoolResults(dev.device, ts_pool, tn * r.slot, tn, sizeof t, t, 8, VK_QUERY_RESULT_64_BIT) == VK_SUCCESS; + if (tn == 3) sim_collect(r, got ? t : nullptr); + if (!got) return; + if (t[tn - 1] <= t[0]) return; + const int64_t ns = (int64_t)((t[tn - 1] - t[0]) * (double)dev.props.limits.timestampPeriod); st_gpu_ns += ns; st_gpu_max_ns = std::max(st_gpu_max_ns, ns); if (r.segment) st_gpu_seg_ns += ns; @@ -595,13 +823,14 @@ void gpu_time_begin(Recording& r) { if (!ts_pool) { VkQueryPoolCreateInfo qi{VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO}; qi.queryType = VK_QUERY_TYPE_TIMESTAMP; - qi.queryCount = 2 * kRecordings; + qi.queryCount = ts_count() * kRecordings; if (vkCreateQueryPool(dev.device, &qi, nullptr, &ts_pool) != VK_SUCCESS) ts_pool = VK_NULL_HANDLE; if (!ts_pool) return; + qi.queryCount = 2 * kRecordings; if (vkCreateQueryPool(dev.device, &qi, nullptr, &ts_up_pool) != VK_SUCCESS) ts_up_pool = VK_NULL_HANDLE; } - vkCmdResetQueryPool(r.cmd, ts_pool, 2 * r.slot, 2); - vkCmdWriteTimestamp2(r.cmd, VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT, ts_pool, 2 * r.slot); + vkCmdResetQueryPool(r.cmd, ts_pool, ts_count() * r.slot, ts_count()); + vkCmdWriteTimestamp2(r.cmd, VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT, ts_pool, ts_count() * r.slot); if (ts_up_pool) { // (in the upload command buffer, begun just before) vkCmdResetQueryPool(r.upload, ts_up_pool, 2 * r.slot, 2); vkCmdWriteTimestamp2(r.upload, VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT, ts_up_pool, 2 * r.slot); @@ -610,7 +839,8 @@ void gpu_time_begin(Recording& r) { /// uploads: the upload command buffer goes to the queue (submit_recording); one that does not is never read void gpu_time_end(Recording& r, bool uploads) { if (!ts_pool) return; - vkCmdWriteTimestamp2(r.cmd, VK_PIPELINE_STAGE_2_BOTTOM_OF_PIPE_BIT, ts_pool, 2 * r.slot + 1); + if (ts_count() == 3) sim_burn(r); // ((Triforce, 11-22, OS-49) RX_VK_SIM_GPU_SLOW: its work's end, then the burn) + vkCmdWriteTimestamp2(r.cmd, VK_PIPELINE_STAGE_2_BOTTOM_OF_PIPE_BIT, ts_pool, ts_count() * r.slot + ts_count() - 1); ts_pending[r.slot] = true; if (ts_up_pool) { vkCmdWriteTimestamp2(r.upload, VK_PIPELINE_STAGE_2_BOTTOM_OF_PIPE_BIT, ts_up_pool, 2 * r.slot + 1); @@ -664,6 +894,14 @@ void gpu_time_end(Recording& r, bool uploads) { // the draws and clears it gates (occ_cond, occ::cond_decide). RX_VK_OCC_COND=0: a condition on a samples-passed word // always renders (the counts are still written). In the lab knob's off phases the answer is the one from memory, as // before. RX_VK_OCC_LOG_COND=1: one such condition in 499 logged (300 lines), with both words and their sources. +// (11-22) A draw that a written count's condition turned off, between a reset and its report, gates that count +// (occ_gate_note, occ::Counter::gate): its 0 is the skip's, not the object's. The game's CPU reads a count after its +// frame wait and draws a hidden object's proxy instead of the object; most counts land after the CPU has read the +// 4096 (the line: written after the next present, recordings read at a recording's begin, not at a game's wait), so the +// game keeps drawing the object under the condition, the condition reads the skip's 0 and the object stays hidden +// (tickets: items and trees in the Forest Temple, black interiors; RX_VK_OCC_QUERY=0 showed them). The line counts the +// gated counts and the conditions that read one; RX_VK_OCC_GATE=1: such a condition renders (cond_decide), the +// count is still written for the CPU. Off by default. // The writes, the coherence: a count is written as the replay writes a report (the guest page marked written, the // write-back diff's shadows told, memory_cpu_owned: the word is cut out of the mirrors' write-backs, so a mirror's stale // copy never lands over it; with the lazy write-back such a page is never owed, and a CPU write to an owed page pays first), @@ -692,11 +930,15 @@ bool occ_host_reset = false; // (hostQueryReset: a block's used queries reset bool occ_env_real = false; // (RX_VK_OCC_QUERY=1: the counts written; else only for reports in the lab knob's phases) bool occ_cond_on = true; // (RX_VK_OCC_COND=0: a render condition on a samples-passed word always renders) bool occ_log_cond = false; // (RX_VK_OCC_LOG_COND=1: one such condition in 499 logged, 300 lines at most) +bool occ_gate_on = true; // (on by default, RX_VK_OCC_GATE=0: off: a condition on a count over a draw a written count skipped renders) std::vector occ_skip; std::mutex occ_mu; // (the sealed slots, the writers, the stats, the sites; a placeholder's write with its entry; the counts' writes) // (the replay thread's, in stream order (the serial replay or the dry pass): the queue state whose render condition a // written count turned off, until its next condition or render enable; the draws and clears it skips, for the line) const NvnState* occ_cond_off = nullptr; +// ((11-22) a parallel replay's segment thread: the queue state the dry pass's condition turned off from a written count, as +// par_conds tells it (2); until its next condition or render enable. The counting it gates: occ_gate_note) +thread_local const NvnState* occ_seg_off = nullptr; std::atomic occ_skips_draw{0}, occ_skips_clear{0}; occ::Writers occ_writers; occ::Stats occ_st; @@ -729,15 +971,19 @@ bool occ_init() { occ_skip = occ::parse_addrs(getenv("RX_VK_OCC_QUERY_SKIP")); occ_cond_on = !getenv("RX_VK_OCC_COND") || atoi(getenv("RX_VK_OCC_COND")) != 0; occ_log_cond = getenv("RX_VK_OCC_LOG_COND") && atoi(getenv("RX_VK_OCC_LOG_COND")) != 0; + // ((Triforce, 11-22) on by default: the lab A/B at the Market, 3+3, 77.6 against 77.1 fps with it (1% low 49.5 against 47.8), + // and a player's invisible trees and drops (Forest Temple) came back with the queries off. RX_VK_OCC_GATE=0: off) + occ_gate_on = !(getenv("RX_VK_OCC_GATE") && atoi(getenv("RX_VK_OCC_GATE")) == 0); logf("vk: occlusion queries (RX_VK_OCC_QUERY%s): samples-passed reports written 4096 at the replay as before, and their " "counts written into the records once the GPU is done%s; %s counts, %d recordings x %u queries, reset %s; done " - "recordings found on %s; %zu report addresses kept fixed (RX_VK_OCC_QUERY_SKIP); render conditions on their words: %s%s", + "recordings found on %s; %zu report addresses kept fixed (RX_VK_OCC_QUERY_SKIP); render conditions on their words: %s%s%s", occ_env_real ? "=1" : ", lab knob occq", occ_env_real ? "" : " (for reports in the knob's phases)", occ_flags ? "precise" : "non-precise (no occlusionQueryPrecise)", kRecordings, kOccPerSlot, occ_host_reset ? "on the host once read" : "in each recording (no hostQueryReset)", rec_timeline ? "the recordings' timeline" : "their fences (no recordings' timeline: the replay thread only)", occ_skip.size(), occ_cond_on ? "a count still to come renders, a written one is read from memory (a 0 skips)" : "always rendered (RX_VK_OCC_COND=0)", + occ_gate_on ? "; RX_VK_OCC_GATE=1: one on a count over a draw a written count skipped renders" : "", occ_log_cond ? "; RX_VK_OCC_LOG_COND=1: one in 499 logged" : ""); return true; } @@ -885,6 +1131,7 @@ void occ_segment_edge(VkCommandBuffer cmd) { occ::Counter& c = occ_ctr(); occ_close(c, cmd); c.segment(); + occ_seg_off = nullptr; } /// A command buffer replayed in parallel (replay_handle, before its dry pass; the replay thread, replay_mu): the replay's /// counting given up, its query ended in cmd (the segments record what follows, a small one in cmd's open pass): a report @@ -923,12 +1170,13 @@ void occ_report_measure(VkCommandBuffer cmd, uint64_t addr, uint64_t pos, bool w std::lock_guard lk(occ_mu); if (o != occ::Counter::kMeasured) occ_writers.settle(addr, pos); // (no count comes: a condition reads memory, as before) switch (o) { - case occ::Counter::kMeasured: occ_st.measured++; break; + case occ::Counter::kMeasured: occ_st.measured++, occ_st.gated += rep.gated ? 1 : 0; break; case occ::Counter::kNoReset: occ_st.no_reset++; break; case occ::Counter::kDouble: occ_st.doubles++; break; case occ::Counter::kIncomplete: occ_st.incomplete++; - (c.why == occ::Counter::kOtherCmd ? occ_st.inc_other_cmd : c.why == occ::Counter::kFull ? occ_st.inc_full : occ_st.inc_skipped)++; + (c.why == occ::Counter::kOtherCmd ? occ_st.inc_other_cmd : c.why == occ::Counter::kFull ? occ_st.inc_full + : c.why == occ::Counter::kColour ? occ_st.inc_colour : occ_st.inc_skipped)++; // ((Triforce, 11-22, GPUWORK-01) kColour) break; } } @@ -940,7 +1188,10 @@ void occ_report_measure(VkCommandBuffer cmd, uint64_t addr, uint64_t pos, bool w void occ_placed(uint64_t addr, uint64_t pos, bool dry, bool flag, uint64_t value) { occ_st.reports++; occ_st.in_segments += dry ? 1 : 0; - occ_distinct.note(addr); + // ((Triforce, 11-21) the distinct addresses, counted only for the log line's "addresses this period", only when asked: + // RX_VK_OCC_DISTINCT=1. A probe into a 1 MB table at every report was 1% of the Presentation Thread; off, the line says 0) + static const bool distinct_on = getenv("RX_VK_OCC_DISTINCT") && atoi(getenv("RX_VK_OCC_DISTINCT")) != 0; + if (distinct_on) occ_distinct.note(addr); occ_sites.reported(addr); bool write = !flag; if (flag) occ_st.flagged++; @@ -969,14 +1220,16 @@ void occ_cond(const NvnState& s, uint64_t addr, int mode, uint64_t& x, uint64_t& const occ::Writers::Src sx = occ_writers.word(addr, now), sy = occ_writers.word(addr + 16, now); if (sx == occ::Writers::kNone && sy == occ::Writers::kNone) return; // (no samples-passed report: as before) const bool active = occ_env_real || lab_ab::on("occq"); - if (active && (sx == occ::Writers::kWrittenSrc || sy == occ::Writers::kWrittenSrc)) { + const bool gated = sx == occ::Writers::kGatedSrc || sy == occ::Writers::kGatedSrc; // (11-22: a skip's count, Counter::gate) + if (active && (sx == occ::Writers::kWrittenSrc || sy == occ::Writers::kWrittenSrc || gated)) { // (read again under occ_mu: a count written between the replay's read and here is what memory holds now) host->gpu_read(addr, &x, 8); host->gpu_read(addr + 16, &y, 8); *enabled = mode == 0 ? x == y : x != y; } - const occ::Cond c = occ::cond_decide(*enabled, sx, sy, active, occ_cond_on); + const occ::Cond c = occ::cond_decide(*enabled, sx, sy, active, occ_cond_on, occ_gate_on); occ_st.cond(c.kind); + occ_st.cond_gated_read += active && gated ? 1 : 0; *enabled = c.enabled; if (c.kind == occ::Cond::kWrittenOff) occ_cond_off = &s; static int64_t seen = 0, logged = 0; @@ -985,6 +1238,7 @@ void occ_cond(const NvnState& s, uint64_t addr, int mode, uint64_t& x, uint64_t& logf("vk: occlusion condition at %#llx mode %d: %llu (%s) vs %llu (%s): %s (present %lld)", (unsigned long long)addr, mode, (unsigned long long)x, occ::src_name(sx), (unsigned long long)y, occ::src_name(sy), c.kind == occ::Cond::kUnknown ? "renders, a count pending" + : c.kind == occ::Cond::kGated ? "renders, a count over a draw a written count skipped (RX_VK_OCC_GATE=1)" : c.kind == occ::Cond::kForced ? "renders (RX_VK_OCC_COND=0)" : c.kind == occ::Cond::kBase ? (c.enabled ? "on, as before (knob occq off)" : "off, as before (knob occq off)") : c.kind == occ::Cond::kWrittenOff ? "SKIPS, from a written count" @@ -999,6 +1253,24 @@ void occ_skip_note(const NvnState& s, bool clear) { if (s.render_enabled || occ_cond_off != &s) return; (clear ? occ_skips_clear : occ_skips_draw).fetch_add(1, std::memory_order_relaxed); } +/// (11-22) A draw the thread recording it passes with rendering off (the serial replay or a segment's thread, not the dry +/// pass): when a written count's condition turned it off, the samples-passed count it is in, if any, is gated (Counter::gate: +/// NVN counts nothing for it, and the count says what the skip did, not what the object shows). worker: a segment's, whose +/// condition the dry pass told it (occ_seg_off) +void occ_gate_note(const NvnState& s, bool worker) { + if (s.render_enabled || s.raster_discard || (worker ? occ_seg_off : occ_cond_off) != &s) return; + occ_ctr().gate(occ_keep.load(std::memory_order_relaxed)); +} +/// ((Triforce, 11-22, the performance audit's GPUWORK-01, after Source A's) RX_VK_OCC_DEPTH_ONLY=1, or the lab's "occdepth" +/// phases: a count is given up at its window's first draw that writes colour (occ_draw_colour). Off unless set: fewer +/// objects are culled with it, and whether the GPU time it saves is worth that is for a measurement to say. And what is +/// counted either way, for the line: the windows that met a colour draw, the colour draws recorded under an open query +/// (in 64s, from each thread), the counts given up +bool occ_depth_only() { + static const bool env = getenv("RX_VK_OCC_DEPTH_ONLY") && atoi(getenv("RX_VK_OCC_DEPTH_ONLY")) != 0; + return env || lab_ab::on("occdepth"); +} +std::atomic occ_colour_windows{0}, occ_colour_open{0}, occ_colour_lost{0}; /// Once a present: the sealed recordings done read; Writers' addresses not reported for occ::kKeepAge presents pruned every /// 60, and the line every 600 presents (and the first 64 report addresses at its 1st and every 10th) void occ_present_tick() { @@ -1022,6 +1294,12 @@ void occ_present_tick() { } else { logf("%s", line); } + // ((Triforce, 11-22, the performance audit's GPUWORK-01) the colour draws in counting windows, a line of their own) + occ_st.colour_windows = occ_colour_windows.exchange(0, std::memory_order_relaxed); + occ_st.colour_open = occ_colour_open.exchange(0, std::memory_order_relaxed); + occ_st.colour_lost = occ_colour_lost.exchange(0, std::memory_order_relaxed); + occ_st.colour_line(line, sizeof line, 600, occ_depth_only()); + logf("%s", line); if (lines++ % 10 == 0) occ_sites.log([](const char* l) { logf("%s", l); }); occ_st = occ::Stats{}; } @@ -1029,6 +1307,24 @@ void occ_present_tick() { bool occ_live = false; // (vk_draw.h: RX_VK_OCC_QUERY's queries run) void occ_draw_enter() { occ_ctr().draw_enter(occ_keep.load(std::memory_order_relaxed)); } void occ_draw_issued() { occ_ctr().draw_issued(occ_keep.load(std::memory_order_relaxed)); } +bool occ_counting_now() { return occ_live && occ_ctr().counting(occ_keep.load(std::memory_order_relaxed)); } // ((Triforce, PIPE-29)) +/// ((Triforce, 11-22, the performance audit's GPUWORK-01, after Source A's) A draw that writes colour, its pass open, before +/// occ_draw_begin (vk_draw.cpp draw). Counted for the line; and with RX_VK_OCC_DEPTH_ONLY the count is given up there: its +/// open query ends before this draw, inside the pass it began in, no query begins again until the next reset +/// (Counter::may_begin), and its report is incomplete, so the record keeps 4096 and what it gates is drawn +void occ_draw_colour(VkCommandBuffer cmd) { + occ::Counter& c = occ_ctr(); + const occ::Counter::Colour m = c.colour_draw(occ_keep.load(std::memory_order_relaxed), occ_depth_only()); + if (m.first) occ_colour_windows.fetch_add(1, std::memory_order_relaxed); + if (m.open) { + static thread_local uint32_t batch = 0; + if (++batch == 64) occ_colour_open.fetch_add(64, std::memory_order_relaxed), batch = 0; + } + if (!m.lose) return; + occ_close(c, cmd); + c.lose_colour(); + occ_colour_lost.fetch_add(1, std::memory_order_relaxed); +} void occ_draw_begin(VkCommandBuffer cmd) { occ::Counter& c = occ_ctr(); const bool keep = occ_keep.load(std::memory_order_relaxed); @@ -1041,6 +1337,7 @@ void occ_draw_begin(VkCommandBuffer cmd) { vkCmdBeginQuery(cmd, occ_pool, (uint32_t)c.slot * kOccPerSlot + (uint32_t)c.open, occ_flags); occ_open_here = true; } +bool occ_counting() { return occ_ctr().open >= 0; } // ((Triforce, 11-22, DESC-82) the census's question, vk_draw.h) void occ_pass_end(VkCommandBuffer cmd) { // (a pass of this thread's: only where this thread began the query open now. The if (occ_open_here) occ_close(occ_ctr(), cmd); // replay's counting is under replay_mu, which a pass's end may not hold) } @@ -1063,6 +1360,9 @@ Recording& begin_recording(uint64_t value) { r.segment = false; r.seg_unbegun = false; vkResetCommandPool(dev.device, r.pool, 0); vkResetCommandPool(dev.device, r.upload_pool, 0); + // ((Triforce, 11-21, the performance audit's REPLAY-04) the slot's descriptor pools reset here, where the recorder has + // nothing held (the resets above waited for it): after the packets below they were a wait at every recording's start) + draw_slot_reset(r.slot); VkCommandBufferBeginInfo bi{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; bi.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; vkBeginCommandBuffer(r.cmd, &bi); @@ -1142,6 +1442,12 @@ void submit_recording(Recording& r, bool signal_value = true, uint64_t signal_at signals[nsig].value = seq; signals[nsig].stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT; nsig++; + } + // ((Triforce) the recording made known to the waiters: before the queue's lock with batching off or RX_VK_PEND_HELD=0, as + // before; else under it, once the submit is held with pend_held raised, or before the submit when it goes straight to the + // queue. rec_map_mu takes no other lock inside it) + const auto register_recording = [&] { + if (!seq) return; std::lock_guard mk(rec_map_mu); // (upstream #2419, behind RX_VK_REC_LAST: a submission recorded in parts maps its value to its LAST part. emplace keeps // the first, and a wait for the submission through wait_recorded comes back after its first ~120 draws) @@ -1159,7 +1465,9 @@ void submit_recording(Recording& r, bool signal_value = true, uint64_t signal_at for (uint64_t v : rec_open_qvalues) whole_val[v & (kWholeSlots - 1)].store(v, std::memory_order_release); rec_open_qvalues.clear(); while (value_rec.size() > 8192) value_rec.erase(value_rec.begin()); - } + }; + const bool batching = batch_on(), registered_late = batching && pend_held_on(); + if (!registered_late) register_recording(); si.signalSemaphoreInfoCount = nsig; si.pSignalSemaphoreInfos = nsig ? signals : nullptr; // (Triforce, RX_VK_OCC_QUERY: once on the queue, its occlusion queries are the readers', by seq. Declared before the lock @@ -1182,6 +1490,8 @@ void submit_recording(Recording& r, bool signal_value = true, uint64_t signal_at if (signal_value) pend.signal = std::max(pend.signal, signal.value), pend.any_signal = true; if (seq) pend.rec = std::max(pend.rec, seq); pend.recs++; + pend_held.store(true, std::memory_order_release); + if (registered_late) register_recording(); // ((Triforce) RX_VK_PEND_HELD: a waiter that finds it finds it held) r.batch = batch_next; r.submitted = true; // (RX_VK_BATCH_MAX=: held at most this many, default 32; RX_VK_BATCH=parts: recordings, default 4) @@ -1189,10 +1499,13 @@ void submit_recording(Recording& r, bool signal_value = true, uint64_t signal_at : way == barrier_rules::Batch::Parts ? 4 : 32; if (barrier_rules::batch_flush(way, hold, way == barrier_rules::Batch::Parts ? pend.recs : pend.cbs.size(), cap)) flush_reason = hold ? FR_CAP : FR_OTHER, queue_flush_locked(); + // ((Triforce, 11-22, SYNC-09) RX_VK_BATCH_IDLE=1: what is still held goes now while the GPU has nothing left) + if (barrier_rules::batch_idle(way, !pend.cbs.empty(), batch_idle_on()) && batch_gpu_idle()) flush_reason = FR_IDLE, queue_flush_locked(); memory_lazy_submitted(); // (Triforce, lazy write-back: on the queue as far as a payment is concerned: it flushes first) return; } } + if (registered_late) register_recording(); // ((Triforce) RX_VK_PEND_HELD: not held, straight to the queue below) PROF(PROF_QSUBMIT); VkSemaphoreSubmitInfo bind_wait; // (Triforce: after the memory of the mirrors' new pieces is bound, vk_memory.cpp) if (memory_mirror_bind_wait(&bind_wait)) si.waitSemaphoreInfoCount = 1, si.pWaitSemaphoreInfos = &bind_wait; @@ -1278,6 +1591,12 @@ std::mutex ev_cpu_mu; std::vector> ev_cpu_pending; std::atomic ev_cpu_any{false}; std::unordered_set cpu_events; // (events the CPU has signalled, nvnEventSignal; ev_cpu_mu) +// ((Triforce, 11-22, the performance audit's SYNC-29, after Source A's; settle_gate.h) For the reads' gate, RX_VK_SETTLE_GATE: +// the CPU's signals counted (event_signal_queued, after the signal is queued), the count the last finished look at the stopped +// queues began with (burst_begin notes it, burst_end publishes it after finish_recording), and whether a stopped queue waits +// on an event nothing has set, whose value is read from guest memory (update_blocked): a plain store there is seen by no count +std::atomic ev_cpu_signals{0}, ev_cpu_settled{0}; +std::atomic ev_wait_in_memory{false}; void ev_cpu_apply() { if (!ev_cpu_any.load(std::memory_order_acquire)) return; std::lock_guard lk(ev_cpu_mu); @@ -1641,7 +1960,10 @@ void ub_copy_keep(uint64_t addr, uint64_t n, const uint8_t* base, size_t first, c.on = true, c.base = base, c.addr = addr, c.n = n, c.at = at, c.first = first, c.lim = lim; c.rec = rec_value_now, c.view = view, c.epoch = epoch; } -uint64_t ub_overlay(uint64_t addr, uint64_t n, uint8_t* base, const std::function& alloc) { +// ((Triforce, 11-22, the performance audit's DESC-06, after Source A's) alloc(n, align): the caller's allocator as a plain +// function with the alignment it wants (vk_draw.cpp's stream_alloc). It was a std::function, made of a capturing lambda at +// every uniform buffer of every draw) +uint64_t ub_overlay(uint64_t addr, uint64_t n, uint8_t* base, uint64_t (*alloc)(uint64_t, uint64_t), uint64_t align) { UbReadGuard ub_guard; // (RX_VK_SEG_EARLY) // (a worker: those before its place in the stream; a background thread: none, not even the list's size) const size_t lim = ub_limit ? std::min(ub_limit, ub_updates.size()) : 0; @@ -1662,7 +1984,7 @@ uint64_t ub_overlay(uint64_t addr, uint64_t n, uint8_t* base, const std::functio } } if (!any) return UINT64_MAX; - const uint64_t at = alloc(n); + const uint64_t at = alloc(n, align); if (at == UINT64_MAX) return UINT64_MAX; uint8_t* p = base + at; if (!memory_read_as_submitted(addr, p, n)) std::memset(p, 0, n); @@ -1716,16 +2038,119 @@ namespace { /// The recording's uniform updates into guest memory (after its draws: the ones that came after an update read /// their overlay copies). ((Triforce) lim: those before that entry only, the early tail's cut in a parallel replay's recording /// after its segments, whose dry pass listed the tail's updates already: tail_begin) +// ((Triforce, 11-22, the performance audit's SYNC-21, after Source A's; RX_VK_UB_BATCH: unset or 0 as before, 1 the batch, +// 2 as before with the overlaps counted; off until it is measured) ub_batch.h says what the batch is. The counts, for the +// line every 600 flushes that wrote any update with the switch at 1 or 2 or in the lab knob's phases: the updates, the +// runs ended by an overlap and by another destination buffer, the copies and their regions, the pieces written the old +// way because the stream was full, and the pieces outside the pools (on the CPU, as before) +int ub_batch_mode() { + static const int mode = [] { + const char* e = getenv("RX_VK_UB_BATCH"); + const int v = e && *e ? atoi(e) : 0; + if (v == 1) logf("vk: a recording's uniform updates as copies of many regions from its stream, a barrier where they overlap (RX_VK_UB_BATCH=1)"); + if (v == 2) logf("vk: a recording's uniform updates as before, their overlaps counted (RX_VK_UB_BATCH=2)"); + return v == 1 || v == 2 ? v : 0; + }(); + return mode; +} +std::atomic st_ubb_flushes{0}, st_ubb_updates{0}, st_ubb_overlaps{0}, st_ubb_imports{0}, st_ubb_copies{0}, st_ubb_regions{0}, + st_ubb_fallbacks{0}, st_ubb_cpu{0}; +void ub_batch_line(bool batched) { + if (st_ubb_flushes.fetch_add(1, std::memory_order_relaxed) % 600 != 599) return; + logf("vk: uniform updates (600 flushes, %s): %lld updates; %lld times one wrote bytes an earlier one of its run had written%s, " + "%lld times the destination buffer changed; %lld copies of %lld regions, %lld pieces written singly (the stream full), " + "%lld on the CPU (outside the pools) (RX_VK_UB_BATCH)", batched ? "batched" : "as before, counted", + (long long)st_ubb_updates.exchange(0), (long long)st_ubb_overlaps.exchange(0), batched ? " (a barrier each)" : "", + (long long)st_ubb_imports.exchange(0), (long long)st_ubb_copies.exchange(0), (long long)st_ubb_regions.exchange(0), + (long long)st_ubb_fallbacks.exchange(0), (long long)st_ubb_cpu.exchange(0)); +} +/// The updates [first, end) batched (RX_VK_UB_BATCH=1): between the flush's two barriers, the same pieces gpu_write makes +/// and the same binding of each, in stream order. A piece's bytes go into the recording's stream and its place into the +/// run's copy; a run ends as ub_batch.h says. A piece outside the pools is written on the CPU, as before; a piece that is +/// not 4-byte aligned is a region like any other (a copy has no such rule), no longer a CPU write at record time. With the +/// stream full, what is batched is copied and the piece is written as gpu_write writes it +void flush_ub_batched(VkCommandBuffer cmd, size_t first, size_t end, GprofSpan& gp) { + static thread_local ub_batch::Run run; + static thread_local std::vector regions; + run.clear(), regions.clear(); + int64_t overlaps = 0, imports = 0, copies = 0, copied = 0, fallbacks = 0, on_cpu = 0; + const auto copy = [&] { // (the regions so far, as one copy) + if (!regions.empty()) { + vkCmdCopyBuffer(cmd, run.src, run.dst, (uint32_t)regions.size(), regions.data()); + copies++, copied += (int64_t)regions.size(); + } + regions.clear(), run.copied(); + }; + for (size_t i = first; i < end; i++) { + const UbUpdate& u = ub_updates[i]; + gp.bytes += u.bytes.size(); + uint64_t gpu = u.dst, n = u.bytes.size(); + const uint8_t* d = u.bytes.data(); + while (n) { + const uint64_t k = std::min(n, 65536); + const BufferRef b = gpu_buffer_bind(gpu, k, true); + if (!b) { + guest_cpu_write(gpu, d, k), on_cpu++; + } else { + const BufferRef s = stream_copy(d, k); + const ub_batch::Before what = run.before(s.buffer, b.buffer, b.offset, b.offset + k); + if (what == ub_batch::RUN_OVERLAP || what == ub_batch::RUN_IMPORT) { + copy(); + full_barrier(cmd); // (the next run's writes come after this one's: the same bytes, or the same memory through another buffer) + (what == ub_batch::RUN_OVERLAP ? overlaps : imports)++; + run.begin(b.buffer); + } else if (what == ub_batch::COPY || !s) { + copy(); // (the bytes are in another stream buffer, or this piece is written singly below: what is batched goes first) + } + if (!run.any) run.begin(b.buffer); + if (s) { + regions.push_back({s.offset, b.offset, k}); + run.add(s.buffer, b.offset, b.offset + k); + } else { // (no room in the stream: as gpu_write writes it) + fallbacks++; + if ((b.offset & 3) || (k & 3)) guest_cpu_write(gpu, d, k); + else vkCmdUpdateBuffer(cmd, b.buffer, b.offset, k, d), run.wrote(b.offset, b.offset + k); + } + } + gpu += k, d += k, n -= k; + } + } + copy(); + st_ubb_updates.fetch_add((int64_t)(end - first), std::memory_order_relaxed); + st_ubb_overlaps.fetch_add(overlaps, std::memory_order_relaxed), st_ubb_imports.fetch_add(imports, std::memory_order_relaxed); + st_ubb_copies.fetch_add(copies, std::memory_order_relaxed), st_ubb_regions.fetch_add(copied, std::memory_order_relaxed); + st_ubb_fallbacks.fetch_add(fallbacks, std::memory_order_relaxed), st_ubb_cpu.fetch_add(on_cpu, std::memory_order_relaxed); +} void flush_ub_updates(VkCommandBuffer cmd, size_t lim = SIZE_MAX) { UbReadGuard ub_guard; // (RX_VK_SEG_EARLY) const size_t end = std::min(lim, ub_updates.size()); if (ub_first_of_recording >= end) return; GprofSpan gp(cmd, "ubupd (the stream's uniform updates -> guest memory, with its barriers)"); gp.items = (int)(end - ub_first_of_recording); + // ((Triforce, 11-22, SYNC-21) the batch with the switch at 1, or in the lab knob ubbatch's phases) + const int mode = ub_batch_mode(); + const bool batched = mode == 1 || lab_ab::on("ubbatch"); full_barrier(cmd); - for (size_t i = ub_first_of_recording; i < end; i++) - gpu_write(cmd, ub_updates[i].dst, ub_updates[i].bytes.data(), ub_updates[i].bytes.size()), gp.bytes += ub_updates[i].bytes.size(); + if (batched) { + flush_ub_batched(cmd, ub_first_of_recording, end, gp); + } else { + if (mode == 2) { // (counted by guest address: what a batch's runs would end at, the destination buffers not known here) + static thread_local ub_batch::Run seen; + seen.clear(), seen.begin(0); + int64_t overlaps = 0; + for (size_t i = ub_first_of_recording; i < end; i++) { + const uint64_t at = ub_updates[i].dst, to = at + ub_updates[i].bytes.size(); + if (seen.before(0, 0, at, to) == ub_batch::RUN_OVERLAP) overlaps++, seen.begin(0); + seen.add(0, at, to); + } + st_ubb_updates.fetch_add((int64_t)(end - ub_first_of_recording), std::memory_order_relaxed); + st_ubb_overlaps.fetch_add(overlaps, std::memory_order_relaxed); + } + for (size_t i = ub_first_of_recording; i < end; i++) + gpu_write(cmd, ub_updates[i].dst, ub_updates[i].bytes.data(), ub_updates[i].bytes.size()), gp.bytes += ub_updates[i].bytes.size(); + } full_barrier(cmd); + if (batched || mode == 2) ub_batch_line(batched); } /// (mesh shading: the stages its resources are bound to, beyond NVN's six) @@ -1735,6 +2160,25 @@ void log_stage_bind(uint32_t op, int stage) { } void maybe_split(VkCommandBuffer& cmd); +// ((Triforce, 11-22, the performance audit's REPLAY-14, after Source A's) Intel's two workarounds at a submission's splits, +// barrier_rules.h, each read once and both logged in one line: a part ends only right before a plain draw +// (RX_VK_SPLIT_BEFORE_DRAW), and a part's end writes no uniform updates where there are no mirrors (RX_VK_SPLIT_UBFLUSH). +// Off and on in that order on every other vendor's GPU, which is the replay as it was +extern bool part_end_wanted, part_end_now; +bool dry_point_wanted = false; // (the dry pass's count asked for a point and the rule holds it for the next plain draw) +bool split_rules(int which) { + static const bool before_draw = barrier_rules::split_before_draw(getenv("RX_VK_SPLIT_BEFORE_DRAW"), dev.props.vendorID); + static const bool ub_flush = [] { + const bool flush = barrier_rules::split_ub_flush(getenv("RX_VK_SPLIT_UBFLUSH"), dev.props.vendorID, pools_mirrored()); + logf("vk: a submission's parts: ended only right before a plain draw %s, uniform updates written at each part's end %s " + "(RX_VK_SPLIT_BEFORE_DRAW, RX_VK_SPLIT_UBFLUSH: 0 or 1; Intel's drivers by default)", + before_draw ? "on" : "off", flush ? "on" : "off"); + return flush; + }(); + return which ? ub_flush : before_draw; +} +inline bool split_before_draw_on() { return split_rules(0); } +inline bool split_ub_flush_on() { return split_rules(1); } // ((Triforce) the early tail, below maybe_split: RX_VK_TAIL_EARLY, tail_early.h) bool tail_on(); bool tail_last(uint64_t gpu, uint32_t* v); @@ -1837,6 +2281,24 @@ void replay(VkCommandBuffer& cmd, NvnState& s, const uint8_t* p, const uint8_t* } const uint8_t* a = p + 8; const uint32_t avail = h.size - 8; + // ((Triforce, 11-22, the performance audit's REPLAY-14, after Source A's) A part, or a helper thread's segment, that + // was due to end waits for the next plain draw and ends right before it (barrier_rules.h split_before_draw: Intel's + // GPUs, whose driver draws the plain draws of a command buffer that begins with mesh draws as shards). Both flags are + // only ever set with the rule on: nothing here runs on another vendor's GPU + if (!worker && (part_end_wanted || dry_point_wanted) && + barrier_rules::cut_ok(h.op == NOP_DRAW || h.op == NOP_DRAW_IDX || h.op == NOP_DRAW_INDIRECT, s.render_enabled, s.raster_discard)) { + if (dry && dry_point_wanted) { + dry_point_wanted = false; // (the point the count asked for: here, with the state before this draw) + if (!seg_early_hook || seg_early_hook->wants(dry_work, p)) { + par_points.push_back({p, s, ub_updates.size(), par_conds.size(), dry_work}); + if (seg_early_hook) seg_early_hook->on_point(par_points.back()); + } + } else if (!dry && part_end_wanted) { + part_end_now = true; + maybe_split(cmd); + part_end_now = false; + } + } if (dry && dry_tail_find) dry_tail_check(p, h, a, avail, s); if (dry && dry_up_find) dry_up_check(p, h, s); // ((Triforce) the early tail: par_up) if (mesh_ahead::t_table.n && !mesh_run_kind(h.op, p)) mesh_ahead::t_table.clear(); // ((Triforce) the run ends here) @@ -2320,12 +2782,16 @@ void replay(VkCommandBuffer& cmd, NvnState& s, const uint8_t* p, const uint8_t* case NOP_RENDER_ENABLE: s.render_enabled = payload(a, avail).value != 0; if (occ_live && !worker && occ_cond_off == &s) occ_cond_off = nullptr; // (Triforce: no condition's from here) + if (occ_live && worker && occ_seg_off == &s) occ_seg_off = nullptr; // (11-22: a segment's, as above) if (dump_on()) logf("vk: dump-op %03d RENDER_ENABLE %d", dump_seq(), (int)s.render_enabled); break; case NOP_RENDER_COND: { // (read on the CPU for now; VK_EXT_conditional_rendering later) if (worker) { // (as the dry pass found it, with the reports before it written and none after) UbReadGuard g; // (RX_VK_SEG_EARLY: the dry pass may still append) - s.render_enabled = cond_at < par_conds_n.load(std::memory_order_acquire) ? par_conds[cond_at] != 0 : true; + // ((11-22) 1 on, 0 off, 2 off from a written count: occ_gate_note) + const uint8_t v = cond_at < par_conds_n.load(std::memory_order_acquire) ? par_conds[cond_at] : 1; + s.render_enabled = v == 1; + occ_seg_off = v == 2 ? &s : nullptr; cond_at++; break; } @@ -2349,11 +2815,13 @@ void replay(VkCommandBuffer& cmd, NvnState& s, const uint8_t* p, const uint8_t* if (always) s.render_enabled = true; } if (dry) { // (RX_VK_SEG_EARLY: the lock only when the append moves the buffer; the count published after it) + // ((11-22) 2: off from a written count, for the segment's occ_gate_note) + const uint8_t v = s.render_enabled ? 1 : occ_live && occ_cond_off == &s ? 2 : 0; if (par_conds.size() == par_conds.capacity()) { UbGrowGuard g; - par_conds.push_back(s.render_enabled ? 1 : 0); + par_conds.push_back(v); } else { - par_conds.push_back(s.render_enabled ? 1 : 0); + par_conds.push_back(v); } par_conds_n.store(par_conds.size(), std::memory_order_release); } @@ -2486,12 +2954,15 @@ void replay(VkCommandBuffer& cmd, NvnState& s, const uint8_t* p, const uint8_t* if (h.op == NOP_DRAW || h.op == NOP_DRAW_IDX || h.op == NOP_DRAW_INDIRECT || h.op == NOP_DRAW_MESH || h.op == NOP_DISPATCH || h.op == NOP_DISPATCH_INDIRECT) { if (occ_live && !worker) occ_skip_note(s, false); // (Triforce: one a written count's condition skips, counted) + if (occ_live && !dry && h.op != NOP_DISPATCH && h.op != NOP_DISPATCH_INDIRECT) occ_gate_note(s, worker); // (11-22) if (dry) { const bool compute = h.op == NOP_DISPATCH || h.op == NOP_DISPATCH_INDIRECT; if (s.render_enabled && (compute || !s.raster_discard)) dry_mark_written(s, compute, dry_work == 0); if (++dry_work % par_point_every() == 0) { + // ((Triforce, 11-22, REPLAY-14) on Intel the point is made at the next plain draw: the loop's top) + if (split_before_draw_on()) dry_point_wanted = true; // (RX_VK_SEG_EARLY: the early path takes one point a boundary; a point it does not take is not copied: ~27 KB of state) - if (!seg_early_hook || seg_early_hook->wants(dry_work, p + h.size)) { + else if (!seg_early_hook || seg_early_hook->wants(dry_work, p + h.size)) { par_points.push_back({p + h.size, s, ub_updates.size(), par_conds.size(), dry_work}); if (seg_early_hook) seg_early_hook->on_point(par_points.back()); } @@ -2517,6 +2988,10 @@ Recording* rec_cur = nullptr; uint64_t rec_cur_value = 0; int rec_parts = 0; int64_t part_work = 0; +// ((Triforce, 11-22, REPLAY-14) the count asked for a part's end and the rule holds it for the next plain draw (part_end_wanted); +// the op loop's top is calling maybe_split for that cut (part_end_now); how many cuts were made that way. The replay's thread +bool part_end_wanted = false, part_end_now = false; +int64_t st_split_deferred = 0; // (a slot is used again once its own submit is done (recording_wait): a submission's parts may outnumber the slots, // which a wait on the timeline value it signals at its end kept them from. A parallel replay's segments are recorded // at once, kMaxBatch of them at most: fewer than the slots) @@ -2531,10 +3006,44 @@ int par_seg_most() { return v; } +/// ((Triforce, 11-22, the performance audit's SYNC-66, after Source B's; barrier_rules.h part_draws) A part's size: RX_VK_CHUNK +/// when it is set, read once; else the RX_VK_LAB_AB knob chunk: in its phases, read at each call, and the driver's default +/// outside them (120 for every driver until the lab picks). It was a static of maybe_split: no knob could change it in a run +int64_t part_draws() { + static const char* const env = getenv("RX_VK_CHUNK"); + static const int64_t env_value = env ? atoll(env) : 0; + static const int64_t by_driver = [] { + bool radv = false; + VkPhysicalDeviceDriverProperties dp{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES}; + VkPhysicalDeviceProperties2 p2{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2}; + p2.pNext = &dp; + if (dev.physical) vkGetPhysicalDeviceProperties2(dev.physical, &p2), radv = dp.driverID == VK_DRIVER_ID_MESA_RADV; + const int64_t v = barrier_rules::part_draws_default(dev.props.vendorID, radv); + logf("vk: a submission's parts: %lld draws and dispatches each (RX_VK_CHUNK%s%s; 0: whole submissions; this driver's default " + "%lld; the lab knob chunk: in its phases when the variable is unset)", + (long long)barrier_rules::part_draws(env != nullptr, env_value, v), env ? "=" : " unset", env ? env : "", (long long)v); + return v; + }(); + return barrier_rules::part_draws(env != nullptr, env_value, lab_ab::value_of("chunk", by_driver)); +} + void maybe_split(VkCommandBuffer& cmd) { - static const int64_t every = getenv("RX_VK_CHUNK") ? atoll(getenv("RX_VK_CHUNK")) : 120; + const int64_t every = part_draws(); if (tail_on()) return; // ((Triforce) the early tail stays one command buffer, to its recording's end) - if (!rec_cur || every <= 0 || ++part_work < every || rec_parts + 1 >= kMaxParts || dump_on()) return; + if (part_end_now) { // ((Triforce, 11-22, REPLAY-14) the cut the count asked for, right before a plain draw: the op loop's top) + part_end_wanted = false; + if (!rec_cur || rec_parts + 1 >= kMaxParts || dump_on()) return; + if (st_split_deferred++ % 3600 == 0) + logf("vk: a submission's parts: %lld ended right before a plain draw so far (RX_VK_SPLIT_BEFORE_DRAW)", + (long long)st_split_deferred); + } else { + if (part_end_wanted) return; // (waiting for that draw) + if (!rec_cur || every <= 0 || ++part_work < every || rec_parts + 1 >= kMaxParts || dump_on()) return; + if (split_before_draw_on()) { + part_end_wanted = true; + return; + } + } // ((Triforce) RX_VK_SPLIT_BARRIER, barrier_rules.h: a pass open at the split goes on in the next part behind a barrier of the // attachments' outputs alone, where nothing else is recorded at the split; otherwise a full barrier at the part's end and the // next one's start, as before) @@ -2558,7 +3067,9 @@ void maybe_split(VkCommandBuffer& cmd) { // ((Triforce, P4) "recorded" by this thread, into any command buffer, or submitted to the queue by any (cmd_work.h work_between). // No parallel replay's argument: its segments are never split (maybe_split runs in the serial replay, never while they record)) const uint64_t ended = work_between(); - flush_ub_updates(cmd); + // ((Triforce, 11-22, REPLAY-14) not on Intel without mirrors (barrier_rules.h split_ub_flush): the submission's last part + // writes them all, as an unsplit submission does; ub_first_of_recording stays at its first, and the overlays cover them) + if (split_ub_flush_on()) flush_ub_updates(cmd); texture_end_recording(cmd); memory_end_submission(cmd); const bool quiet = work_between() == ended; @@ -2566,7 +3077,7 @@ void maybe_split(VkCommandBuffer& cmd) { if (!carry) full_barrier(cmd, "barrier (recording end)"); split_count(in_pass, open, quiet, held, carry); submit_recording(*rec_cur, false, 0, true); // (a part) - ub_first_of_recording = ub_updates.size(); + if (split_ub_flush_on()) ub_first_of_recording = ub_updates.size(); const uint64_t sent = work_between(); Recording& r = begin_recording(rec_cur_value); memory_begin_submission(); @@ -2812,7 +3323,7 @@ int replay_threads_automatic() { facts.driver_id = drv.driverID; facts.device_type = p2.properties.deviceType; facts.driver_info = drv.driverInfo; - const unsigned logical = std::thread::hardware_concurrency(); + const unsigned logical = cpu_count(); const bool amd_older = amd_older_compiler(facts); const int threads = replay_threads::automatic(logical, amd_older); logf("vk: replay helper threads: Automatic gives this PC %d (%u logical processors: %s). %s", threads, logical, @@ -3130,6 +3641,12 @@ bool par_backoff_on() { static const bool v = !getenv("RX_VK_PARALLEL_BACKOFF") || atoi(getenv("RX_VK_PARALLEL_BACKOFF")) != 0; return v; } +/// (Triforce, 11-22) RX_VK_TUNER_SETTLED=0: the rule judges every window and a trial the scene moved under decides, as before +/// (replay_threads.h Tuner, settled) +bool par_settled_on() { + static const bool v = !getenv("RX_VK_TUNER_SETTLED") || atoi(getenv("RX_VK_TUNER_SETTLED")) != 0; + return v; +} /// At each present (the replay thread): the tuner, replay_threads::Tuner (its trials as they were here, and the rule that turns /// the helper threads on without a trial where they plainly pay, or spares a trial where they cannot). Each counted frame's /// parts are summed beside its time (par_frame_now), and a trial's line is followed by one with them (par_trial_parts) @@ -3144,6 +3661,7 @@ void par_tune_present() { replay_threads::Tuner t; t.use_rule = par_rule_on(); t.backoff = par_backoff_on(); + t.settled = par_settled_on(); t.threads = par_threads_max() + (par_own_on() ? 1 : 0); // (the replay thread records segments too: replay_handle) return t; }(); @@ -3167,7 +3685,9 @@ void par_tune_present() { // when the next may come) if (s.trial && (s.better || s.trials <= 8 || s.trials % 20 == 0 || tuner.backoff)) { char more[200] = ""; - if (s.recheck) + if (s.unsettled) // (Triforce, 11-22: settled) + snprintf(more, sizeof more, "; the scene changed under it: nothing decided, the next trial once it settles"); + else if (s.recheck) snprintf(more, sizeof more, "; a short trial (%d frames the other way) found it shorter: a full trial follows", replay_threads::Tuner::kTakeShort); else if (tuner.backoff) @@ -3176,10 +3696,17 @@ void par_tune_present() { logf("vk: parallel replay, trial %lld: %s %.2f ms a frame, the other way %.2f, %s again %.2f: %s (%lld changes so far)%s", (long long)s.trials, par_auto_on != s.better ? "parallel" : "serial", s.a, s.b, par_auto_on != s.better ? "parallel" : "serial", s.c, - s.better ? (par_auto_on ? "parallel from now on" : "serial from now on") : (s.ok ? "as it was" : "not measured (a load)"), + s.better ? (par_auto_on ? "parallel from now on" : "serial from now on") + : s.unsettled ? "as it was, not judged" : (s.ok ? "as it was" : "not measured (a load)"), (long long)s.changes, more); par_trial_parts((long long)s.trials, s.parts); } + if (s.settle) { // (Triforce, 11-22: after a trial the scene moved under) + static int told; + if (told++ < 40) + logf("vk: parallel replay, the scene settled at present %lld: the next trial at present %lld at the earliest", + (long long)tuner.frames(), (long long)s.next); + } if (s.scene) { // (Triforce, 11-16: the back-off's early trial) static int told; if (told++ < 40) @@ -3257,7 +3784,7 @@ const uint8_t* replay_handle_early(VkCommandBuffer& cmd, const uint8_t* h, bool NvnState& state = *cur_state; const auto t_dry0 = std::chrono::steady_clock::now(); const size_t ub0 = ub_updates.size(); - par_points.clear(), par_conds_reset(), dry_work = 0; + par_points.clear(), par_conds_reset(), dry_work = 0, dry_point_wanted = false; dry_marks_begin(); dry_tail_find = false, par_tail_at = false; if (occ_live) occ_serial_give_up(cmd), occ_dry_cb = cb; // (as replay_handle: the segments count from their own resets) @@ -3614,7 +4141,7 @@ const uint8_t* replay_handle(VkCommandBuffer& cmd, const uint8_t* h, bool may_st const auto t_dry0 = std::chrono::steady_clock::now(); const NvnState start = state; const size_t ub0 = ub_updates.size(); - par_points.clear(), par_conds_reset(), dry_work = 0; + par_points.clear(), par_conds_reset(), dry_work = 0, dry_point_wanted = false; dry_marks_begin(); dry_tail_find = hooks, par_tail_at = false; if (occ_live) occ_serial_give_up(cmd), occ_dry_cb = cb; // (Triforce, RX_VK_OCC_QUERY: the segments count from here, each @@ -3935,7 +4462,7 @@ void bench_replay(const std::vector& handles) { Recording& first = begin_recording(value); memory_begin_submission(); full_barrier(first.cmd); - rec_cur = &first, rec_cur_value = value, rec_parts = 0, part_work = 0; + rec_cur = &first, rec_cur_value = value, rec_parts = 0, part_work = 0, part_end_wanted = false; VkCommandBuffer cmd = first.cmd; for (uint64_t h : handles) if (h) replay(cmd, *cur_state, reinterpret_cast(h)); @@ -4101,12 +4628,39 @@ std::atomic blocked_n{0}, drain_waiters{0}; // (queues stopped, for the std::chrono::steady_clock::time_point blocked_since; uint64_t done_max = 0, signaled = 0; // (the newest value recorded whole; the timeline's last signal) int64_t st_stops, st_resumed, st_forced_waits, st_forced[4], st_resume_captures; // (made to go by: presents, releases, waits, time) +// ((Triforce, 11-22, the performance audit's THREAD-11) the holds of the fence waits that need stopped work (wait(), wait_hold.h): +// counted for the order line; and RX_VK_WAIT_HOLD_PAUSE_US, read once: 0, a yield a look, as before) +wait_hold::Stats hold_stats; +int64_t wait_hold_pause_ns() { + static const int64_t v = wait_hold::pause_ns(getenv("RX_VK_WAIT_HOLD_PAUSE_US")); + return v; +} double st_stopped_max_ms; extern int64_t st_present_waits, st_present_waits_out; bool event_waits_on() { static const bool v = !getenv("RX_VK_EVENT_WAITS") || atoi(getenv("RX_VK_EVENT_WAITS")) != 0; return v; } +// ((Triforce, 11-22, the performance audit's SYNC-59, after Source B's; RX_VK_ORDER_WAKE=1, off unless set; the lab knob +// orderwake) A queue stopped at a wait for an event goes on when a submit's schedule() finds the wait met. A signal of +// the CPU's (nvnEventSignal) wakes no one: event_signal_queued only queues it, on purpose (its thread is the frame's +// critical path), and the stop waits for the order thread's next Sleep(1), or for a game thread's wait that yields up +// to 3 ms and then forces it. With this on the order thread waits on an event, 1 ms at most, which the signal sets +// while a queue is stopped (one SetEvent, no lock of the renderer's, no replay on the signalling thread); and a game +// thread's wait that sees a signal pending lets the stopped queues whose waits are met go on, as sync_settle does, +// before it yields. Nothing is forced: schedule() resumes met waits only +bool order_wake_on() { + static const bool env = [] { + const char* e = getenv("RX_VK_ORDER_WAKE"); + const bool v = e && atoi(e) != 0; + if (v) logf("vk: the order thread woken by the CPU's event signals while a queue is stopped (RX_VK_ORDER_WAKE=1)"); + return v; + }(); + return env || lab_ab::on("orderwake"); +} +std::atomic order_wake_event{nullptr}; // (auto-reset; made by the order thread, sync_watch, at its start) +std::atomic st_wake_sets{0}; // (signals that set it) +int64_t st_wake_looks, st_wake_resumes; // (sync_order_mu: looks a wake or a waiting thread made, and those that let work on) int hold_ms() { // (50: with the replay caught up, the worker queue's submission came up to 16 ms before the one that lets it on; // at 3 ms it went past its waits ~80 times a race. The game never waits for it before that submit) @@ -4140,7 +4694,8 @@ uint64_t oldest_unfinished() { // its wait, a present or hold_ms; the release follows it. RX_VK_RELEASE_DRAIN=1: as before ---- void do_texture_released(const nvn_tex* t); void do_program_released(const nvn_prog* p); -std::mutex held_mu; // (its own: sync_order_mu is held while whole submissions replay, and every release waited for them) +// ((Triforce, THREAD-68) counted by the lock census, RX_VK_LOCK_CENSUS; a std::mutex otherwise) +lock_census::CensusMutex held_mu{"held_mu"}; // (its own: sync_order_mu is held while whole submissions replay, and every release waited for them) std::vector held_tex_releases; // (held_mu) std::vector held_prog_releases; // (held_mu) std::atomic st_held_releases{0}; @@ -4148,10 +4703,67 @@ bool hold_releases() { static const bool v = !getenv("RX_VK_RELEASE_DRAIN") || atoi(getenv("RX_VK_RELEASE_DRAIN")) == 0; return v; } +// ((Triforce, 11-22, the performance audit's MEM-85, after Source B's idea; release_queue.h) RX_VK_RELEASE_QUEUE=1, off unless set +// (the lab knob relq in its phases): every release without the replay thread goes to these lists, stopped queue or not, a +// texture's with the write generation of its call, and the replaying thread applies them (releases_apply_locked) +struct QueuedTexRelease { nvn_tex t; WriteGen gen; }; +std::vector queued_tex_releases; // (held_mu) +std::vector queued_prog_releases; // (held_mu) +std::atomic queued_releases{0}; // (both lists' length: written under held_mu, read without it) +std::mutex rel_waits_mu; +release_queue::Waits rel_waits; // (rel_waits_mu: the releases' count, release_count) +bool release_queue_on() { + static const char* env = getenv("RX_VK_RELEASE_QUEUE"); + static const bool told = [] { + if (release_queue::on(env, false)) + logf("vk: RX_VK_RELEASE_QUEUE=1: texture and program releases go to the replaying thread, applied at the next " + "submission's replay or present with no queue stopped (after the stopped work otherwise)"); + return true; + }(); + (void)told; + return release_queue::on(env, lab_ab::on("relq")); +} +/// A release counted: took_ns its time the old way (sync_drain and the replay's lock), or -1 queued; a line every 600 releases +/// when one of them waited 0.5 ms or more or went to the queue +void release_count(int64_t took_ns) { + std::lock_guard lk(rel_waits_mu); + if (took_ns < 0) rel_waits.note_queued(); + else rel_waits.note(took_ns); + if (!rel_waits.due()) return; + if (rel_waits.worth_a_line()) + logf("vk: releases (%lld): %lld waited for the replay or its order (%.1f ms in all, %.2f ms at most), %lld queued for the " + "replaying thread (RX_VK_RELEASE_QUEUE)", (long long)(rel_waits.n + rel_waits.queued), (long long)rel_waits.waited, + rel_waits.ns / 1e6, rel_waits.max_ns / 1e6, (long long)rel_waits.queued); + rel_waits.reset(); +} +/// (replay_mu held) The queued releases applied, when no queue is stopped: at a submission's replay start and in a present, +/// and by update_blocked after the stopped work +void releases_apply_locked() { + if (!release_queue::apply_now(queued_releases.load(std::memory_order_acquire), blocked_n.load())) return; + std::vector ts; + std::vector ps; + { + std::lock_guard hl(held_mu); + ts.swap(queued_tex_releases); + ps.swap(queued_prog_releases); + queued_releases.store(0, std::memory_order_release); + } + for (const QueuedTexRelease& q : ts) { // (as do_texture_released, the replay's lock held already) + mkwvk::texture_released(q.t.serial); + mkwvk::texture_released_at(&q.t, q.gen); + } + for (const nvn_prog& p : ps) program_released_after_reads(&p); // (as do_program_released) +} void update_blocked() { int n = 0; for (const auto& [queue, q] : qruns) n += q.blocked(); if (n && !blocked_n.load()) blocked_since = std::chrono::steady_clock::now(); + // ((Triforce, 11-22, SYNC-29) a stopped queue waits on an event nothing has set: its value comes from guest memory + // (event_value_in), and a plain store there lets it on with no signal counted. Such a wait keeps every look + bool in_memory = false; + for (const auto& [queue, q] : qruns) + if (q.waiting && ev_values.find(wait_at(q.p).gpu) == ev_values.end()) in_memory = true; + ev_wait_in_memory.store(in_memory, std::memory_order_release); blocked_n = n; if (!n) { // (the stopped work has gone on: its releases now) std::vector ts; @@ -4163,6 +4775,10 @@ void update_blocked() { } for (const nvn_tex& t : ts) do_texture_released(&t); for (const nvn_prog& p : ps) do_program_released(&p); + if (queued_releases.load(std::memory_order_acquire)) { // ((Triforce, 11-22, MEM-85) RX_VK_RELEASE_QUEUE's, after them) + std::lock_guard lk(replay_mu); + releases_apply_locked(); + } } } void log_order_stats() { @@ -4172,9 +4788,19 @@ void log_order_stats() { (long long)st_forced_waits, (long long)st_forced[0], (long long)st_forced[1], (long long)st_forced[2], hold_ms(), (long long)st_forced[3], st_stopped_max_ms, (long long)st_waits_met, (long long)st_waits_unmet, (long long)st_resume_captures); + // ((Triforce, 11-22, SYNC-59) the wakes: under about one that lets work on a second, the switch is not worth having) + if (st_wake_sets.load(std::memory_order_relaxed) || st_wake_looks) + logf("vk: the order's wakes (RX_VK_ORDER_WAKE): %lld CPU signals woke the order thread while a queue was stopped; %lld looks " + "after a wake or from a waiting thread, %lld of them let stopped work on", (long long)st_wake_sets.load(std::memory_order_relaxed), + (long long)st_wake_looks, (long long)st_wake_resumes); if (st_present_waits) logf("vk: presents that waited for their frame's signal: %lld (%lld in vain)", (long long)st_present_waits, (long long)st_present_waits_out); + if (hold_stats.holds.load(std::memory_order_relaxed)) { // ((Triforce, THREAD-11) wait_hold.h) + char line[512]; + wait_hold::describe(hold_stats, wait_hold_pause_ns(), line, sizeof line); + logf("%s", line); + } } // ---- a burst: the pieces replayed one after the other with no memory work between them (a submit's job: its own @@ -4184,9 +4810,11 @@ void log_order_stats() { // that one is done; the timeline is signalled at its end as far as allowed, as for any). ---- bool burst_open = false; uint64_t burst_value = 0; +uint64_t burst_signals = 0; // ((Triforce, 11-22, SYNC-29) the CPU's signals counted when the burst began: sync_order_mu) void burst_begin(uint64_t value) { burst_open = event_waits_on() && !dump_on(); burst_value = value; + burst_signals = ev_cpu_signals.load(std::memory_order_acquire); // (noted at the look's start, published at its end: burst_end) for (const auto& [queue, q] : qruns) { if (q.waiting) burst_value = std::max(burst_value, q.value); if (!q.later.empty()) burst_value = std::max(burst_value, q.later.back().value); @@ -4198,6 +4826,9 @@ void burst_end() { burst_open = false; std::lock_guard lk(replay_mu); if (rec_cur) finish_recording(burst_value); + // ((Triforce, 11-22, SYNC-29) the signals this look began with are acted on and what they let on is recorded: published + // only now, after finish_recording, and as noted at the look's START (a signal that came during the look is not covered) + ev_cpu_settled.store(burst_signals, std::memory_order_release); } /// The recording being made: ended and submitted, the timeline signalled as far as no older submission is unfinished @@ -4250,8 +4881,106 @@ uint64_t replay_pin_mask() { return 0; #endif } +// ---- the soft pin's processors (Triforce, 11-22, the performance audit's THREAD-09, after Source A's: replay_pin.h) ---- +// The reserved core is kept from the game's threads, not from other programs': where another program's thread holds it, a +// thread pinned to it alone waits for it, ready to run. On a processor with cores of more than one kind the pin lets the +// thread also run on the fastest-kind cores among its own processors, prefers the reserved core (its ideal processor) and +// leaves out the core it is on, so that it moves at once. Windows only, as the pin is. tests/test_replay_pin_soft.py +// compiles what is between this line and the pin's thread-locals against Windows itself. +struct ReplayPinCpu { + replay_pin::Kinds kinds; // (the fastest kind of core of the calling thread's processor group, and whether there is another) + uint64_t core_of[64] = {}; // (a logical processor's core: all of that core's logical processors) +}; +const ReplayPinCpu& replay_pin_cpu() { + static const ReplayPinCpu cpu = [] { + ReplayPinCpu c; +#if defined(_WIN32) + static uint64_t masks[256]; + static unsigned char classes[256]; + int n = 0; + DWORD len = 0; + GROUP_AFFINITY ga; + const WORD group = GetThreadGroupAffinity(GetCurrentThread(), &ga) ? ga.Group : 0; + if (GetLogicalProcessorInformationEx(RelationProcessorCore, nullptr, &len) || GetLastError() != ERROR_INSUFFICIENT_BUFFER) return c; + char* buf = (char*)malloc(len); + if (buf && GetLogicalProcessorInformationEx(RelationProcessorCore, (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX*)buf, &len)) + for (DWORD at = 0; at < len && n < 256;) { + const SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX* e = (const SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX*)(buf + at); + if (!e->Size) break; + if (e->Relationship == RelationProcessorCore && e->Processor.GroupCount && e->Processor.GroupMask[0].Group == group) { + masks[n] = (uint64_t)e->Processor.GroupMask[0].Mask; + classes[n] = e->Processor.EfficiencyClass; + n++; + } + at += e->Size; + } + free(buf); + c.kinds = replay_pin::kinds(masks, classes, n); + for (int i = 0; i < n; i++) + for (int lp = 0; lp < 64; lp++) + if (masks[i] >> lp & 1) c.core_of[lp] = masks[i]; +#endif + return c; + }(); + return cpu; +} +/// The calling thread's affinity in its processor group (0: not read) +uint64_t replay_pin_own() { +#if defined(_WIN32) + GROUP_AFFINITY ga; + return GetThreadGroupAffinity(GetCurrentThread(), &ga) ? (uint64_t)ga.Mask : 0; +#else + return 0; +#endif +} +/// The logical processors of the core the calling thread is on now (0: not known) +uint64_t replay_pin_core_now() { +#if defined(_WIN32) + const DWORD lp = GetCurrentProcessorNumber(); + return lp < 64 ? replay_pin_cpu().core_of[lp] : 0; +#else + return 0; +#endif +} +/// The calling thread's ideal processor (-1: not read), and setting it (false: Windows refused) +int replay_pin_ideal_now() { +#if defined(_WIN32) + PROCESSOR_NUMBER pn; + return GetThreadIdealProcessorEx(GetCurrentThread(), &pn) ? (int)pn.Number : -1; +#else + return -1; +#endif +} +bool replay_pin_set_ideal(int lp) { +#if defined(_WIN32) + return lp >= 0 && SetThreadIdealProcessor(GetCurrentThread(), (DWORD)lp) != (DWORD)-1; +#else + (void)lp; + return false; +#endif +} +// ---- (the soft pin's processors end) ---- +/// RX_VK_REPLAY_PIN_SOFT, replay_pin.h: the fastest-kind processors a soft pin may also use, 0 where the pin is strict. lab: the +/// lab's knob pinsoft too, which makes the pin soft in its phases on any processor (a thread that is pinned for good has no phases) +uint64_t replay_pin_soft(bool lab = true) { + static const int mode = [] { + const int m = replay_pin::mode(getenv("RX_VK_REPLAY_PIN_SOFT")); + const ReplayPinCpu& c = replay_pin_cpu(); + if (replay_pin_mask()) + logf("vk: a replay's pin is %s: this processor has %s, its fastest on logical processors %#llx (RX_VK_REPLAY_PIN_SOFT=0: " + "the reserved core alone everywhere, 2: also the other fastest cores everywhere)", + c.kinds.fastest && replay_pin::soft(m, c.kinds.hybrid) ? "soft, the reserved core preferred" : "the reserved core alone", + c.kinds.hybrid ? "cores of more than one kind" : "one kind of core", (unsigned long long)c.kinds.fastest); + return m; + }(); + const ReplayPinCpu& c = replay_pin_cpu(); + return c.kinds.fastest && (replay_pin::soft(mode, c.kinds.hybrid) || (lab && lab_ab::on("pinsoft"))) ? c.kinds.fastest : 0; +} thread_local int replay_pin_depth = 0; thread_local DWORD_PTR replay_pin_prev = 0; +// ((Triforce, THREAD-09) the thread's own ideal processor while a soft pin has it on the reserved core; kIdealNone: not moved) +constexpr int kIdealNone = -1; +thread_local int replay_pin_ideal_prev = kIdealNone; // ((Triforce) RX_VK_REPLAY_PRIO, replay_prio.h: the thread's own priority while the outermost pin has it raised; kPrioNone: not raised) constexpr int kPrioNone = -1000; thread_local int replay_prio_prev = kPrioNone; @@ -4261,7 +4990,21 @@ struct ReplayPin { if (async_on()) return; counted = true; if (replay_pin_depth++) return; // (the outermost one: a submit's replay inside its capture's) - if (const uint64_t m = replay_pin_mask()) replay_pin_prev = SetThreadAffinityMask(GetCurrentThread(), (DWORD_PTR)m); + if (const uint64_t m = replay_pin_mask()) { + const uint64_t fast = replay_pin_soft(), own = fast ? replay_pin_own() : 0; + if (!own) replay_pin_prev = SetThreadAffinityMask(GetCurrentThread(), (DWORD_PTR)m); + else { + // ((Triforce, THREAD-09) soft, replay_pin.h: widened first, so that the reserved core is among the thread's + // processors when it becomes its ideal one; then without the core it is on, so that it moves now) + const uint64_t wide = replay_pin::wide(m, fast, own); + const int ideal = replay_pin_ideal_now(); + if ((replay_pin_prev = SetThreadAffinityMask(GetCurrentThread(), (DWORD_PTR)wide))) { + if (ideal >= 0 && replay_pin_set_ideal(replay_pin::ideal(m))) replay_pin_ideal_prev = ideal; + const uint64_t moved = replay_pin::moved(wide, m, replay_pin_core_now()); + if (moved != wide) SetThreadAffinityMask(GetCurrentThread(), (DWORD_PTR)moved); + } + } + } // ((Triforce) raised while it replays, its own priority back after: replay_prio.h) if (const int want = replay_prio_level()) { const int now = GetThreadPriority(GetCurrentThread()); @@ -4272,6 +5015,8 @@ struct ReplayPin { if (!counted || --replay_pin_depth) return; if (replay_prio_prev != kPrioNone) SetThreadPriority(GetCurrentThread(), replay_prio_prev), replay_prio_prev = kPrioNone; if (replay_pin_prev) SetThreadAffinityMask(GetCurrentThread(), replay_pin_prev), replay_pin_prev = 0; + // ((Triforce, THREAD-09) after the affinity, which holds the old ideal processor again) + if (replay_pin_ideal_prev != kIdealNone) replay_pin_set_ideal(replay_pin_ideal_prev), replay_pin_ideal_prev = kIdealNone; } ReplayPin(const ReplayPin&) = delete; ReplayPin& operator=(const ReplayPin&) = delete; @@ -4283,6 +5028,7 @@ struct ReplayPin { bool run_submit(const uint64_t* handles, int n, uint64_t value, uint64_t queue, bool memory_work, int cb = 0, const uint8_t* p = nullptr, bool may_stop = false) { std::lock_guard lk(replay_mu); + releases_apply_locked(); // ((Triforce, 11-22, MEM-85) RX_VK_RELEASE_QUEUE's releases, with no queue stopped: before this one's draws) ReplayPin pin; SlowWatch slow("submit"); PROF(PROF_SUBMIT); @@ -4310,7 +5056,7 @@ bool run_submit(const uint64_t* handles, int n, uint64_t value, uint64_t queue, Recording& first = begin_recording(tag); memory_begin_submission(); full_barrier(first.cmd, "barrier (recording start)"); - rec_cur = &first, rec_cur_value = tag, rec_parts = 0, part_work = 0; + rec_cur = &first, rec_cur_value = tag, rec_parts = 0, part_work = 0, part_end_wanted = false; // ((Triforce) tex_warm.h, RX_VK_TEX_WARM: a game's submit after a light frame (a loading screen): the textures the // game registered are made and uploaded here, in this recording, before the area's first frame asks for them all) if (memory_work && !p) // (the heavier of the frame before and this frame so far: not in an area's first frame once it draws) @@ -4471,12 +5217,13 @@ bool per_queue_fences_off() { struct OrderMutex { std::mutex mu; std::atomic entered{0}, left{0}; + std::atomic owner{nullptr}; // ((Triforce, THREAD-68) the holder, for the lock census; unset when it is off) void lock() { - mu.lock(); + lock_census::lock(mu, owner, "sync_order_mu"); // (mu.lock(), as before, unless RX_VK_LOCK_CENSUS counts) entered.fetch_add(1, std::memory_order_seq_cst); } bool try_lock() { - if (!mu.try_lock()) return false; + if (!lock_census::try_lock(mu, owner)) return false; entered.fetch_add(1, std::memory_order_seq_cst); return true; } @@ -4489,6 +5236,7 @@ std::mutex order_wait_mu; std::condition_variable order_wait_cv; void OrderMutex::unlock() { left.fetch_add(1, std::memory_order_seq_cst); // (after the section's registrations: release) + lock_census::unlocking(owner); mu.unlock(); // (a readback waiting for this section: woken. Its waiters++ and this load are both seq_cst, as are this left++ and its // check, so either it sees left or this sees it waiting and the notify comes after it is in the wait: no lost wake-up) @@ -4532,17 +5280,51 @@ void sync_settle() { burst_end(); update_blocked(); } +// ((Triforce, 11-22, the performance audit's SYNC-29, after Source A's; settle_gate.h) A read's look at the stopped queues, +// through the gate where RX_VK_SETTLE_GATE says so: 1 the readback's look (RX_VK_RB_SETTLE's), 2 the texture sites' too. A +// gated site looks only while a queue is stopped and the CPU has signalled since the last look, or a stopped queue waits on +// an event read from guest memory; any other met wait was met by a signal in the stream, and the thread that replayed it +// looked before it let sync_order_mu go. Unset or 0: sync_settle() at every site, as before. The lab knobs settlegate and +// texgate are levels 1 and 2 in their phases +int settle_gate_level() { + static const int env = [] { + const int l = settle_gate::level(getenv("RX_VK_SETTLE_GATE")); + if (l) logf("vk: RX_VK_SETTLE_GATE=%d: %s at the stopped queues only after a new CPU signal, or while one of them waits on an " + "event read from guest memory", l, l >= 2 ? "a readback and a texture's catch-up look" : "a readback looks"); + return l; + }(); + return settle_gate::level_now(env, lab_ab::on("settlegate"), lab_ab::on("texgate")); +} +std::atomic st_settle_looks[settle_gate::kSites], st_settle_skips[settle_gate::kSites]; +void sync_settle_at(int site) { + const int what = settle_gate::decide(settle_gate_level(), site, blocked_n.load() != 0, ev_cpu_signals.load(std::memory_order_acquire), + ev_cpu_settled.load(std::memory_order_acquire), ev_wait_in_memory.load(std::memory_order_acquire)); + if (what == settle_gate::kNothing) return; + if (what == settle_gate::kSkip) { + st_settle_skips[site].fetch_add(1, std::memory_order_relaxed); + return; + } + st_settle_looks[site].fetch_add(1, std::memory_order_relaxed); + sync_settle(); +} void sync_watch() { SetThreadDescription(GetCurrentThread(), L"mkw_vk order"); + // ((Triforce, 11-22, SYNC-59) the event a CPU signal wakes this thread with: auto-reset, not signalled) + const HANDLE wake = CreateEventA(nullptr, FALSE, FALSE, nullptr); + order_wake_event.store(wake, std::memory_order_release); for (;;) { - Sleep(1); + bool woken = false; + if (wake && order_wake_on()) woken = WaitForSingleObject(wake, 1) == WAIT_OBJECT_0; // (1 ms at most, as the sleep) + else Sleep(1); if (!blocked_n.load()) continue; std::unique_lock lk(sync_order_mu, std::try_to_lock); if (!lk) continue; + const int64_t resumed0 = st_resumed; burst_begin(0); // (a signal from the CPU the thread that made it could not act on) schedule(); burst_end(); update_blocked(); + if (woken) st_wake_looks++, st_wake_resumes += st_resumed != resumed0; const double held = std::chrono::duration(std::chrono::steady_clock::now() - blocked_since).count(); if (blocked_n.load() && held > hold_ms()) { drain(3); @@ -4592,6 +5374,12 @@ bool wait_recorded(uint64_t f, uint64_t timeout_ns, int* ok) { if (it == value_rec.end()) return false; seq = it->second; } + // ((Triforce) RX_VK_PEND_HELD: a recording the recordings' timeline has passed is done: no flush, no wait) + uint64_t rec_done = 0; + if (pend_held_on() && vkGetSemaphoreCounterValue(dev.device, rec_timeline, &rec_done) == VK_SUCCESS && rec_done >= seq) { + *ok = 1; + return true; + } // (a batched submit holds its recordings' signal until the batch goes out: the readback wait in invalidate came here // without a flush, and the job that waited held up the main thread, so nothing flushed it: the load hang on RADV) queue_flush_for(FR_WAIT); @@ -4602,6 +5390,56 @@ bool wait_recorded(uint64_t f, uint64_t timeout_ns, int* ok) { *ok = vkWaitSemaphores(dev.device, &wi, timeout_ns) == VK_SUCCESS; return true; } +// ((Triforce, 11-22, the performance audit's SYNC-122, after Source B's) A readback whose write-back is not recorded yet +// (wait_recorded said so: its writer is still being recorded, or stopped at an event wait). It waited on the submissions' +// timeline with no bound and never looked again: recorded and done a moment later, the read still waited for every older +// stopped submission. Counted always (how many, how long: the 3000-readback line). With RX_VK_WB_WAIT_REC=1 or 2 +// (wb_rec_wait.h fallback_wait) it waits in 1 ms slices and looks the recording up between them: 1 ends the wait with that +// recording, as vk_timeline_wait does, with no overall bound (the read needs its bytes); 2 ends where the old wait ends and +// counts the part after the recording was done, what 1 would not have waited. Unset: the one wait it was (wait_value). +// by_recording: the caller waits by recordings at all (not with RX_VK_PER_QUEUE_FENCES=0 or RX_VK_SYNC_WAITS=0) +std::atomic st_rb_fb{0}, st_rb_fb_ns{0}, st_rb_fb_held{0}, st_rb_fb_held_ns{0}, st_rb_fb_at_rec{0}; +void readback_fallback_wait(uint64_t v, bool by_recording) { + namespace wr = ::mkwvk::wb_rec_wait; + struct Calls { + uint64_t v; + std::chrono::steady_clock::time_point t0; + void plain() { wait_value(v); } + void flush() { queue_flush_for(FR_VALUE); } + uint64_t lookup() { + std::lock_guard mk(rec_map_mu); + return fin_rec.of(v); + } + int rec_wait(uint64_t seq) { + queue_flush_for(FR_WAIT); // (registered and held since the look: as wait_recorded) + VkSemaphoreWaitInfo ri{VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO}; + ri.semaphoreCount = 1; + ri.pSemaphores = &rec_timeline; + ri.pValues = &seq; + return vkWaitSemaphores(dev.device, &ri, UINT64_MAX) == VK_SUCCESS ? 0 : 2; + } + bool rec_done(uint64_t seq) { + uint64_t done = 0; + return vkGetSemaphoreCounterValue(dev.device, rec_timeline, &done) == VK_SUCCESS && done >= seq; + } + int slice() { + VkSemaphoreWaitInfo wi{VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO}; + wi.semaphoreCount = 1; + wi.pSemaphores = &timeline; + wi.pValues = &v; + const VkResult res = vkWaitSemaphores(dev.device, &wi, wr::kSliceNs); + return res == VK_SUCCESS ? 0 : res == VK_TIMEOUT ? 1 : 2; + } + int64_t now() { return std::chrono::duration_cast(std::chrono::steady_clock::now() - t0).count(); } + } calls{v, std::chrono::steady_clock::now()}; + const wr::FallbackResult r = wr::fallback_wait(wb_wait_rec_mode(), by_recording && rec_timeline != VK_NULL_HANDLE, calls); + st_rb_fb.fetch_add(1, std::memory_order_relaxed), st_rb_fb_ns.fetch_add(r.total_ns, std::memory_order_relaxed); + if (r.at_recording) st_rb_fb_at_rec.fetch_add(1, std::memory_order_relaxed); + if (const int64_t held = wr::fallback_held_ns(r)) { + st_rb_fb_held.fetch_add(1, std::memory_order_relaxed), st_rb_fb_held_ns.fetch_add(held, std::memory_order_relaxed); + stall_ct.wb_held_by_ns[wr::kReadback].fetch_add(held, std::memory_order_relaxed); + } +} uint64_t fence(); /// (hookshot) A fence of one queue (nvnQueueFenceSync): the value of that queue's last submission bool async_unforced(); @@ -4665,24 +5503,58 @@ int wait(uint64_t f, uint64_t timeout_ns) { // RX_VK_WAIT_HOLD_US: how long the waiter gives it. A wait for older work than the stopped ones needs nothing of them) if (timeout_ns && !async_on() && blocked_n.load()) { static const int64_t hold_ns = (getenv("RX_VK_WAIT_HOLD_US") ? atoll(getenv("RX_VK_WAIT_HOLD_US")) : 3000) * 1000; + // ((Triforce, 11-22, the performance audit's THREAD-11, after Source B's: wait_hold.h) The hold is counted. With + // RX_VK_WAIT_HOLD_PAUSE_US set, a gap of PAUSEs stands between two looks instead of a yield, and the order lock is + // taken only when a section of it has left since this waiter asked (what it asks changes only inside such sections) or + // while a signal of the CPU's is queued (ev_cpu_any: it reaches the order without a section, and a section acts on it). + // The fence's own value is read at every look either way. 0, the default: a yield a look and the lock at each, as before) + const int64_t pause_ns = wait_hold_pause_ns(); const auto t0 = std::chrono::steady_clock::now(); + const auto since = [t0] { return (int64_t)std::chrono::duration_cast(std::chrono::steady_clock::now() - t0).count(); }; + int64_t looks = 0, locks = 0; + uint64_t seen_left = 0; + bool asked = false, needs = true; + wait_hold::End end = wait_hold::kLetOn; for (;;) { queue_flush_for(FR_WAIT); - if (completed() >= f) return 1; - bool needs; - { - std::lock_guard lk(sync_order_mu); - needs = oldest_unfinished() <= f; - } - if (!needs) break; - const int64_t waited = std::chrono::duration_cast(std::chrono::steady_clock::now() - t0).count(); - if ((uint64_t)waited >= timeout_ns) break; // (sync_watch's, past hold_ms) - if (waited >= hold_ns) { - sync_drain(2); + looks++; + if (completed() >= f) { + end = wait_hold::kMet; break; } - std::this_thread::yield(); + if (wait_hold::takes_lock(pause_ns, asked, seen_left, sync_order_mu.left.load(std::memory_order_acquire), + ev_cpu_any.load(std::memory_order_acquire))) { + std::lock_guard lk(sync_order_mu); + // ((Triforce, 11-22, SYNC-59) a signal of the CPU's is pending: the stopped queues whose waits it met go on + // now, on this thread, which waits for them anyway; as sync_settle, none forced) + if (ev_cpu_any.load(std::memory_order_acquire) && order_wake_on()) { + const int64_t resumed0 = st_resumed; + burst_begin(0); + schedule(); + burst_end(); + update_blocked(); + st_wake_looks++, st_wake_resumes += st_resumed != resumed0; + } + needs = oldest_unfinished() <= f; + seen_left = sync_order_mu.left.load(std::memory_order_relaxed); // (inside this section: one more once it has left) + asked = true, locks++; + } + if (!needs) break; + const int64_t waited = since(); + if ((uint64_t)waited >= timeout_ns) { // (sync_watch's, past hold_ms) + end = wait_hold::kTimedOut; + break; + } + if (waited >= hold_ns) { + sync_drain(2); + end = wait_hold::kForced; + break; + } + if (!pause_ns) std::this_thread::yield(); + else wait_hold::gap(pause_ns, since, gate_wait::pause); } + hold_stats.done(end, looks, locks, since()); + if (end == wait_hold::kMet) return 1; } VkSemaphoreWaitInfo wi{VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO}; wi.semaphoreCount = 1; @@ -4783,8 +5655,101 @@ bool tex_pages_any(uint64_t gpu, uint64_t size) { } return false; } -void texture_created_of(const nvn_tex* t) { tex_pages_add(t, +1); } +void texture_created_of(const nvn_tex* t) { + tex_pages_add(t, +1); + texture_warm_born(t); // ((Triforce, 11-22, TEX-23) RX_VK_TEX_WARM_LIST: when the game made it, vk_texture.cpp) +} uint64_t fence(); +/// ((Triforce, 11-22, the performance audit's THREAD-25, after Source A's: page_marks.h) 1: invalidate(gpu, size) below would +/// return without waiting for anything, so the host leaves the call out and its thread keeps its virtual core +/// (mkwvk_invalidate_quick; host/rx_backend.c rx_nvn_invalidate). That is: nothing to invalidate; or, below 512 GB, no order +/// ticket of a fast fence poll still to wait for (the wait invalidate() begins with), no live texture's storage in the range +/// (tex_pages_any), no texture the GPU wrote ever listed there and no write-back ever made into its pages. With those, +/// invalidate() does its ticket wait in two loads, finds "never written back", and returns at "no texture's storage there". +/// No lock is taken and nothing waited for. 0: the call is made as before; always 0 unless RX_VK_INVALIDATE_QUICK=1, and +/// with RX_VK_LOG_INVALIDATE, RX_VK_LOG_READBACK or RX_VK_INVALIDATE_WAIT set, which log or wait at every call. Every 10 s +/// a line counts the calls left out and those made. Lab: the knob noinvq makes every call in its phases +int invalidate_quick(uint64_t gpu, uint64_t size) { + static const bool asked = inv_quick::on(getenv("RX_VK_INVALIDATE_QUICK"), getenv("RX_VK_LOG_INVALIDATE"), + getenv("RX_VK_LOG_READBACK"), getenv("RX_VK_INVALIDATE_WAIT")); + if (!asked) return 0; + inv_quick::Look l; + l.gpu = gpu, l.size = size; + bool nothing = false; + if (!lab_ab::on("noinvq")) { + if (size && gpu < PageMarks::kSpan && size <= PageMarks::kSpan - gpu) { + // (the ticket first, as order_ticket_wait reads it: a section that has left has made its marks) + const uint64_t ticket = fence_poll_fast_on() ? order_ticket.load(std::memory_order_seq_cst) : 0; + l.ticket_pending = sync_order_mu.left.load(std::memory_order_acquire) < ticket; + l.tex_storage = tex_pages_any(gpu, size); + l.vw_marked = texture_gpu_written_marked(gpu, size); + l.wb_marked = memory_writeback_marked(host->gpu_guest(gpu), size); + } + nothing = inv_quick::nothing_to_wait_for(l); + } + static std::atomic left_out{0}, made{0}, next_ns{0}; + const int64_t n = (nothing ? left_out : made).fetch_add(1, std::memory_order_relaxed); + if ((n & 1023) == 0) { // (a look at the clock every 1024 calls of a kind) + const int64_t now = std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); + int64_t at = next_ns.load(std::memory_order_relaxed); + if (now >= at && next_ns.compare_exchange_strong(at, now + 10000000000LL, std::memory_order_relaxed) && at) + logf("vk: invalidates: %lld left out with nothing to wait for (the caller kept its virtual core), %lld made " + "(RX_VK_INVALIDATE_QUICK=0: all made)", (long long)left_out.exchange(0), (long long)made.exchange(0)); + } + return nothing ? 1 : 0; +} +// ((Triforce, 11-22, the performance audit's SYNC-19, after Source A's; readback_late.h) RX_VK_READBACK_LATE names ranges the +// CPU reads back without the wait for their write-back, from what guest memory holds: a size in hex, or `newest` (the writer +// is the newest submission handed out). UNSET: NO RULE, every readback waits as before (the audit's lab runs it on a handheld +// first); =0: none either. The lab knob rblate: the camera block's size (3980) in its phases, when the variable is unset +const readback_late::Rules& rb_late_rules() { + static const readback_late::Rules rules = [] { + const char* e = getenv("RX_VK_READBACK_LATE"); + const readback_late::Rules r = readback_late::parse(e); + if (r.any() || r.bad) { + char sizes[200] = ""; + int o = 0; + for (int i = 0; i < r.n && o < 180; i++) o += snprintf(sizes + o, sizeof sizes - o, " %#llx", (unsigned long long)r.sizes[i]); + logf("vk: RX_VK_READBACK_LATE=%s: read back without their write-back's wait: ranges of size%s%s%s; %d parts of the text " + "not understood", e, r.n ? sizes : " (none)", r.newest ? ", and ranges whose writer is the newest submission handed out" : "", + r.any() ? "" : " (no rule: every readback waits)", r.bad); + } + return r; + }(); + return rules; +} +bool rb_late(uint64_t size, uint64_t writer, uint64_t done, uint64_t handed_out) { + const readback_late::Rules& r = rb_late_rules(); + if (r.any()) return readback_late::late(r, size, writer, done, handed_out); + static const bool unset = !getenv("RX_VK_READBACK_LATE") || !*getenv("RX_VK_READBACK_LATE"); + if (!unset || !lab_ab::on("rblate")) return false; + static const readback_late::Rules lab = readback_late::default_rules(); + return readback_late::late(lab, size, writer, done, handed_out); +} +/// (RX_VK_LOG_READBACK_RANGES=1: the readbacks counted by range, readback_late.h RangeLog; nothing noted without it) +bool rb_ranges_on() { + static const bool v = getenv("RX_VK_LOG_READBACK_RANGES") && atoi(getenv("RX_VK_LOG_READBACK_RANGES")) != 0; + return v; +} +std::mutex rb_ranges_mu; +readback_late::RangeLog rb_ranges; // (rb_ranges_mu) +void rb_ranges_line() { + std::vector top; + readback_late::RangeLog::Row other; + size_t n = 0; + { + std::lock_guard lk(rb_ranges_mu); + top = rb_ranges.top(12), other = rb_ranges.other, n = rb_ranges.rows.size(); + } + logf("vk: readbacks by range (RX_VK_LOG_READBACK_RANGES; %zu ranges since the start, the 12 that waited longest):", n); + for (const auto& l : top) + logf("vk: %#llx +%#llx: %lld readbacks, %lld waited %.2f ms each (%.1f ms in all), the writer the newest submission at %lld, " + "read late %lld", (unsigned long long)l.gpu, (unsigned long long)l.size, (long long)l.row.readbacks, (long long)l.row.waits, + l.row.waits ? l.row.wait_ns / (double)l.row.waits / 1e6 : 0.0, l.row.wait_ns / 1e6, (long long)l.row.newest, (long long)l.row.late); + if (other.readbacks) + logf("vk: past the table's %zu ranges: %lld readbacks, %lld waited %.1f ms in all", n, (long long)other.readbacks, + (long long)other.waits, other.wait_ns / 1e6); +} void invalidate(uint64_t gpu, uint64_t size) { // RX_VK_READBACK_TICKET: the fast fence poll (wait) no longer waits for the sync_order_mu section in progress (a submit_q / // present replay); it notes that section's number t and the CPU's readback here waits until section t has left, before @@ -4830,9 +5795,11 @@ void invalidate(uint64_t gpu, uint64_t size) { if (size) { static const bool off = getenv("RX_VK_NO_READBACK_WAIT") != nullptr; bool ever = false; - if (!off && (memory_writeback_value(host->gpu_guest(gpu), size, &ever), ever)) { + uint64_t v0 = 0; // ((Triforce, 11-22, SYNC-19) the write-back's value as the readback came: the late rule's) + if (!off && (v0 = memory_writeback_value(host->gpu_guest(gpu), size, &ever), ever)) { memory_note_readback(host->gpu_guest(gpu), size); // (PR 430: the CPU may write it back: memory_rb_refresh) static std::atomic calls{0}, waited{0}, ns{0}; + static std::atomic lates{0}; // ((Triforce, 11-22, SYNC-19) read late since the 3000-call line) const auto w0 = std::chrono::steady_clock::now(); bool w = false; const uint64_t f0 = fence(), c0 = completed(); @@ -4851,13 +5818,19 @@ void invalidate(uint64_t gpu, uint64_t size) { // nothing forced, was tried against the player's character drawn black for a frame after a tour, and taken out again: // it cost a quarter of the frame rate (67.7 fps for 85 to 96: the readbacks are the update's workers', about 30 a // frame), and RX_VK_LOG_READBACK_ZERO below showed no readback that read 0 at any of 43 such frames) - if (behind0 && !no_catchup && !unforced) wait_queue_empty(), w = true; // (work handed over may write it back too) + // ((Triforce, 11-22, SYNC-19; readback_late.h) a range named by RX_VK_READBACK_LATE whose write-back is still + // ahead of the GPU: read from what guest memory holds, with neither the catch-up nor the wait below. It keeps its + // note above. No rule (the default): never late + const bool is_late = rb_late(size, v0, c0, f0); + if (is_late) lates.fetch_add(1, std::memory_order_relaxed); + else if (behind0 && !no_catchup && !unforced) wait_queue_empty(), w = true; // (work handed over may write it back too) // (upstream #2125, behind RX_VK_RB_SETTLE: the stopped work whose waits are met goes on first, never forced: it may // be what writes this range, and its write-back is known only once it is recorded) - else if (behind0 && !no_catchup && !async_on() && rb_fix(2)) sync_settle(), w = true; + // ((Triforce, 11-22, SYNC-29) through the gate with RX_VK_SETTLE_GATE=1 or 2; unset, sync_settle() as it was) + else if (behind0 && !no_catchup && !async_on() && rb_fix(2)) sync_settle_at(settle_gate::kReadback), w = true; const auto w1 = std::chrono::steady_clock::now(); const uint64_t v = memory_writeback_value(host->gpu_guest(gpu), size, nullptr); - if (v > completed()) { + if (!is_late && v > completed()) { int ok = 0; if (!(unforced && wait_recorded(v, UINT64_MAX, &ok))) { if (unforced && !async_on() && replay_behind()) wait_queue_empty(); // (not recorded whole: as before) @@ -4865,7 +5838,9 @@ void invalidate(uint64_t gpu, uint64_t size) { while (completed() < v && !wait_recorded(v, UINT64_MAX, &ok) && async_wait_step(v)) {} if (completed() < v && !ok) wait_value(v); } else { - wait_value(v); + // ((Triforce, 11-22, SYNC-122) the fallback: counted; in slices that look the recording up under + // RX_VK_WB_WAIT_REC=1 or 2, else wait_value(v) as it was) + readback_fallback_wait(v, unforced); } } w = true; @@ -4882,6 +5857,11 @@ void invalidate(uint64_t gpu, uint64_t size) { std::chrono::duration_cast(w1 - w0).count() / 1e6, (unsigned long long)v, (unsigned long long)f0, (unsigned long long)c0, (int)behind0); } + if (rb_ranges_on()) { // ((Triforce, 11-22, SYNC-19) by range: which ones wait, how long, and whose writer is the newest) + const int64_t d = std::chrono::duration_cast(std::chrono::steady_clock::now() - w0).count(); + std::lock_guard rk(rb_ranges_mu); + rb_ranges.note(gpu, size, w, d, v0 >= f0, is_late); + } { // ((Triforce, a diagnostic) RX_VK_LOG_READBACK_ZERO=1: a range whose first words are all 0 as the game is about to // read them, where they were not at its readback before: which readback hands the game a word it set to 0 // itself, the frame, and how far the replay was. 6000 lines a run at most @@ -4908,6 +5888,27 @@ void invalidate(uint64_t gpu, uint64_t size) { const int64_t n = waited.exchange(0); logf("vk: readbacks of GPU results: %lld of 3000 waited for their write-back, %.2f ms each", (long long)n, n ? ns.exchange(0) / (double)n / 1e6 : 0.0); + // ((Triforce, 11-22, SYNC-19) those read late, and the ranges' table) + if (const int64_t late_n = lates.exchange(0)) + logf("vk: readbacks of GPU results: %lld of 3000 read late, without their write-back's wait (RX_VK_READBACK_LATE)", + (long long)late_n); + if (rb_ranges_on()) rb_ranges_line(); + // ((Triforce, 11-22, SYNC-29) the readback's looks at the stopped queues (RX_VK_RB_SETTLE), and those the gate skipped + { + const int64_t looks = st_settle_looks[settle_gate::kReadback].exchange(0), skips = st_settle_skips[settle_gate::kReadback].exchange(0); + if (looks || skips) + logf("vk: readbacks of GPU results: %lld looked at the stopped queues first (RX_VK_RB_SETTLE), %lld skipped the " + "look (RX_VK_SETTLE_GATE: no CPU signal since the last look)", (long long)looks, (long long)skips); + } + // ((Triforce, 11-22, SYNC-122) those whose write-back was not recorded yet: the submissions' timeline's wait) + if (const int64_t fb = st_rb_fb.exchange(0)) { + const int64_t fb_ns = st_rb_fb_ns.exchange(0), held = st_rb_fb_held.exchange(0), + held_ns = st_rb_fb_held_ns.exchange(0), at_rec = st_rb_fb_at_rec.exchange(0); + logf("vk: readbacks of GPU results: %lld of them waited on the submissions' timeline, their write-back not " + "recorded yet, %.2f ms each; %lld ended at their writer's recording (RX_VK_WB_WAIT_REC=1); %lld went on " + "%.2f ms in all after it was done (counted with RX_VK_WB_WAIT_REC=2: what 1 would not have waited)", + (long long)fb, fb_ns / (double)fb / 1e6, (long long)at_rec, (long long)held, held_ns / 1e6); + } } } } @@ -4937,17 +5938,23 @@ void invalidate(uint64_t gpu, uint64_t size) { // one that orders the exposure meter. Without it the Market flickered at once, 4,416 cell blinks in 6,732 frames against // 0, for 70.3 fps against 67.6-68.0: the game reads each frame's sums whole, after that frame's replay and GPU work) if (static const bool exact = getenv("RX_VK_TEX_EXACT") != nullptr; exact && !always && async_unforced() && !tex_exact_any(gpu, size)) return; - unforced ? sync_settle() : wait_queue_empty(); + unforced ? sync_settle_at(settle_gate::kTexture) : wait_queue_empty(); // ((Triforce, 11-22, SYNC-29) gated at level 2 only) if (!texture_gpu_written_in(gpu, size)) return; } { // (what the catch-ups for a texture cost the game's thread, every 300 calls) static std::atomic calls{0}, ns{0}; const auto w0 = std::chrono::steady_clock::now(); - unforced ? sync_settle() : wait_queue_empty(); + unforced ? sync_settle_at(settle_gate::kTexture) : wait_queue_empty(); // ((Triforce, 11-22, SYNC-29) gated at level 2 only) ns += std::chrono::duration_cast(std::chrono::steady_clock::now() - w0).count(); - if (++calls % 300 == 0) + if (++calls % 300 == 0) { logf("vk: invalidate %#llx +%#llx: textures there; the catch-ups waited %.2f ms each (300 calls)", (unsigned long long)gpu, (unsigned long long)size, ns.exchange(0) / 300.0 / 1e6); + // ((Triforce, 11-22, SYNC-29) the texture sites' looks at the stopped queues, and those the gate skipped + const int64_t looks = st_settle_looks[settle_gate::kTexture].exchange(0), skips = st_settle_skips[settle_gate::kTexture].exchange(0); + if (looks || skips) + logf("vk: invalidate: the texture catch-ups looked at the stopped queues %lld times, %lld looks skipped " + "(RX_VK_SETTLE_GATE=2: no CPU signal since the last look)", (long long)looks, (long long)skips); + } } std::lock_guard lk(replay_mu); texture_flush_to_guest(gpu, size); @@ -5510,6 +6517,18 @@ void log_frame_stats() { (up - l_up) / 120.0, (upb - l_upb) / 120.0 / 1048576.0, (st_ub_updates - l_ubu) / 120.0, (st_ub_overlays - l_ubo) / 120.0, (st_wait_ns - l_wait) / 120.0 / 1e6, (long long)pl, pms, (long long)tr, (long long)fl, texture_count(), (unsigned long long)(texture_bytes() >> 20)); + { // ((Triforce, 11-22, the performance audit's SYNC-27) every thread's time in calls that block on the GPU, by kind: + // what "GPU-bound wait" above, the recording slots' waits alone, does not count (gpu_wait_kinds.h) + static ::gpu_wait_kinds::Reader waits; + int64_t ns[::gpu_wait_kinds::KINDS], n[::gpu_wait_kinds::KINDS]; + const int64_t all = waits.take(ns, n); + if (::gpu_wait_kinds::on()) + logf("vk: host waits for the GPU: %.2f ms a frame (fences %.2f, timelines %.2f, idle %.2f, queries %.2f; %.1f waits a " + "frame; all threads; RX_VK_GPU_WAITS=0: not counted)", all / 120.0 / 1e6, ns[::gpu_wait_kinds::FENCE] / 120.0 / 1e6, + ns[::gpu_wait_kinds::TIMELINE] / 120.0 / 1e6, ns[::gpu_wait_kinds::IDLE] / 120.0 / 1e6, + ns[::gpu_wait_kinds::QUERY] / 120.0 / 1e6, + (n[::gpu_wait_kinds::FENCE] + n[::gpu_wait_kinds::TIMELINE] + n[::gpu_wait_kinds::IDLE] + n[::gpu_wait_kinds::QUERY]) / 120.0); + } draw_skip_report(120.0); { // (hookshot: the full barriers recorded and the ones left out as the second of two in a row, RX_VK_BARRIER_DEDUPE_OFF=1) static int64_t l_made, l_left; @@ -5692,6 +6711,32 @@ void status_tick() { namespace sl = status_line; static uint64_t frames = 0; const int lab_col = lab_ab::phase_tick(); // (lab: this frame's A/B phase, the trace's last column) + if (lock_census::enabled()) { // ((Triforce, THREAD-68) lab: with a knob lockcensus the census counts only in that knob's phases) + static const bool knob = [] { + bool k = false; + for (int i = 0; i < lab_ab::n_knobs; i++) k = k || occ::knob_names(lab_ab::knobs[i].name, "lockcensus"); + return k; + }(); + if (knob) lock_census::phase_on.store(lab_ab::on("lockcensus"), std::memory_order_relaxed); + } + if (lab_ab::each_line[0]) logf("%s", lab_ab::each_line), lab_ab::each_line[0] = 0; // ((Triforce, 11-22, REPLAY-51) an e_ phase's line + { // ((Triforce, 11-22, REPLAY-51) the e_ knobs move with the draws between threads unless the helpers' count is pinned) + static bool told; + if (!told && lab_ab::n_knobs > 0) { + told = true; + if (lab_ab::any_each() && !getenv("RX_VK_PARALLEL")) + logf("vk: lab A/B: the e_ knobs wait on whichever thread records a draw: pin RX_VK_PARALLEL (0, or a count) for " + "phases that compare"); + } + } + if (occ_lean_census_on()) { // ((Triforce, 11-22, DESC-82) RX_VK_OCC_LEAN=3: the occlusion proxies' census, every 600 presents + static uint64_t presents = 0; + if (++presents % 600 == 0) { + char line[1024]; + occ_lean::census.line(line, sizeof line, 600); + logf("%s", line); + } + } // (the slowest frame since the last line, present to present: a stall shows here, to be matched with what the log says // around it, such as DLSS's feature made again) static int64_t last_ns = 0, slowest_ns = 0; @@ -5749,6 +6794,9 @@ void status_tick() { sp.captures = (cap_ns - l_cap_ns) / 1e6; sp.wb_wait = (wb_ns - l_wb_ns) / 1e6, sp.wb_held = (wb_held_ns - l_wb_held_ns) / 1e6; sp.qs_bg = (qsbg_ns - l_qsbg_ns) / 1e6; + // ((Triforce, 11-22, SYNC-27) this frame's time in calls that block on the GPU, every thread: the trace's own reader) + static ::gpu_wait_kinds::Reader trace_waits; + sp.gpu_waits = trace_waits.take(nullptr, nullptr) / 1e6; char split[256]; frame_trace::line(sp, split, sizeof split); char split_lab[288]; // ((Triforce, lab) the in-run A/B phase after the split: the line's last column, 'lab') @@ -5821,6 +6869,9 @@ void status_tick() { game_ns = stall_ct.game_wait_ns.load(std::memory_order_relaxed), game_n = stall_ct.game_waits.load(std::memory_order_relaxed), present_ns = stall_ct.present_ns.load(std::memory_order_relaxed); const int64_t wb_held = stall_ct.wb_held.load(std::memory_order_relaxed), wb_held_ns = stall_ct.wb_held_ns.load(std::memory_order_relaxed); + static int64_t s_wb_by[::mkwvk::wb_rec_wait::kCallers]; // ((Triforce, 11-22, SYNC-122) the held part by caller) + int64_t wb_by[::mkwvk::wb_rec_wait::kCallers]; + for (int i = 0; i < ::mkwvk::wb_rec_wait::kCallers; i++) wb_by[i] = stall_ct.wb_held_by_ns[i].load(std::memory_order_relaxed); const int64_t present_wait = present_waited_ns(); MemoryCaptureStats caps; // ((Triforce) the submit-time captures on the game's threads) memory_capture_stats(&caps); @@ -5842,6 +6893,7 @@ void status_tick() { f.wb_wait = (wb_ns - s_wb_ns) / 1e6, f.wb_waits = wb_n - s_wb_n, f.wb_ahead = (int64_t)ahead, f.wb_behind_present = wb_bp - s_wb_bp; f.wb_unsent = wb_un - s_wb_un, f.wb_unsent_ms = (wb_un_ns - s_wb_un_ns) / 1e6; f.wb_held = wb_held - s_wb_held, f.wb_held_ms = (wb_held_ns - s_wb_held_ns) / 1e6; + for (int i = 0; i < ::mkwvk::wb_rec_wait::kCallers; i++) f.wb_held_by[i] = (wb_by[i] - s_wb_by[i]) / 1e6; f.pipe_waits = pw - s_pw, f.pipe_wait = pwms - s_pwms, f.left_out = ps - s_ps; f.pipes_first = pf - s_pf, f.pipes_ahead = pah - s_pah, f.pipes_order = por - s_por; f.pipes_links = pln - s_pln, f.pipes_libs = plb - s_plb; @@ -5903,6 +6955,7 @@ void status_tick() { s_cap_ns = cap_ns; s_wb_un = wb_un, s_wb_un_ns = wb_un_ns; s_wb_held = wb_held, s_wb_held_ns = wb_held_ns; + for (int i = 0; i < ::mkwvk::wb_rec_wait::kCallers; i++) s_wb_by[i] = wb_by[i]; s_gpu_ns = st_wait_ns; s_ps = ps, s_pw = pw, s_pwms = pwms, s_up = up, s_upb = upb, s_imp = imp, s_imp_ns = imp_ns; s_pr = pr, s_pr_ns = pr_ns, s_pa = pa, s_later = later; @@ -6010,6 +7063,7 @@ void do_present(const nvn_tex* t, const int crop[4]) { const int* show_crop = crop; // ((Triforce) a held frame's: the held picture's own) { std::lock_guard lk(replay_mu); + releases_apply_locked(); // ((Triforce, 11-22, MEM-85) RX_VK_RELEASE_QUEUE's releases, with no queue stopped) bench_frame(presents); SlowWatch slow("present"); frame_submits.clear(); @@ -6242,6 +7296,7 @@ void do_present(const nvn_tex* t, const int crop[4]) { // generation rate) DLSS frame generation's frames go out inside this present, through Streamline: the frames it shows // for it, its multiplier, are what the screen gets; 1 while it does not generate) else if (!held) menu_note_frame(true, dlssg::multiplier()); + framegen_idle(interpolated); // ((Triforce, 11-21, PRESENT-18) after the queue: off for 120 presents, its images go) // (Triforce: a held frame is not counted, nothing new is shown; the window texture of a frame shown is noted for the // held picture of a hold that may follow) if (!held && tex && src && !dev.offscreen) { @@ -6871,6 +7926,58 @@ void present_thread_line() { } } // namespace +// ---- (Triforce) RX_VK_ACQUIRE_FENCE, on by default; =0: as before. The swapchain's image waited for on the CPU, not on the +// replay's queue: the acquire signals a fence, and present_image waits for it, at most 250 ms, before it records anything for +// the image; the blit's submit then waits for nothing. AMD's Windows driver returns from the acquire at once, the image not yet +// free: with the window covered the blit's wait for the acquire's semaphore held the queue, every later submit of the replay +// queued behind it, and the game froze for as long as the cover (its write-back waits timed out). An image not back in time +// is held: no frame is shown, the game goes on, and later presents look at it again (without waiting) instead of acquiring +// another. A swapchain made again abandons a held image's fence (a new one for the next acquire). With DLSS frame generation +// the semaphore as before: Streamline's acquire hook has not been tried with a fence +namespace { +bool acquire_fence_on() { + static const bool v = [] { + const char* e = getenv("RX_VK_ACQUIRE_FENCE"); + return !e || atoi(e) != 0; + }(); + return v; +} +struct HeldImage { + bool on = false; + uint32_t index = 0; + uint64_t generation = 0; // (dev.swap_generation of the swapchain it came from) + int presents = 0; // (the presents that found it not free yet) + bool logged = false; // (its hold logged: the first 16 are) +}; +HeldImage held_image; // (present_image_mu) +VkFence acquire_fence_now = VK_NULL_HANDLE; // (present_image_mu; unsignaled whenever no acquire signals it) +int held_logged = 0; +/// The fence the next acquire signals (made the first time, and after an abandoned one); VK_NULL_HANDLE: none could be made +VkFence acquire_fence() { + if (!acquire_fence_now) { + VkFenceCreateInfo fi{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; + if (vkCreateFence(dev.device, &fi, nullptr, &acquire_fence_now) != VK_SUCCESS) acquire_fence_now = VK_NULL_HANDLE; + } + return acquire_fence_now; +} +/// A held image of a swapchain made again since: let go, its fence abandoned (an acquire of the old swapchain may still +/// signal it, so it is neither reset nor destroyed) +void held_image_drop() { + held_image = {}; + acquire_fence_now = VK_NULL_HANDLE; +} +/// Whether the image just acquired, or the one held, is free: its fence waited for up to timeout_ns, and reset once signaled +/// for the next acquire. VK_TIMEOUT: not yet +VkResult image_free(uint64_t timeout_ns) { + VkResult r = vkWaitForFences(dev.device, 1, &acquire_fence_now, VK_TRUE, timeout_ns); + if (r == VK_SUCCESS) r = vkResetFences(dev.device, 1, &acquire_fence_now); + return r; +} +// ((Triforce, 11-22, the performance audit's PRESENT-45, after Source A's) a present on the present thread answered that the +// window's surface is lost (present_out): the next present_image, the thread that makes the swapchain, lets it go) +std::atomic surface_lost_noted{false}; +} // namespace + // (hookshot) One thread in here at a time; the queue's own lock (dev.present_mu, the replay's submits take it too) is held only // for the submit and the present. Held over vkAcquireNextImageKHR and the fence, it stopped the replay's submits for as long as // the swapchain had no image free, twice a frame with frame interpolation. @@ -6886,6 +7993,12 @@ void present_image(VkImage src, uint32_t sw, uint32_t sh, const int crop[4], VkI } timed; std::lock_guard lk(present_image_mu); present_thread_settle(); // ((Triforce) RX_VK_PRESENT_THREAD: the present handed to the present thread before this one out first) + // ((Triforce, 11-22, the performance audit's PRESENT-45, after Source A's) that present found the window's surface lost: + // what hangs on it is let go here, and the check below (no swapchain) makes a surface and a swapchain again) + if (surface_lost_noted.exchange(false, std::memory_order_acq_rel)) { + std::lock_guard qk(dev.present_mu); + invalidate_lost_surface(); + } static int hdr_was = -1; // (the Insert menu's HDR as the swapchain was made for it; -1: not yet looked at) const bool hdr_changed = hdr_was >= 0 && hdr_was != opts.hdr; // (turned on or off: the swapchain's format) if (hdr_was < 0) hdr_was = opts.hdr; @@ -6921,35 +8034,82 @@ void present_image(VkImage src, uint32_t sw, uint32_t sh, const int crop[4], VkI if (vkWaitForFences(dev.device, 1, &f.done, VK_TRUE, UINT64_MAX) == VK_ERROR_DEVICE_LOST) device_lost(); framelog_collect(log_slot); uint32_t index = 0; + // ((Triforce) RX_VK_ACQUIRE_FENCE: an image held from a swapchain made again since is let go; one still held is looked at + // again instead of acquiring another, and a new acquire signals the fence, not the semaphore, unless the switch is off or + // DLSS frame generation runs) + if (held_image.on && held_image.generation != dev.swap_generation) held_image_drop(); + bool fenced = held_image.on; + VkResult res = VK_SUCCESS; + if (held_image.on) { + index = held_image.index; + } else { #ifdef _WIN32 - const uint64_t acquire_timeout = UINT64_MAX; + const uint64_t acquire_timeout = UINT64_MAX; #else - // (linux: a Wayland compositor gives no image back while the window cannot be seen: its screen asleep, another workspace. A wait - // without end there held the presentation thread, and with it the game, until the screen came back. 100 ms, then the window is - // taken as not shown: frames are not waited for and not shown, and the game goes on at its speed, as on Windows, until an image - // comes back) - static bool unseen = false; - const uint64_t acquire_timeout = unseen ? 0 : 100000000ull; + // (linux: a Wayland compositor gives no image back while the window cannot be seen: its screen asleep, another workspace. A + // wait without end there held the presentation thread, and with it the game, until the screen came back. 100 ms, then the + // window is taken as not shown: frames are not waited for and not shown, and the game goes on at its speed, as on Windows, + // until an image comes back) + static bool unseen = false; + const uint64_t acquire_timeout = unseen ? 0 : 100000000ull; #endif - VkResult res = vkAcquireNextImageKHR(dev.device, dev.swapchain, acquire_timeout, f.acquired, VK_NULL_HANDLE, &index); - if (res == VK_ERROR_DEVICE_LOST) device_lost(); + const VkFence fence = acquire_fence_on() && !dlssg::active() ? acquire_fence() : VK_NULL_HANDLE; + fenced = fence != VK_NULL_HANDLE; + res = vkAcquireNextImageKHR(dev.device, dev.swapchain, acquire_timeout, fenced ? VK_NULL_HANDLE : f.acquired, fence, &index); + if (res == VK_ERROR_DEVICE_LOST) device_lost(); #ifndef _WIN32 - if (res == VK_TIMEOUT || res == VK_NOT_READY) { - if (!unseen) logf("vk: the window is not shown (no image back in 100 ms): frames are not shown until it is"); - unseen = true; - return; - } - if (unseen && (res == VK_SUCCESS || res == VK_SUBOPTIMAL_KHR)) { - unseen = false; - logf("vk: the window is shown again"); - } + if (res == VK_TIMEOUT || res == VK_NOT_READY) { + if (!unseen) logf("vk: the window is not shown (no image back in 100 ms): frames are not shown until it is"); + unseen = true; + return; + } + if (unseen && (res == VK_SUCCESS || res == VK_SUBOPTIMAL_KHR)) { + unseen = false; + logf("vk: the window is shown again"); + } #endif - if (res == VK_ERROR_OUT_OF_DATE_KHR) { - std::lock_guard qk(dev.present_mu); - create_swapchain(dev.swap_extent.width, dev.swap_extent.height); - return; + // ((Triforce, 11-22, the performance audit's PRESENT-45, after Source A's) the window's surface lost at the acquire + // (RX_VK_SURFACE_RECOVER: vk_main.cpp surface_recover): no image was given. What hangs on the surface is let go (the + // present before is out: present_thread_settle above), and the next present makes them again, as after a resize; this + // result was not handled before, so every later acquire failed and the picture stayed as it was) + if (surface_recover(res)) { + std::lock_guard qk(dev.present_mu); + invalidate_lost_surface(); + uint64_t none = 0; + dev.swap_resize.compare_exchange_strong(none, (uint64_t)dev.swap_extent.width << 32 | dev.swap_extent.height, + std::memory_order_release, std::memory_order_relaxed); + return; + } + if (res == VK_ERROR_OUT_OF_DATE_KHR) { + std::lock_guard qk(dev.present_mu); + create_swapchain(dev.swap_extent.width, dev.swap_extent.height); + return; + } + if (res != VK_SUCCESS && res != VK_SUBOPTIMAL_KHR) return; + } + if (fenced) { // ((Triforce) RX_VK_ACQUIRE_FENCE: the image free before anything is recorded for it; the wait is the pacing's) + const auto w0 = std::chrono::steady_clock::now(); + const VkResult ready = image_free(held_image.on ? 0 : 250000000ull); + if (ready == VK_ERROR_DEVICE_LOST) device_lost(); + present_fig.paced_ns.fetch_add( + std::chrono::duration_cast(std::chrono::steady_clock::now() - w0).count(), + std::memory_order_relaxed); + if (ready != VK_SUCCESS) { // (not free: held, this frame not shown) + if (!held_image.on) { + held_image = {true, index, dev.swap_generation, 0, held_logged < 16}; + if (held_image.logged) { + held_logged++; + logf("vk: swapchain image %u not free 250 ms after its acquire (the window covered?): held, frames not " + "shown until it is (RX_VK_ACQUIRE_FENCE)", index); + } + } + held_image.presents++; + return; + } + if (held_image.on && held_image.logged) + logf("vk: swapchain image %u free again after %d presents not shown", index, held_image.presents); + held_image = {}; } - if (res != VK_SUCCESS && res != VK_SUBOPTIMAL_KHR) return; if (vkResetFences(dev.device, 1, &f.done) == VK_ERROR_DEVICE_LOST) device_lost(); if (vkResetCommandPool(dev.device, f.pool, 0) == VK_ERROR_DEVICE_LOST) device_lost(); VkCommandBufferBeginInfo bi{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; @@ -6957,7 +8117,8 @@ void present_image(VkImage src, uint32_t sw, uint32_t sh, const int crop[4], VkI if (vkBeginCommandBuffer(f.cmd, &bi) == VK_ERROR_DEVICE_LOST) device_lost(); VkImage dst = dev.swap_images[index]; full_barrier(f.cmd); // (after the game's work for the frame: earlier submits on this queue) - // ((Triforce) RX_VK_CHECKPOINTS: the present's blit has markers: begun (after the image's acquire, which its submit waits for), + // ((Triforce) RX_VK_CHECKPOINTS: the present's blit has markers: begun (after the image's acquire, which its submit waits for + // unless the image was waited for on the CPU, RX_VK_ACQUIRE_FENCE), // the menu, and its end. The owner's device losses stopped between two frames, where the present had none: checkpoint_kinds.h) checkpoint_mark(f.cmd, index, 'p'); image_barrier(f.cmd, dst, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL); @@ -7048,7 +8209,8 @@ void present_image(VkImage src, uint32_t sw, uint32_t sh, const int crop[4], VkI checkpoint_mark(f.cmd, index, 'P'); // ((Triforce) RX_VK_CHECKPOINTS) if (vkEndCommandBuffer(f.cmd) == VK_ERROR_DEVICE_LOST) device_lost(); // after the image's acquisition. (Not on the timeline: one queue runs submits in order, and a wait for a value - // another thread had counted but not yet submitted blocked the queue for good) + // another thread had counted but not yet submitted blocked the queue for good). ((Triforce) RX_VK_ACQUIRE_FENCE: none when + // the image was waited for on the CPU: it is free already) VkSemaphoreSubmitInfo waits[1] = {{VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO}}; waits[0].semaphore = f.acquired; waits[0].stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT; @@ -7058,8 +8220,8 @@ void present_image(VkImage src, uint32_t sw, uint32_t sh, const int crop[4], VkI VkCommandBufferSubmitInfo cb{VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO}; cb.commandBuffer = f.cmd; VkSubmitInfo2 si{VK_STRUCTURE_TYPE_SUBMIT_INFO_2}; - si.waitSemaphoreInfoCount = 1; - si.pWaitSemaphoreInfos = waits; + si.waitSemaphoreInfoCount = fenced ? 0 : 1; + si.pWaitSemaphoreInfos = fenced ? nullptr : waits; si.commandBufferInfoCount = 1; si.pCommandBufferInfos = &cb; si.signalSemaphoreInfoCount = 1; @@ -7070,12 +8232,14 @@ void present_image(VkImage src, uint32_t sw, uint32_t sh, const int crop[4], VkI static int64_t presents_counted, st_flushes0, st_cbs0, r0[FR_COUNT]; if (st_flushes && ++presents_counted % 120 == 0) { logf("vk: batched submits: %lld flushes, %.1f command buffers each, over 120 presents; by reason: poll %lld, wait %lld, " - "slot %lld, present %lld, cap %lld, value %lld, other %lld", (long long)(st_flushes - st_flushes0), + "slot %lld, present %lld, cap %lld, value %lld, idle %lld, other %lld; %lld looks with nothing held, without the queue's " + "lock (RX_VK_PEND_HELD)", (long long)(st_flushes - st_flushes0), (double)(st_batched_cbs - st_cbs0) / std::max(1, st_flushes - st_flushes0), (long long)(st_reason[FR_POLL] - r0[FR_POLL]), (long long)(st_reason[FR_WAIT] - r0[FR_WAIT]), (long long)(st_reason[FR_SLOT] - r0[FR_SLOT]), (long long)(st_reason[FR_PRESENT] - r0[FR_PRESENT]), (long long)(st_reason[FR_CAP] - r0[FR_CAP]), (long long)(st_reason[FR_VALUE] - r0[FR_VALUE]), - (long long)(st_reason[FR_OTHER] - r0[FR_OTHER])); + (long long)(st_reason[FR_IDLE] - r0[FR_IDLE]), // ((Triforce, 11-22, SYNC-09) RX_VK_BATCH_IDLE's + (long long)(st_reason[FR_OTHER] - r0[FR_OTHER]), (long long)st_flush_lockfree.exchange(0, std::memory_order_relaxed)); st_flushes0 = st_flushes, st_cbs0 = st_batched_cbs; std::memcpy(r0, st_reason, sizeof r0); } @@ -7162,6 +8326,18 @@ static void present_out(const PresentOut& o, std::unique_lock& qk, b if (++frames % 60 == 0) std::fflush(fl); } } + // ((Triforce, 11-22, the performance audit's PRESENT-45, after Source A's) the window's surface lost at the present: + // let go here on the presenting thread; on the present thread only noted, for the next present_image (it waits for + // this present first, and it alone makes and lets go of the swapchain) + if (surface_recover(res)) { + if (on_thread) { + surface_lost_noted.store(true, std::memory_order_release); + return; + } + if (!qk.owns_lock()) qk.lock(); + invalidate_lost_surface(); + return; + } if (res == VK_ERROR_OUT_OF_DATE_KHR || res == VK_SUBOPTIMAL_KHR) { if (on_thread) { // ((Triforce) as a resize: unless a newer one is noted already, which the next present takes) uint64_t none = 0; @@ -7468,8 +8644,15 @@ void replay_thread_main() { // (on the core the host keeps free of the game's threads, hos_reserved_cpus: its last physical one) if (host->reserved_cpus) { if (const uint64_t m = host->reserved_cpus()) { - SetThreadAffinityMask(GetCurrentThread(), (DWORD_PTR)m); - logf("vk: the replay thread runs on logical processors %#llx, kept from the game's threads", (unsigned long long)m); + // ((Triforce, 11-22, THREAD-09, replay_pin.h) where the cores are of more than one kind it may also run on the + // other fastest ones, the reserved core its ideal processor: another program's thread there does not hold it up. + // RX_VK_REPLAY_PIN_SOFT=0: the reserved core alone, as before) + const uint64_t fast = replay_pin_soft(false), own = fast ? replay_pin_own() : 0; + const uint64_t on = own ? replay_pin::wide(m, fast, own) : m; + SetThreadAffinityMask(GetCurrentThread(), (DWORD_PTR)on); + if (on != m) replay_pin_set_ideal(replay_pin::ideal(m)); + logf("vk: the replay thread runs on logical processors %#llx, %#llx of them kept from the game's threads", + (unsigned long long)on, (unsigned long long)m); } } // ((Triforce) RX_VK_ASYNC_SPIN_US: the thread looks for its next job a moment before it sleeps (async_rules.h): a frame's @@ -7794,6 +8977,7 @@ void present(const nvn_tex* t, const int crop[4]) { if (sync_waits_on()) { if (vk_ft_on()) vk_ft("pdrain>"); drain(0); // (the frame's work whole before it is shown) + lab_ab::delay("present_after"); // ((Triforce, 11-22, REPLAY-51) the lab: d_present's control, just after the wait if (vk_ft_on()) vk_ft("pdrain<"); update_blocked(); present_value = ++q_values; @@ -7809,12 +8993,25 @@ void present(const nvn_tex* t, const int crop[4]) { void texture_released_of(const nvn_tex* t) { if (!t) return; tex_pages_add(t, -1); + texture_warm_gone(t); // ((Triforce, 11-22, TEX-23) and forgotten) if (!async_on()) { // (PR 430: after the stopped work that may use it, when it goes on. The lock is held_mu, only while queues are // stopped: sync_order_mu, taken on every release, made the thread finalizing textures (OptimizeChangeBank) wait for whole // submissions being replayed under it, 45% of its time in Kokiri. A release just after the stopped work went on waits // for the next update_blocked: the next submit or present) - if (hold_releases() && sync_waits_on() && blocked_n.load()) { + // ((Triforce, 11-22, the performance audit's MEM-85) RX_VK_RELEASE_QUEUE=1: to the replaying thread, stopped queue or not, + // with this call's write generation; unset, way() is the old decision: release_queue.h) + const release_queue::Way way = release_queue::way(release_queue_on(), hold_releases(), sync_waits_on(), blocked_n.load() != 0); + if (way == release_queue::Way::Queued) { + { + std::lock_guard hl(held_mu); + queued_tex_releases.push_back({*t, memory_generation_any_thread()}); + queued_releases.fetch_add(1, std::memory_order_release); + } + release_count(-1); + return; + } + if (way == release_queue::Way::Held) { { std::lock_guard hl(held_mu); held_tex_releases.push_back(*t); @@ -7822,8 +9019,11 @@ void texture_released_of(const nvn_tex* t) { if (++st_held_releases % 2000 == 1) logf("vk: releases kept for stopped work: %lld", (long long)st_held_releases.load()); return; } + const auto r0 = std::chrono::steady_clock::now(); // ((Triforce, 11-22, MEM-85) the old way's time, counted) sync_drain(1); // (after the stopped work that may use it) - return do_texture_released(t); + do_texture_released(t); + release_count(std::chrono::duration_cast(std::chrono::steady_clock::now() - r0).count()); + return; } Job j{JOB_TEXTURE_RELEASED}; j.tex = *t, j.has_tex = true; @@ -7832,7 +9032,18 @@ void texture_released_of(const nvn_tex* t) { void program_released_locked(const nvn_prog* p) { if (!p) return; if (!async_on()) { - if (hold_releases() && sync_waits_on() && blocked_n.load()) { // (PR 430: as texture releases, on held_mu) + // ((Triforce, 11-22, MEM-85) RX_VK_RELEASE_QUEUE=1: to the replaying thread, as texture releases) + const release_queue::Way way = release_queue::way(release_queue_on(), hold_releases(), sync_waits_on(), blocked_n.load() != 0); + if (way == release_queue::Way::Queued) { + { + std::lock_guard hl(held_mu); + queued_prog_releases.push_back(*p); + queued_releases.fetch_add(1, std::memory_order_release); + } + release_count(-1); + return; + } + if (way == release_queue::Way::Held) { // (PR 430: as texture releases, on held_mu) { std::lock_guard hl(held_mu); held_prog_releases.push_back(*p); @@ -7840,8 +9051,11 @@ void program_released_locked(const nvn_prog* p) { st_held_releases++; return; } + const auto r0 = std::chrono::steady_clock::now(); // ((Triforce, 11-22, MEM-85) the old way's time, counted) sync_drain(1); - return do_program_released(p); + do_program_released(p); + release_count(std::chrono::duration_cast(std::chrono::steady_clock::now() - r0).count()); + return; } Job j{JOB_PROGRAM_RELEASED}; j.prog = *p; @@ -7871,9 +9085,18 @@ void register_texture_queued(int id, const nvn_tex* t, const nvn_view* v) { } void event_signal_queued(uint64_t gpu, uint32_t value) { if (!async_on()) { - std::lock_guard lk(ev_cpu_mu); - ev_cpu_pending.emplace_back(gpu, value); - ev_cpu_any.store(true, std::memory_order_release); + { + std::lock_guard lk(ev_cpu_mu); + ev_cpu_pending.emplace_back(gpu, value); + ev_cpu_any.store(true, std::memory_order_release); + } + // ((Triforce, 11-22, SYNC-29) counted for the reads' gate, AFTER it is queued: a look that notes this count finds the + // signal in the list. Counted first, a look could publish a signal it never saw + ev_cpu_signals.fetch_add(1, std::memory_order_release); + // ((Triforce, 11-22, SYNC-59) a queue is stopped: the order thread is woken to look now, not at its next sleep's + // end. After the store, with ev_cpu_mu let go: the thread it wakes takes that lock to read the signal) + if (blocked_n.load(std::memory_order_relaxed) && order_wake_on()) + if (const HANDLE wake = order_wake_event.load(std::memory_order_acquire)) SetEvent(wake), st_wake_sets.fetch_add(1, std::memory_order_relaxed); return; // (sync_watch or the next submit lets the stopped ones on: not this thread, ~1.7 ms a frame of the critical path) } Job j{JOB_EVENT_SIGNAL}; @@ -8059,7 +9282,9 @@ void fill_replay_table(MkwVkRenderer& t) { if (id == HOS_OPT_SHARE_ASK) return menu_take_share_ask(); // (Triforce: the System tab's save rows, host/save_share.c) if (id == HOS_OPT_BUG_SHOT) return menu_bug_shot(); // (and a bug capture's picture: host/bug_capture.c) if (id == HOS_OPT_MODCHEAT_ASK) return menu_take_mod_cheat(); // (Triforce: a mod's cheat switched on its row, host/mod_cheats.c) + if (id == HOS_OPT_ACH_ASK) return 1; // (Triforce: this renderer shows the achievements: host/achievements.c names them) if (id == HOS_OPT_MODS_SELECTED) return menu_mods_selected(); // (Triforce: the Mods entry has the cursor, host/oot_menu.c) + if (id == HOS_OPT_MODS_MUSIC) return menu_mods_music(); // ((Triforce, 11-20) the Mods screen's music, host/mods_music.c) if (id == HOS_OPT_TEXT_WANTED) return menu_text_wanted(); // (Triforce: the Mods screen's search takes text, host/display.c) return id == 0 ? opts.realtime : id == 1 ? menu_is_open() : id == 2 ? resolution_auto() && !temporal_full_scale() : id == 3 ? (temporal_full_scale() ? 10000 : resolution_scale()) : id == 4 ? opts.pad_background : id == 5 ? opts.text_speed : id == 6 ? opts.mouse_native : id == 7 ? opts.mouse_sens : id == 8 ? opts.mouse_invert : id == 9 ? opts.rumble : id == 16 ? opts.invert_cam_x : id == 17 ? opts.invert_cam_y : id == HOS_OPT_GYRO ? opts.gyro : id == HOS_OPT_GYRO_SENS_X ? opts.gyro_sens_x : id == HOS_OPT_GYRO_SENS_Y ? opts.gyro_sens_y : id == HOS_OPT_GYRO_INV_X ? opts.gyro_inv_x : id == HOS_OPT_GYRO_INV_Y ? opts.gyro_inv_y : id == HOS_OPT_GYRO_DEADZONE ? opts.gyro_deadzone : @@ -8143,3 +9368,15 @@ __attribute__((visibility("default"))) int mkwvk_replay_idle_wait(uint64_t timeout_ns) { return mkwvk::replay_idle_wait(timeout_ns) ? 1 : 0; } +// ((Triforce, 11-22, the performance audit's THREAD-25) host/rx_backend.c's rx_nvn_invalidate asks first, found by name as the two +// above (MkwVkRenderer stays as it is; an older executable never asks): 1 when the invalidate has nothing to wait for and need not +// be made. invalidate_quick has the rule +extern "C" +#ifdef _WIN32 +__declspec(dllexport) +#else +__attribute__((visibility("default"))) +#endif +int mkwvk_invalidate_quick(uint64_t gpu, uint64_t size) { + return mkwvk::invalidate_quick(gpu, size); +} diff --git a/gpu/vk/vk_salesman.cpp b/gpu/vk/vk_salesman.cpp index 9056b8e..7c153fb 100644 --- a/gpu/vk/vk_salesman.cpp +++ b/gpu/vk/vk_salesman.cpp @@ -1,5 +1,6 @@ // (Triforce) The Happy Mask Salesman on the Mods page (vk_salesman.h): his files, his poses, and a small rasterizer. #include "vk_salesman.h" +#include "cpu_count.h" // ((Triforce, 11-22, OS-49) the threads to size by: HOS_SIM_CPU #include #include @@ -892,7 +893,7 @@ void draw(Model& md, float secs, float lift, int out_w, int out_h, Frame& out) { } } } - const int bands = std::clamp((int)std::thread::hardware_concurrency() - 1, 1, 6); + const int bands = std::clamp((int)cpu_count() - 1, 1, 6); std::vector th; for (int bi = 0; bi < bands; bi++) { const int ry0 = rs.H * bi / bands, ry1 = rs.H * (bi + 1) / bands; diff --git a/gpu/vk/vk_shader.cpp b/gpu/vk/vk_shader.cpp index 1a57f05..f462858 100644 --- a/gpu/vk/vk_shader.cpp +++ b/gpu/vk/vk_shader.cpp @@ -781,6 +781,47 @@ void exact_note(VkShaderModule m, const std::vector& code) { else if (!exact_of.erase(m)) return; // (a handle used again; nothing kept and nothing there: no thread's answer changes) exact_gen.fetch_add(1, std::memory_order_release); } +// ((Triforce, 11-22, the performance audit's PIPE-52, after Source A's) The modules looked at and found without a barrier of +// Subgroup scope (compute_pin.h subgroup_barrier): the lowering's barrier between a warp's shared-memory store and load takes +// the subgroup to be the warp, and no flag of Shader::Info records it. Only where RX_VK_COMPUTE_PIN is not "all" (Mesa's +// Intel driver by default): with "all" no module is looked at and nothing is kept. A handle the driver uses again is noted +// again by make_module, and forgotten by stage_module_destroy +// ((Triforce, 11-22, the performance audit's PIPE-56, after Source A's) A compute module's OpPhi and OpFunctionCall counts, +// taken here while the SPIR-V is in hand (program_pack may drop the module) and only on the two drivers that have a limit +// (fallback_size_policy.h: RADV on GCN 1 to 4, Mesa's Intel driver): on every other one nothing is counted or kept. Moved +// with the stage's key by program_pack as exact_of's entry is, forgotten by stage_module_destroy +std::mutex size_mu; +std::unordered_map size_of; +void size_note(VkShaderModule m, const std::vector& code) { + if (!m || fallback_size_driver() == fallback_size::Driver::Other) return; + const bool compute = compute_pin::compute_entry(code.data(), code.size()); + const fallback_size::Size s = compute ? fallback_size::count(code.data(), code.size()) : fallback_size::Size{}; + std::lock_guard lk(size_mu); + if (compute) size_of[m] = s; + else size_of.erase(m); // (a handle used again by a graphics stage's module) +} +void size_move(VkShaderModule from, VkShaderModule to) { + if (fallback_size_driver() == fallback_size::Driver::Other) return; + std::lock_guard lk(size_mu); + const auto it = size_of.find(from); + if (it == size_of.end()) { + size_of.erase(to); + return; + } + const fallback_size::Size s = it->second; + size_of.erase(it); + size_of[to] = s; +} +std::mutex lanes_mu; +std::unordered_set lanes_free; +void lanes_note(VkShaderModule m, const std::vector& code) { + if (!m || compute_pin_mode() == compute_pin::Mode::All) return; + // (a graphics stage's module is never asked about: nothing kept for it, and a handle of an earlier compute module's forgotten) + const bool barrier = !compute_pin::compute_entry(code.data(), code.size()) || compute_pin::subgroup_barrier(code.data(), code.size()); + std::lock_guard lk(lanes_mu); + if (barrier) lanes_free.erase(m); + else lanes_free.insert(m); +} // ---- (Triforce) RX_VK_SPV_PACK: a ready program's stages keep their SPIR-V packed and no module (spv_store.h, where the // reasons and the measurements are). make_module packs the words it hands to the driver and keeps the store beside the // module's handle (pack_note); program_pack, called where a program becomes ready, gives each stage its store and destroys @@ -870,6 +911,7 @@ void program_pack(Program& p) { } if (p.failed || p.compute) continue; // (its store is let go, its module stays) exact_move(st.module, (VkShaderModule)(uintptr_t)code.get()); + size_move(st.module, (VkShaderModule)(uintptr_t)code.get()); // ((Triforce, 11-22, PIPE-56) nothing off the two drivers) module_forget(st.module); vkDestroyShaderModule(dev.device, st.module, nullptr); st.module = VK_NULL_HANDLE; @@ -1109,12 +1151,16 @@ VkShaderModule make_module(const std::vector& spirv, uint32_t* renodx vkCreateShaderModule(dev.device, &ci, nullptr, &m); module_note(m, salted.size()); exact_note(m, code); + lanes_note(m, code); + size_note(m, code); pack_note(m, salted.data(), salted.size()); return m; } vkCreateShaderModule(dev.device, &ci, nullptr, &m); module_note(m, code.size()); exact_note(m, code); + lanes_note(m, code); // ((Triforce, 11-22, PIPE-52) nothing with RX_VK_COMPUTE_PIN "all", the default off Mesa's Intel driver) + size_note(m, code); // ((Triforce, 11-22, PIPE-56) nothing but on RADV for GCN 1 to 4 and Mesa's Intel driver) pack_note(m, code.data(), code.size()); return m; } @@ -1134,9 +1180,45 @@ bool spv_pack_on() { static const bool on = !(getenv("RX_VK_SPV_PACK") && atoi(getenv("RX_VK_SPV_PACK")) == 0); return on; } +compute_pin::Mode compute_pin_mode() { + static const compute_pin::Mode mode = [] { + const char* env = getenv("RX_VK_COMPUTE_PIN"); + const compute_pin::Mode m = compute_pin::mode_for(env, dev.driver_id == VK_DRIVER_ID_INTEL_OPEN_SOURCE_MESA); + // (no line where it is "all" by default: every driver but Mesa's Intel one, whose log is what it was) + if (m != compute_pin::Mode::All || env) + logf("vk: compute programs held to 32 lanes: %s (RX_VK_COMPUTE_PIN=all, lanes or shared; lanes: only the programs that " + "can tell, the default on Mesa's Intel driver)", compute_pin::name(m)); + return m; + }(); + return mode; +} +bool compute_lanes_free(VkShaderModule m) { + if (!m || compute_pin_mode() == compute_pin::Mode::All) return false; + std::lock_guard lk(lanes_mu); + return lanes_free.count(m) != 0; +} +fallback_size::Driver fallback_size_driver() { + static const fallback_size::Driver d = + fallback_size::driver_of(dev.radv_gcn, dev.driver_id == VK_DRIVER_ID_INTEL_OPEN_SOURCE_MESA); + return d; +} +fallback_size::Size stage_size(const Stage& st) { + if (fallback_size_driver() == fallback_size::Driver::Other) return fallback_size::Size{}; + std::lock_guard lk(size_mu); + const auto it = size_of.find(stage_exact_key(st)); + return it == size_of.end() ? fallback_size::Size{} : it->second; +} void stage_module_destroy(VkShaderModule m) { if (!m) return; module_forget(m); + if (fallback_size_driver() != fallback_size::Driver::Other) { // ((Triforce, 11-22, PIPE-56) its counts go with it) + std::lock_guard lk(size_mu); + size_of.erase(m); + } + if (compute_pin_mode() != compute_pin::Mode::All) { // ((Triforce, 11-22, PIPE-52) its note goes with it: lanes_note) + std::lock_guard lk(lanes_mu); + lanes_free.erase(m); + } if (spv_pack_on()) { // (a store made for it that no program took: let go with it) std::lock_guard lk(pack_mu); pack_made.erase(m); @@ -3818,7 +3900,9 @@ size_t program_cache_load_some(const std::unordered_set& wanted, const for (auto it = disk_records.begin(); it != disk_records.end();) { // (fallbacks: and every program translated for the mesh fallback (kind 4), not only those play has drawn, whatever the // compute list says: their compute pipelines, made in play, took AMD up to 12 s each on first sight, the foliage and the - // door white until then. Only a GPU without mesh shaders has such programs) + // door white until then. (Triforce, 11-22, PIPE-62) Such programs are those of a GPU on the task and mesh fallback: one + // without mesh shaders, an RX 6000 or an RDNA2 integrated GPU on Windows, the menu's Compute row, + // RX_VK_TASK_MESH_FALLBACK=1. Only the records still on the disk are found here: the walk at the end takes the rest) if (wanted.count(it->first) || (!it->second.empty() && ((computes && (it->second[0].kind & 1)) || (fallbacks && (it->second[0].kind & 4))))) { work.push_back({it->first, std::move(it->second)}); @@ -3854,6 +3938,28 @@ size_t program_cache_load_some(const std::unordered_set& wanted, const for (auto& p : made[i]) variants.push_back(std::move(p)); for (auto& p : variants) n++, fn(*p); } + // ((Triforce, 11-22, the performance audit's PIPE-62, after Source A's) And the mesh fallback's programs that are in memory + // already, ready and not failed. The loop above hands over only what was still on the disk, and in a start that prepares, + // state_list_prepare has taken the state lists' records before the precompile asks (with fallbacks off): most of the + // fallback programs play draws were then never handed to the precompile, got no compute pipelines before the title, and + // each one's first draw waited up to 50 ms on the recording thread and was left out until its pipelines were built + // (Source A, a preparing start on the fallback: 850 programs ready where a later start had about 2,096). The precompile's + // worker takes a program's compute_state before it builds, as a draw does, so none is built twice. Under shader_mu, as the + // loop above. RX_VK_FALLBACK_LOADED=0: not, as before (read once) + if (fallbacks) { + static const bool loaded = !(getenv("RX_VK_FALLBACK_LOADED") && atoi(getenv("RX_VK_FALLBACK_LOADED")) == 0); + if (loaded) { + std::unordered_set taken; + for (const auto& w : work) taken.insert(w.first); + size_t found = 0; + for (auto& kv : programs) { + if (taken.count(kv.first)) continue; + for (auto& p : kv.second) + if (p->fallback && !p->failed && p->ready.load(std::memory_order_acquire)) found++, n++, fn(*p); + } + if (found) logf("vk: %zu mesh fallback programs read in earlier in this start are made ready with the rest", found); + } + } return n; } /// ((Triforce, 11-19) water_quality_mode for vk_draw.cpp's warmed stamp (module_patches): where make_module simplifies the water's diff --git a/gpu/vk/vk_shader.h b/gpu/vk/vk_shader.h index 8e08d8e..8aa4d09 100644 --- a/gpu/vk/vk_shader.h +++ b/gpu/vk/vk_shader.h @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -13,6 +14,8 @@ #include "shader_recompiler/shader_info.h" #include "spv_store.h" +#include "compute_pin.h" // ((Triforce, 11-22, PIPE-52) which compute programs are held to 32 lanes: compute_pin_mode) +#include "fallback_size_policy.h" // ((Triforce, 11-22, PIPE-56) the mesh fallback's stages too big for two Mesa drivers) #include "vk_internal.h" #include "vk_state.h" @@ -65,6 +68,16 @@ inline VkShaderModule stage_exact_key(const Stage& st) { } /// Whether stages are packed in this run (on unless RX_VK_SPV_PACK=0) bool spv_pack_on(); +/// ((Triforce, 11-22, the performance audit's PIPE-52) RX_VK_COMPUTE_PIN on this driver (compute_pin.h; read once, logged once +/// where it is not "all"), and whether a compute module was looked at and has no barrier of Subgroup scope (false for a +/// module nobody looked at, and always with "all", where no module is looked at) +compute_pin::Mode compute_pin_mode(); +bool compute_lanes_free(VkShaderModule m); +/// ((Triforce, 11-22, the performance audit's PIPE-56) Which of the two drivers with a limit on a mesh-fallback stage's size +/// this is (fallback_size_policy.h; Other on every other driver, where no module is counted), and a stage's OpPhi and +/// OpFunctionCall counts as make_module took them (0 and 0 for a stage nobody counted). Any thread +fallback_size::Driver fallback_size_driver(); +fallback_size::Size stage_size(const Stage& st); /// (Triforce) A program stage's module let go (vkDestroyShaderModule), and forgotten by the memory line's count (mem_parts.h): /// every place that lets a program's module go does it through this void stage_module_destroy(VkShaderModule m); @@ -77,6 +90,12 @@ struct Program { uint64_t key = 0; bool compute = false, failed = false; std::atomic ready{false}; // translated (graphics programs are translated on a worker: vk_jobs.h) + // ((Triforce, 11-22, the performance audit's DESC-22, after Source A's; desc_counts.h) Its descriptor counts as + // build_descriptors sums them (buffers, images, texel views), packed: ~0u until its first build once it is ready. + // It sits in the four bytes of padding before stages, and THIS FIELD MUST NOT GROW THE STRUCT: sizeof(Program) is in + // the program cache's stamp (vk_shader.cpp disk_stamp), and a change of it throws every player's shader and pipeline + // caches away (it did on 11-18, 272 to 280: vk_draw.cpp). The static_assert under the struct holds it there) + mutable std::atomic desc_n{~0u}; std::vector stages; std::array workgroup{1, 1, 1}; uint32_t shared_memory = 0; @@ -121,6 +140,10 @@ struct Program { std::vector questions; uint64_t codes[4]{}; // (hookshot) its stages' code keys: vertex (mesh), fragment, task, the header-only stage (the state list) }; +// ((Triforce, 11-22, DESC-22) desc_n in the padding between ready and stages, where it changes no size: see the field) +static_assert(sizeof(std::atomic) == 4 && offsetof(Program, desc_n) + 4 <= offsetof(Program, stages) && + offsetof(Program, stages) == 16, + "Program::desc_n must fit the padding before Program::stages: sizeof(Program) is in the program cache's stamp"); enum class ProgramKind { Graphics, Compute, Mesh }; diff --git a/gpu/vk/vk_temporal.cpp b/gpu/vk/vk_temporal.cpp index f58c133..efc91d6 100644 --- a/gpu/vk/vk_temporal.cpp +++ b/gpu/vk/vk_temporal.cpp @@ -55,9 +55,11 @@ #include #include +#include "capture_tables.h" // ((Triforce, the upscaler audit's UPS-02) the stored draws' tables, kept from frame to frame) #include "dlss_actor.h" #include "dlss_checks.h" #include "dlss_math.h" +#include "redraw_state.h" // ((Triforce, the upscaler audit's UPS-03) what a redraw still has to set) #include "framegen_rows.h" // ((Triforce) pose_of, camera_cut: a camera cut starts a new history) #include "fsr3_math.h" // ((Triforce, FSR4-10) jitter_phases: AMD's jitter length, for FSR 4 too) #include "stream_order.h" @@ -334,8 +336,12 @@ struct State { bool capture_camera_own = false; uint64_t capture_unjittered = 0; // (stored draws whose own camera's jitter is not the scene's, since the last status line) std::vector captured; - std::unordered_map captured_models; // (by model: its draws so far this frame, for the key's nth) - std::unordered_map captured_before; + // ((Triforce, 11-21, the upscaler audit's UPS-02) both keep their memory from frame to frame (capture_tables.h): as + // std::unordered_map they took a node from the heap for every stored draw every frame, and gave it back the frame after. + // RX_VK_CAPTURE_TABLES=0: as before + const bool capture_tables_kept = capture_tables::kept(getenv("RX_VK_CAPTURE_TABLES")); + capture_tables::Nth captured_models{capture_tables_kept}; // (by model: its draws so far this frame, for the key's nth) + capture_tables::Before captured_before{capture_tables_kept}; uint64_t captured_frame = 0; struct CaptureBuffer { VkBuffer buffer = VK_NULL_HANDLE; @@ -966,6 +972,9 @@ std::vector take_captures() { State& s = st(); std::vector c = std::move(s.captured); s.captured.clear(), s.captured_models.clear(); + // ((Triforce, UPS-02) the next frame's list at this frame's size from its start: moved from, it grew from nothing again + // every frame, a dozen times over) + s.captured.reserve(capture_tables::room(c.size(), s.capture_tables_kept)); // (Triforce: in stream order and named again, as the skinned draws are in actor_motion) stream_order::renumber(c, [](const State::Captured& k) { return k.k.stream_pos; }, [](State::Captured& k) -> uint64_t& { return k.key; }); @@ -1000,13 +1009,12 @@ int capture_motion(VkCommandBuffer cmd, const Camera& cam, const float* before, std::chrono::steady_clock::time_point t0 = std::chrono::steady_clock::now(); ~Timed() { s.actor_ns += (std::chrono::steady_clock::now() - t0).count(); } } timed{s}; - std::unordered_map old = std::move(s.captured_before); - s.captured_before.clear(); + s.captured_before.turn(); // ((Triforce, UPS-02) what the last call put is the frame before's now: before()) // (the frame before's are of the frame the engine last drew: else a gap, and their positions are older) const bool old_ok = history && s.captured_frame && s.captured_frame == to.last; s.captured_frame = texture_frame_no(); for (const auto& c : caps) - if (c.k.depth == cam.depth) s.captured_before[c.key] = c; + if (c.k.depth == cam.depth) s.captured_before.put(c.key, c); // (RX_VK_DLSS_TRACE=: on vk_draw.cpp's traced frames, each stored draw left out here and why) static const int trace_every = env_switch("RX_VK_DLSS_TRACE"); const uint64_t now = texture_frame_no(); @@ -1028,8 +1036,8 @@ int capture_motion(VkCommandBuffer cmd, const Camera& cam, const float* before, left(c, "another depth target than the scene's"); continue; } - const auto it = old_ok ? old.find(c.key) : old.end(); - if (it == old.end()) { + const State::Captured* const was = old_ok ? s.captured_before.before(c.key) : nullptr; + if (!was) { s.actor_skipped++; left(c, !old_ok ? "no history" : "not drawn the frame before (its key)"); continue; @@ -1044,7 +1052,7 @@ int capture_motion(VkCommandBuffer cmd, const Camera& cam, const float* before, continue; } } - const State::Captured& o = it->second; + const State::Captured& o = *was; dlss_actor::CapturePush push{c.address, o.address, c.first, o.first, c.count, o.count, {}, {}}; for (int i = 0; i < 2; i++) { push.jitter_now[i] = c.own ? c.jitter[i] : cam.jitter[i]; @@ -1131,14 +1139,21 @@ int capture_motion(VkCommandBuffer cmd, const Camera& cam, const float* before, } // (the positions the scene's draws stored: the prep pass's barrier is after them) if (!begin_redraw(cmd, s.capture, cam, before, to.jitter_before, reproject, rw, rh, dview, to.motion)) return 0; + // ((Triforce, 11-21, the upscaler audit's UPS-03) the topology, the restart and the index buffer only where a draw's + // differ from what the pass has set (redraw_state.h): each draw set all three before, three commands a draw of which + // nearly all said what was set already. RX_VK_REDRAW_STATE=0: every draw sets them, as before + static const bool set_every = redraw_state::every(getenv("RX_VK_REDRAW_STATE")); + redraw_state::Pass set; + set.all = set_every; for (const Redraw& d : draws) { // (the same indices, base vertex and topology: VertexIndex names the same vertex) const CaptureDraw& k = d.c->k; vkCmdPushConstants(cmd, s.actor_layout, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof d.push, &d.push); - vkCmdSetPrimitiveTopology(cmd, k.topology); - vkCmdSetPrimitiveRestartEnable(cmd, k.restart ? VK_TRUE : VK_FALSE); + if (redraw_state::topology(set, (uint32_t)k.topology)) vkCmdSetPrimitiveTopology(cmd, k.topology); + if (redraw_state::restart(set, k.restart)) vkCmdSetPrimitiveRestartEnable(cmd, k.restart ? VK_TRUE : VK_FALSE); if (k.itype != dlss_actor::kNoIndices) { const VkIndexType type = k.itype == 0 ? VK_INDEX_TYPE_UINT8_EXT : k.itype == 1 ? VK_INDEX_TYPE_UINT16 : VK_INDEX_TYPE_UINT32; - vkCmdBindIndexBuffer(cmd, d.ib.buffer, d.ib.offset, type); + if (redraw_state::index(set, (uint64_t)d.ib.buffer, (uint64_t)d.ib.offset, (uint32_t)type)) + vkCmdBindIndexBuffer(cmd, d.ib.buffer, d.ib.offset, type); vkCmdDrawIndexed(cmd, (uint32_t)k.count, 1, 0, k.base_vertex, 0); } else { vkCmdDraw(cmd, (uint32_t)k.count, 1, (uint32_t)k.base_vertex, 0); @@ -1498,7 +1513,7 @@ void temporal_capture_draw(const CaptureDraw& k, CaptureSlot* out) { } // (its name across frames: as the actor motion's, the n-th draw of a model this frame the n-th the frame before) const uint64_t model = dlss_actor::key(k.vbuf, k.ibuf, k.count, k.base_vertex, k.program, 0); - const int nth = s.captured_models[model >> 8]++; + const int nth = s.captured_models.next(model >> 8); State::Captured c; c.k = k, c.key = dlss_actor::key(k.vbuf, k.ibuf, k.count, k.base_vertex, k.program, nth); c.address = b.address + s.capture_used, c.first = first, c.count = n; diff --git a/gpu/vk/vk_texture.cpp b/gpu/vk/vk_texture.cpp index 95a48f1..8032c38 100644 --- a/gpu/vk/vk_texture.cpp +++ b/gpu/vk/vk_texture.cpp @@ -18,6 +18,7 @@ #include #if defined(__linux__) #include +#include // ((Triforce) the guest memfd: upload_all_zero) #endif #include #include @@ -25,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -52,6 +54,10 @@ #include "tex_bc1.h" #include "touch_quick.h" #include "tex_warm.h" // (Triforce: textures made and uploaded ahead in light frames, texture_warm_step) +#include "file_whole.h" // ((Triforce, 11-22, TEX-23) the list of what was drawn, saved whole: warm_list_save) +#include "vk_paths.h" +#include "cpu_count.h" // ((Triforce, 11-22, OS-49) the threads to size by: HOS_SIM_CPU +#include "linux_pressure.h" // ((Triforce, 11-22) the performance audit's MEM-141: the swap hold's valves here (arenas, malloc_trim, budget) #include "tex_detail.h" #include "vram_compact.h" #include "texture_gone.h" @@ -66,6 +72,21 @@ #include "upload_earliest.h" // ((Triforce) a parallel replay's upload into the earliest segment still recorded) #include "pass_barrier.h" // ((Triforce) the rule of the barrier a draw pass starts with) #include "redrawn.h" // ((Triforce) whether a target is drawn again frame after frame: the new-texture budget's rule) +#include "lab_ab.h" // ((Triforce, lab) in-run A/B switches, RX_VK_LAB_AB: the texture budget's noexcess) +#include "guest_holes.h" // ((Triforce) the all-zero check reads the guest memfd's data only: upload_all_zero) +#include "bc_scratch.h" // ((Triforce, 11-21, TEX-10) the BC3 scratch's layers: a run's, not 8) +#include "target_pool.h" // ((Triforce, 11-21, TEX-29) the render targets' pool: its bound, its order and its tiers) +#include "wrote_regions.h" // ((Triforce, 11-21, TEX-02) the regions a GPU write ever reached: "not written" without the lock) +#include "drawn_page_map.h" // ((Triforce, 11-21, TEX-21) a drawn texture's page map recorded at its first draw +#include "dead_count.h" // ((Triforce, 11-21, THREAD-21's first step) what destroying the dead images costs a present: counted +#include "finalized_obsolete.h" // ((Triforce, 11-21, TEX-43) a finalized target whose storage the CPU has rewritten goes) +#include "slot_retire.h" // ((Triforce, 11-21, TEX-76) a depth target another size replaced at its start: let go after 16 presents) +#include "tex_count.h" // ((Triforce, 11-22, TEX-86) an image in the game's own BC or ASTC blocks counted by whole blocks) +#include "evict_reason.h" // ((Triforce, 11-22, TEX-R404) an eviction's write-back over storage another image drew into since +#include "staging_keep.h" // ((Triforce, 11-22, TEX-28) the idle staging arenas a load's burst keeps +#include "page_marks.h" // ((Triforce, 11-22, THREAD-25) the pages a GPU-written texture was ever listed in, asked without a lock: vw_note) +#include "new_wait.h" // ((Triforce, 11-22, OS-04) a nothrow new that must answer at once: NoWait) +#include "commit_room.h" // ((Triforce, 11-22, OS-05) the textures near Windows' commit limit, the page files' growth: commit_now) #include "host_shaders/block_linear_unswizzle_2d_comp_spv.h" #include "shaders/swizzle_back_comp_spv.h" #include "vk_work.h" // (after the others: counts the commands that do GPU work, for full_barrier) @@ -260,16 +281,23 @@ std::deque dead; // shows. Only images made for a target (texture_get's !for_sampling) are pooled. The pool is bounded (RX_VK_TARGET_POOL_MB, default // 256; 0: none, every image destroyed as before; and none while VRAM is near what the driver allows), and an image unused in it for // 120 presents is destroyed, at most 16 a present ---- +// ((Triforce, 11-21, the performance audit's TEX-29) the bound by the allowance, the trim oldest first through one list in +// the order the images came in, a short wait for shapes nothing takes again, and tiers by the VRAM in use: target_pool.h struct Pooled { + uint64_t key; // (pool_key_of: its shape) VkImage image; VmaAllocation allocation; std::unordered_map views; uint64_t bytes, since; }; std::mutex pool_mu; -std::unordered_map> target_pool; // by pool_key_of +std::list pool_age; // (every pooled image, the one that has waited longest at the front) +std::unordered_map::iterator>> target_pool; // by pool_key_of: a shape's, oldest first +std::unordered_set pool_taken; // (the shapes taken from the pool lately, target_pool::kTakenKeys of them) +std::deque pool_taken_order; uint64_t pool_bytes, pool_n, st_pool_reused, st_pool_made, st_pool_destroyed; bool pool_pressed; // (VRAM near what the driver allows: texture_frame's budget) +int pool_tier; // ((Triforce, 11-21, TEX-29) 0, 1 above 90% of the allowance, 2 above 97%: target_pool::tier) bool compact_on() { // (hookshot, RX_VK_COMPACT=1: the textures in a pool of their own, compacted: compact_step) // (Triforce) on by default on NVIDIA cards (RX_VK_COMPACT=0: off; =1: on, any card). Measured there: an RTX 4070 SUPER // played with it since 2026-11-10 (a 584,000-present session, 132 compactions, no fault; 85 -> 80.6 fps after a six-area @@ -289,7 +317,16 @@ bool compact_on() { // (hookshot, RX_VK_COMPACT=1: the textures in a pool of t return v; } uint64_t pool_cap() { - static const uint64_t mb = getenv("RX_VK_TARGET_POOL_MB") ? (uint64_t)std::max(0, atoi(getenv("RX_VK_TARGET_POOL_MB"))) : 256; + // ((Triforce, 11-21, TEX-29) with no switch, a twelfth of what the driver allows, 256 to 768 MB: target_pool.h) + static const uint64_t mb = [] { + if (const char* e = getenv("RX_VK_TARGET_POOL_MB")) return (uint64_t)std::max(0, atoi(e)); + uint64_t used = 0, allowed = 0; + const bool known = vram_usage(&used, &allowed) && allowed; + const uint64_t v = target_pool::default_mb(known ? allowed : 0); + logf("vk: the render targets' pool holds up to %llu MB (a twelfth of the %llu MB allowed, 256 to 768; " + "RX_VK_TARGET_POOL_MB=, 0: no pool)", (unsigned long long)v, (unsigned long long)(known ? allowed >> 20 : 0)); + return v; + }(); return mb << 20; } uint64_t pool_key_of(const VkImageCreateInfo& ci) { @@ -304,11 +341,18 @@ bool pool_take(uint64_t key, VkImage& image, VmaAllocation& allocation, std::uno std::lock_guard lk(pool_mu); auto it = target_pool.find(key); if (it == target_pool.end() || it->second.empty()) return false; - Pooled& p = it->second.back(); + const std::list::iterator at = it->second.back(); // (the newest of the shape) + Pooled& p = *at; image = p.image, allocation = p.allocation, views = std::move(p.views); pool_bytes -= std::min(pool_bytes, p.bytes), pool_n--; + pool_age.erase(at); it->second.pop_back(); if (it->second.empty()) target_pool.erase(it); + // ((Triforce, 11-21, TEX-29) a shape that is taken again: its images wait the long time, target_pool::idle_out) + if (pool_taken.insert(key).second) { + pool_taken_order.push_back(key); + if (pool_taken_order.size() > target_pool::kTakenKeys) pool_taken.erase(pool_taken_order.front()), pool_taken_order.pop_front(); + } st_pool_reused++; return true; } @@ -321,7 +365,8 @@ void pool_destroy(Pooled& p) { /// (texture_frame) a dead render target's image the GPU is done with, into the pool void pool_put(Dead& d) { std::lock_guard lk(pool_mu); - target_pool[d.pool_key].push_back(Pooled{d.image, d.allocation, std::move(d.view_map), d.bytes, frame_no}); + pool_age.push_back(Pooled{d.pool_key, d.image, d.allocation, std::move(d.view_map), d.bytes, frame_no}); + target_pool[d.pool_key].push_back(std::prev(pool_age.end())); pool_bytes += d.bytes, pool_n++; } /// (texture_frame, each present) the pool over its bound, or all of it while VRAM is short, and what waits in it unused for 120 @@ -330,22 +375,31 @@ size_t pool_trim() { std::lock_guard lk(pool_mu); // (Triforce, 11-11: with a floor of blocks kept (target_floor_kept) the pool is not emptied while VRAM is short: what it gave // back went into blocks the floor keeps, the driver's count did not move, and every target of the next frames was made new) - const bool give_back = pool_pressed && !target_floor_kept(); - const uint64_t cap = give_back ? 0 : pool_cap(); - const size_t most = give_back ? 64 : 16; + // ((Triforce, 11-21, the performance audit's TEX-29) the bound under the tier, the oldest first whatever its shape, and + // the wait by whether the shape is ever taken again: target_pool.h. RX_VK_POOL_IDLE= (8); + // RX_VK_TARGET_POOL_TRIM=0: no tiers, the pool emptied above 90%, as before) + static const bool tiers = !(getenv("RX_VK_TARGET_POOL_TRIM") && atoi(getenv("RX_VK_TARGET_POOL_TRIM")) == 0); + static const uint64_t idle = getenv("RX_VK_POOL_IDLE") ? (uint64_t)std::max(1, atoi(getenv("RX_VK_POOL_IDLE"))) : target_pool::kIdle; + const bool floor_kept = target_floor_kept(); + const uint64_t cap = target_pool::cap_for(pool_cap(), pool_tier, tiers, floor_kept); + const size_t most = target_pool::trim_most(pool_tier, floor_kept); size_t n = 0; - for (auto it = target_pool.begin(); it != target_pool.end() && n < most;) { - auto& v = it->second; - for (size_t i = 0; i < v.size() && n < most;) { // (the oldest of a shape first: the front) - if (pool_bytes <= cap && frame_no - v[i].since <= 120) { - ++i; - continue; - } - pool_destroy(v[i]); - pool_bytes -= std::min(pool_bytes, v[i].bytes), pool_n--, n++; - v.erase(v.begin() + (ptrdiff_t)i); + for (auto it = pool_age.begin(); it != pool_age.end() && n < most;) { + if (pool_bytes <= cap && !target_pool::idle_out(frame_no, it->since, idle, pool_taken.count(it->key) != 0)) { + if (frame_no - it->since <= idle) break; // (in age order: none after it has waited longer) + ++it; // (a shape taken before: its long wait, the ones behind it looked at) + continue; } - it = v.empty() ? target_pool.erase(it) : std::next(it); + auto of = target_pool.find(it->key); + if (of != target_pool.end()) { // (the oldest of the list is the oldest of its shape: that shape's front) + auto& v = of->second; + const auto mine = std::find(v.begin(), v.end(), it); + if (mine != v.end()) v.erase(mine); + if (v.empty()) target_pool.erase(of); + } + pool_destroy(*it); + pool_bytes -= std::min(pool_bytes, it->bytes), pool_n--, n++; + it = pool_age.erase(it); } return n; } @@ -484,6 +538,9 @@ struct Arena { }; std::vector arenas; constexpr uint64_t kArena = 64ull << 20; +// ((Triforce, 11-22, the performance audit's TEX-28, after Source A's) the arenas made so far and what making them took, for +// the memory line: staging_keep.h's rule is for the ones a load frees and makes again) +std::atomic st_arena_makes{0}, st_arena_make_ns{0}; struct Staging { VkBuffer buffer = VK_NULL_HANDLE; @@ -549,6 +606,8 @@ Staging staging(uint64_t n, uint64_t align = 256) { const double ms = std::chrono::duration(std::chrono::steady_clock::now() - t_make).count(); static int slow_logged; if (ms > 5 && slow_logged++ < 24) logf("vk: a staging arena of %llu MB took %.1f ms to make", (unsigned long long)(a.size >> 20), ms); + st_arena_makes.fetch_add(1, std::memory_order_relaxed); // ((Triforce, 11-22, TEX-28) every arena made, and its time) + st_arena_make_ns.fetch_add((int64_t)(ms * 1e6), std::memory_order_relaxed); } a.mapped = (uint8_t*)m; a.value = rec_value, a.used = n; @@ -564,6 +623,26 @@ Staging staging(uint64_t n, uint64_t align = 256) { struct AstcPush { uint32_t block_dims[2], layer_stride, block_size, x_shift, block_height, block_height_mask, native_tiling; }; +// ((Triforce) RX_VK_TEX_PIPES_AHEAD, on by default; =0: as before) The texture converters' compute pipelines (the ASTC decoder, +// the BC encoder, the unswizzle and the swizzle back) made through the pipeline cache kept across runs, not without one, and +// all four before the game (texture_pipelines_warm): each was compiled inside the first upload that needed it, and with an +// empty driver cache (a first start, a new driver) that upload waited for the compile, a 100-320 ms frame +bool tex_pipes_ahead() { + static const bool v = [] { + const char* e = getenv("RX_VK_TEX_PIPES_AHEAD"); + return !e || atoi(e) != 0; + }(); + return v; +} +/// One converter's pipeline: through the shared cache, then without a cache if that fails (RX_VK_TEX_PIPES_AHEAD=0: without) +VkResult texture_compute_pipeline(const VkComputePipelineCreateInfo& ci, VkPipeline* out) { + if (tex_pipes_ahead()) { + const VkResult r = shared_cache_compute(&ci, out); + if (r == VK_SUCCESS) return r; + *out = VK_NULL_HANDLE; + } + return vkCreateComputePipelines(dev.device, VK_NULL_HANDLE, 1, &ci, nullptr, out); +} VkDescriptorSetLayout astc_set = VK_NULL_HANDLE; VkPipelineLayout astc_layout = VK_NULL_HANDLE; VkPipeline astc_pipe = VK_NULL_HANDLE; @@ -598,7 +677,7 @@ bool astc_ready() { ci.stage.module = m; ci.stage.pName = "main"; ci.layout = astc_layout; - const VkResult r = vkCreateComputePipelines(dev.device, VK_NULL_HANDLE, 1, &ci, nullptr, &astc_pipe); + const VkResult r = texture_compute_pipeline(ci, &astc_pipe); vkDestroyShaderModule(dev.device, m, nullptr); if (r != VK_SUCCESS) return false; state = 1; @@ -644,12 +723,36 @@ VkDescriptorSetLayout bc_set = VK_NULL_HANDLE; VkPipelineLayout bc_layout = VK_NULL_HANDLE; VkPipeline bc_pipe = VK_NULL_HANDLE; constexpr uint32_t kBcDim = 2048, kBcLayers = 8; // the scratch: the largest level a BC3 texture may have +static_assert(kBcDim == bc_scratch::kDim && kBcLayers == bc_scratch::kMost); +/// ((Triforce, 11-21, the performance audit's TEX-10) The layers one run decodes and encodes: 1, or RX_VK_BC_LAYERS= up to +/// kBcLayers (upload_astc_gpu says why one) +uint32_t bc_layers() { + static const uint32_t n = bc_scratch::run_layers(getenv("RX_VK_BC_LAYERS")); + return n; +} +/// ((Triforce, 11-21, TEX-10) The scratch's layers, its output buffer's and the second scratch's: a run's. They had kBcLayers +/// whatever the runs used: 140 of the 160 MB held and never written (bc_scratch.h). RX_VK_BC_SCRATCH_FULL=1: kBcLayers, as before +uint32_t bc_scratch_layers() { + static const uint32_t n = bc_scratch::layers(bc_layers(), getenv("RX_VK_BC_SCRATCH_FULL") != nullptr); + return n; +} VkImage bc_scratch = VK_NULL_HANDLE; VmaAllocation bc_scratch_alloc = nullptr; VkImageView bc_scratch_view = VK_NULL_HANDLE; VkBuffer bc_out = VK_NULL_HANDLE; VmaAllocation bc_out_alloc = nullptr; bool bc_scratch_fresh = false; // (its layout still to be set, in the next upload command buffer) +// ((Triforce, 11-21, the performance audit's TEX-09) the small scratch, for levels of 1024 or less: bc_small_layers() layers +// a run. Made and given back with the first; none at RX_VK_BC_SMALL_LAYERS=1, on AMD's cards, or at reduced texture detail. +// bc_scratch.h has why) +VkImage bc_small = VK_NULL_HANDLE; +VmaAllocation bc_small_alloc = nullptr; +VkImageView bc_small_view = VK_NULL_HANDLE; +bool bc_small_fresh = false; +uint32_t bc_small_layers() { + static const uint32_t n = bc_scratch::small_layers(getenv("RX_VK_BC_SMALL_LAYERS"), dev.props.vendorID); + return n; +} // (hookshot-amd) a second scratch for textures re-encoded at another size (uneven block levels on AMD): the decoded level // is blitted into it, filtered, at the image's size, and encoded from there. Made when an upload of such a texture first // needs it, given back with the first scratch. @@ -657,7 +760,8 @@ VkImage bc_scratch2 = VK_NULL_HANDLE; VmaAllocation bc_scratch2_alloc = nullptr; VkImageView bc_scratch2_view = VK_NULL_HANDLE; bool bc_scratch2_fresh = false; -// (the scratch and its buffer, 160 MB, are made when an upload first needs them and given back after RX_VK_BC_IDLE_S seconds +// (the scratch and its buffer, 160 MB ((Triforce, 11-21, TEX-10) 20 MB a layer of bc_scratch_layers(), so 20 MB by default), +// are made when an upload first needs them and given back after RX_VK_BC_IDLE_S seconds // (20) without one: a title or a load decodes hundreds of textures through them, steady gameplay few (the "given back" line says // whether it did). // 0: kept for good once made. They have memory objects of their own, so that freeing gives the VRAM back: VMA keeps the empty @@ -680,7 +784,7 @@ bool bc_scratch_make() { // (bc_mu held) ii.format = VK_FORMAT_R8G8B8A8_UNORM; ii.extent = {kBcDim, kBcDim, 1}; ii.mipLevels = 1; - ii.arrayLayers = kBcLayers; + ii.arrayLayers = bc_scratch_layers(); ii.samples = VK_SAMPLE_COUNT_1_BIT; ii.usage = VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT; static const bool pool = getenv("RX_VK_BC_POOL") != nullptr; @@ -689,8 +793,30 @@ bool bc_scratch_make() { // (bc_mu held) vi.image = bc_scratch; vi.viewType = VK_IMAGE_VIEW_TYPE_2D_ARRAY; vi.format = VK_FORMAT_R8G8B8A8_UNORM; - vi.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, kBcLayers}; - const uint64_t out = (uint64_t)(kBcDim / 4) * (kBcDim / 4) * kBcLayers * 16; + vi.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, bc_scratch_layers()}; + // ((Triforce, 11-21, TEX-09) the small scratch with it. Not at reduced or low texture detail (a card short of VRAM: + // its levels then go one layer a run through the first scratch, as before), and not kept if it cannot be made whole) + if (bc_small_layers() > 1 && texture_detail_level() == tex_detail::kFull) { + VkImageCreateInfo si = ii; + si.extent = {bc_scratch::kSmallDim, bc_scratch::kSmallDim, 1}; + si.arrayLayers = bc_small_layers(); + const char* why = nullptr; + if (!create_image(si, bc_small, bc_small_alloc, !pool)) why = "could not be made"; + else { + VkImageViewCreateInfo sv = vi; + sv.image = bc_small; + sv.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, bc_small_layers()}; + if (vkCreateImageView(dev.device, &sv, nullptr, &bc_small_view) != VK_SUCCESS) why = "has no view"; + } + if (why) { + if (bc_small) destroy_image(bc_small, bc_small_alloc); + bc_small = VK_NULL_HANDLE, bc_small_alloc = nullptr, bc_small_view = VK_NULL_HANDLE; + static std::atomic small_told{0}; + if (small_told++ < 2) logf("vk: the small BC3 scratch %s: its levels go one layer a run", why); + } + } + const uint64_t small_out = bc_small ? bc_scratch::small_out_bytes(bc_small_layers()) : 0; + const uint64_t out = std::max(bc_scratch::out_bytes(bc_scratch_layers()), small_out); // (one buffer: a run of either scratch) const VkBufferUsageFlags out_usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT; const auto make_out = [&] { if (create_device_buffer(out, out_usage, bc_out, bc_out_alloc, !pool)) return true; @@ -706,10 +832,24 @@ bool bc_scratch_make() { // (bc_mu held) if (bc_scratch_view) vkDestroyImageView(dev.device, bc_scratch_view, nullptr); destroy_image(bc_scratch, bc_scratch_alloc); bc_scratch = VK_NULL_HANDLE, bc_scratch_view = VK_NULL_HANDLE; + if (bc_small_view) vkDestroyImageView(dev.device, bc_small_view, nullptr); + if (bc_small) destroy_image(bc_small, bc_small_alloc); + bc_small = VK_NULL_HANDLE, bc_small_alloc = nullptr, bc_small_view = VK_NULL_HANDLE; return false; } bc_scratch_fresh = true; + bc_small_fresh = bc_small != VK_NULL_HANDLE; bc_have = true; + // ((Triforce, 11-21, the performance audit's TEX-10) its size, by the layers a run uses: bc_scratch.h) + static std::atomic made_told{0}; + if (made_told++ < 4) + logf("vk: the BC3 scratch is made: %u layer%s, %llu MB (RX_VK_BC_SCRATCH_FULL=1: 8 layers, 160 MB, as before); the " + "small one for levels of %u or less: %s%u layer%s a run, %llu MB (RX_VK_BC_SMALL_LAYERS=1: none)", + bc_scratch_layers(), bc_scratch_layers() == 1 ? "" : "s", + (unsigned long long)((bc_scratch::mb(bc_scratch_layers(), false) << 20) - bc_scratch::out_bytes(bc_scratch_layers()) + out) >> 20, + bc_scratch::kSmallDim, bc_small ? "" : "none, ", bc_small ? bc_small_layers() : 1u, + (bc_small ? bc_small_layers() : 1u) == 1 ? "" : "s", + (unsigned long long)(bc_small ? bc_scratch::small_image_bytes(bc_small_layers()) >> 20 : 0)); return true; } @@ -747,7 +887,7 @@ bool bc_ready() { ci.stage.module = m; ci.stage.pName = "main"; ci.layout = bc_layout; - const VkResult r = vkCreateComputePipelines(dev.device, VK_NULL_HANDLE, 1, &ci, nullptr, &bc_pipe); + const VkResult r = texture_compute_pipeline(ci, &bc_pipe); vkDestroyShaderModule(dev.device, m, nullptr); if (r != VK_SUCCESS) return false; state = 1; @@ -762,7 +902,7 @@ bool bc_scratch2_make() { ii.format = VK_FORMAT_R8G8B8A8_UNORM; ii.extent = {kBcDim, kBcDim, 1}; ii.mipLevels = 1; - ii.arrayLayers = kBcLayers; + ii.arrayLayers = bc_scratch_layers(); ii.samples = VK_SAMPLE_COUNT_1_BIT; ii.usage = VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT; static const bool pool = getenv("RX_VK_BC_POOL") != nullptr; @@ -771,7 +911,7 @@ bool bc_scratch2_make() { vi.image = bc_scratch2; vi.viewType = VK_IMAGE_VIEW_TYPE_2D_ARRAY; vi.format = VK_FORMAT_R8G8B8A8_UNORM; - vi.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, kBcLayers}; + vi.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, bc_scratch_layers()}; if (vkCreateImageView(dev.device, &vi, nullptr, &bc_scratch2_view) != VK_SUCCESS) { destroy_image(bc_scratch2, bc_scratch2_alloc); bc_scratch2 = VK_NULL_HANDLE, bc_scratch2_view = VK_NULL_HANDLE; @@ -805,8 +945,14 @@ void bc_encode_into(VkCommandBuffer cmd, uint32_t w, uint32_t h, uint32_t n, Tex vkCmdPushConstants(cmd, bc_layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, &pc); vkCmdDispatch(cmd, (pc.blocks_x + 7) / 8, (pc.blocks_y + 7) / 8, n); full_barrier(cmd); - const VkBufferImageCopy c{0, 0, 0, {VK_IMAGE_ASPECT_COLOR_BIT, level, layer, n}, {dx, dy, 0}, {w, h, 1}}; - vkCmdCopyBufferToImage(cmd, bc_out, dst.image, VK_IMAGE_LAYOUT_GENERAL, 1, &c); + // ((Triforce, 11-21, the performance audit's TEX-09) a region a layer, each at the encoder's own layer stride, in one + // copy call: a guard for the runs of several layers, whose layers past the first once came out wrong at the small levels + // on an RX 6600. One layer: the one region it always was) + VkBufferImageCopy c[kBcLayers]; + for (uint32_t k = 0; k < n && k < kBcLayers; k++) + c[k] = VkBufferImageCopy{bc_scratch::layer_offset(k, w, h, dst.bc1), 0, 0, {VK_IMAGE_ASPECT_COLOR_BIT, level, layer + k, 1}, + {dx, dy, 0}, {w, h, 1}}; + vkCmdCopyBufferToImage(cmd, bc_out, dst.image, VK_IMAGE_LAYOUT_GENERAL, std::min(n, kBcLayers), c); full_barrier(cmd); // (the scratch and the buffer are the next part's) } @@ -833,8 +979,23 @@ bool bc_scratch_layout(VkCommandBuffer cmd, bool resized = false) { bc_last_value = rec_value; } // (a segment's worker: a fresh scratch's layout change in queue order, as a new image's) - InitRedirect redirect((bc_scratch2_fresh && bc_scratch2) || bc_scratch_fresh); + InitRedirect redirect((bc_scratch2_fresh && bc_scratch2) || bc_scratch_fresh || (bc_small_fresh && bc_small)); if (redirect.to) cmd = upload_cmd(); + if (bc_small_fresh && bc_small) { // ((Triforce, 11-21, TEX-09) the small scratch, once made) + bc_small_fresh = false; + VkImageMemoryBarrier2 b{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2}; + b.srcStageMask = b.dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT; + b.dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT; + b.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + b.newLayout = VK_IMAGE_LAYOUT_GENERAL; + b.srcQueueFamilyIndex = b.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + b.image = bc_small; + b.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, bc_small_layers()}; + VkDependencyInfo di{VK_STRUCTURE_TYPE_DEPENDENCY_INFO}; + di.imageMemoryBarrierCount = 1; + di.pImageMemoryBarriers = &b; + vkCmdPipelineBarrier2(cmd, &di); + } if (bc_scratch2_fresh && bc_scratch2) { // (hookshot-amd: the second scratch, once made) bc_scratch2_fresh = false; VkImageMemoryBarrier2 b{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2}; @@ -844,7 +1005,7 @@ bool bc_scratch_layout(VkCommandBuffer cmd, bool resized = false) { b.newLayout = VK_IMAGE_LAYOUT_GENERAL; b.srcQueueFamilyIndex = b.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; b.image = bc_scratch2; - b.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, kBcLayers}; + b.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, bc_scratch_layers()}; VkDependencyInfo di{VK_STRUCTURE_TYPE_DEPENDENCY_INFO}; di.imageMemoryBarrierCount = 1; di.pImageMemoryBarriers = &b; @@ -859,7 +1020,7 @@ bool bc_scratch_layout(VkCommandBuffer cmd, bool resized = false) { b.newLayout = VK_IMAGE_LAYOUT_GENERAL; b.srcQueueFamilyIndex = b.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; b.image = bc_scratch; - b.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, kBcLayers}; + b.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, bc_scratch_layers()}; VkDependencyInfo di{VK_STRUCTURE_TYPE_DEPENDENCY_INFO}; di.imageMemoryBarrierCount = 1; di.pImageMemoryBarriers = &b; @@ -876,12 +1037,17 @@ void bc_scratch_idle() { dead.push_back(std::move(d)); bc_scratch = VK_NULL_HANDLE, bc_scratch_alloc = nullptr, bc_scratch_view = VK_NULL_HANDLE, bc_out = VK_NULL_HANDLE, bc_out_alloc = nullptr; bc_have = false, bc_scratch_fresh = false; + if (bc_small) { // ((Triforce, 11-21, TEX-09) the small scratch with it) + dead.push_back(Dead{bc_last_value, bc_small, bc_small_alloc, {bc_small_view}, {}}); + bc_small = VK_NULL_HANDLE, bc_small_alloc = nullptr, bc_small_view = VK_NULL_HANDLE, bc_small_fresh = false; + } const bool second = bc_scratch2 != VK_NULL_HANDLE; if (second) { // (hookshot-amd: the second scratch, 128 MB, with it) dead.push_back(Dead{bc_last_value, bc_scratch2, bc_scratch2_alloc, {bc_scratch2_view}, {}}); bc_scratch2 = VK_NULL_HANDLE, bc_scratch2_alloc = nullptr, bc_scratch2_view = VK_NULL_HANDLE, bc_scratch2_fresh = false; } - logf("vk: the BC3 scratch (%s) unused for %u s: given back", second ? "288 MB, both" : "160 MB", bc_idle_s()); + logf("vk: the BC3 scratch (%llu MB%s) unused for %u s: given back", + (unsigned long long)bc_scratch::mb(bc_scratch_layers(), second), second ? ", both" : "", bc_idle_s()); } /// The storage view of one level (every layer) the ASTC decoder writes @@ -933,7 +1099,7 @@ const uint8_t* guest_bytes(uint64_t gpu, uint64_t n, std::vector& scrat if (p && avail >= n) return gpu < (1ull << 40) ? memory_as_submitted(gpu, p, n, scratch) : p; if (n >= (32u << 20)) { // (a big sparse texture's pages, gathered on several threads: the course maps' 349 MB) scratch.resize(n); - const uint32_t nt = std::clamp(std::thread::hardware_concurrency() / 2, 1u, 6u); + const uint32_t nt = std::clamp(cpu_count() / 2, 1u, 6u); const uint64_t per = ((n + nt - 1) / nt + 0xFFFF) & ~0xFFFFull; std::vector pool; auto part = [&](uint32_t t) { @@ -994,7 +1160,8 @@ thread_local const char* astc_fail = nullptr; // ---- (Triforce) opaque ASTC textures kept as BC1 instead of BC3: half the video memory, the same texels (tex_bc1.h says // why they are the same; astc_opaque.h which blocks the decoder writes opaque). The Graphics tab's "Opaque textures as BC1" -// row as the options file had it at the start, Off unless set; RX_VK_TEX_BC1=0 or 1 over it. Only a texture kept as BC3 +// row as the options file had it at the start, on unless it says off ((Triforce, 11-21, TEX-50) and on with no options file, +// which started at off); RX_VK_TEX_BC1=0 or 1 over it. Only a texture kept as BC3 // today, not a resized one, and only one first met by a draw that samples it: its blocks in the game's memory are looked // at when it is made, and it is made BC1 when all of them are opaque. Every upload into a BC1 image looks again, at the very // bytes it stages, and content with alpha (the game wrote its memory anew) makes the texture BC3 for good, as a copy on the @@ -1005,6 +1172,9 @@ std::atomic st_bc1_made{0}, st_bc1_saved{0}, st_bc1_back{0}; bool bc1_on() { static const bool on = [] { if (!tex_bc1::wanted(getenv("RX_VK_TEX_BC1"), tex_bc1_at_start)) return false; + // ((Triforce, 11-21, the performance audit's TEX-50) blocks with a mode a partition are looked at too: astc_opaque.h. + // RX_VK_TEX_BC1_PARTS=0: refused unread, as before) + astc_opaque::parts_too = !(getenv("RX_VK_TEX_BC1_PARTS") && atoi(getenv("RX_VK_TEX_BC1_PARTS")) == 0); const VkFormatFeatureFlags need = VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT | VK_FORMAT_FEATURE_TRANSFER_DST_BIT | VK_FORMAT_FEATURE_TRANSFER_SRC_BIT; const bool can = dev.features.textureCompressionBC && @@ -1150,13 +1320,16 @@ bool upload_astc_gpu(Texture& tex, uint64_t total, const Ranges* dirty) { // smaller levels, and Hyrule Field's ground, which samples the terrain's material arrays (1365 x 1365 x 90, ASTC // 8x8) at those levels from a distance, had its tiles outlined in dark lines (RX 6600: none with one layer a run, // nor with the arrays kept as RGBA8). RX_VK_BC_LAYERS=: up to n a run) - static const uint32_t run = getenv("RX_VK_BC_LAYERS") ? std::clamp((uint32_t)atoi(getenv("RX_VK_BC_LAYERS")), 1u, kBcLayers) : 1u; + // ((Triforce, 11-21, the performance audit's TEX-09) a level the small scratch holds: bc_small_layers() + // layers a run through it; any other, bc_layers() through the first (TEX-10: what that one is sized by)) + const bool small = bc_scratch::small_level(g.w, g.h, bc_small != VK_NULL_HANDLE, tex.bc_resized); + const uint32_t run = small ? bc_small_layers() : bc_layers(); while (k1 < layers.size() && k1 - k0 < run && layers[k1] == layers[k1 - 1] + 1) k1++; const uint32_t n = (uint32_t)(k1 - k0); vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, astc_pipe); const uint64_t from = k0 * tail + off; VkDescriptorBufferInfo bi{s.buffer, s.offset + from, staged - from}; - VkDescriptorImageInfo ii{VK_NULL_HANDLE, bc_scratch_view, VK_IMAGE_LAYOUT_GENERAL}; + VkDescriptorImageInfo ii{VK_NULL_HANDLE, small ? bc_small_view : bc_scratch_view, VK_IMAGE_LAYOUT_GENERAL}; VkWriteDescriptorSet w[2]{{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET}, {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET}}; w[0].dstBinding = 0, w[0].descriptorCount = 1, w[0].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; w[0].pBufferInfo = &bi; @@ -1168,7 +1341,7 @@ bool upload_astc_gpu(Texture& tex, uint64_t total, const Ranges* dirty) { if (tex.bc_resized) bc_encode_resized(cmd, g.w, g.h, iw, ih, n, tex, level, layers[k0]); else - bc_encode_into(cmd, iw, ih, n, tex, level, layers[k0], 0, 0); + bc_encode_into(cmd, iw, ih, n, tex, level, layers[k0], 0, 0, small ? bc_small_view : VK_NULL_HANDLE); k0 = k1; } } @@ -1255,7 +1428,7 @@ bool unswizzle_ready() { ci.stage.module = m; ci.stage.pName = "main"; ci.layout = unsw_layout; - const VkResult r = vkCreateComputePipelines(dev.device, VK_NULL_HANDLE, 1, &ci, nullptr, &unsw_pipe); + const VkResult r = texture_compute_pipeline(ci, &unsw_pipe); vkDestroyShaderModule(dev.device, m, nullptr); if (r != VK_SUCCESS) return false; state = 1; @@ -1708,6 +1881,7 @@ const uint8_t* same_snap_read(const Texture& tex, uint64_t off, uint64_t n) { bool same_copy(Texture& tex) { const upload_same::State& s = tex.same; if (tl_same_cap < tex.guest_size) { + mkwvk::new_wait::NoWait at_once; // ((Triforce, OS-04) no buffer: no copy, as the comment above says; not a wait at the commit limit) tl_same_buf.reset(new (std::nothrow) uint8_t[tex.guest_size]); tl_same_cap = tl_same_buf ? tex.guest_size : 0; } @@ -1928,8 +2102,11 @@ void same_unchecked(int why, int route, const Ranges* part, uint64_t whole) { /// made). Such an upload stays owed and is tried again (texture_touch, upload_owed.h); any other failure is of an image no /// upload fills (multisampled, depth and stencil, a size no upload takes), which trying again does not change thread_local bool upload_starved = false; +void warm_learn(const Texture& tex); // ((Triforce, 11-22, TEX-23) below, with the textures made ahead) +thread_local bool warm_looking = false; // ((and this thread is in the list's look ahead, warm_step_listed: it learns nothing) bool upload(Texture& tex, const Ranges* dirty) { const auto t0 = std::chrono::steady_clock::now(); + if (!tex.uploaded) warm_learn(tex); // ((Triforce, 11-22, TEX-23) a first upload: a draw's, never the list's own look) up_route = UP_CPU; upload_same::changed(tex.same); // ((Triforce) whatever it writes into the image: the kept hashes go, but a checked upload's) const uint64_t image0 = same_image_id(tex); // ((Triforce, 11-16) the image before it, for the twin's note below) @@ -1981,38 +2158,33 @@ bool upload_all_zero(Texture& tex, uint64_t total, const Ranges* dirty, bool wid const uint8_t* p = host->gpu_ptr(tex.gpu, &avail); if (!p || avail < total) return false; const auto t0 = std::chrono::steady_clock::now(); - std::atomic found{false}; - const uint32_t nthreads = std::clamp(std::thread::hardware_concurrency() / 2, 1u, 8u); - const uint64_t per = ((total / nthreads) + 4095) & ~4095ull; - auto part = [&](uint32_t t) { - const uint64_t a = std::min(total, (uint64_t)t * per), b = std::min(total, a + per); - for (uint64_t at = a; at < b && !found.load(std::memory_order_relaxed); at += 65536) { - const uint64_t end = std::min(b, at + 65536); - uint64_t acc = 0; - uint64_t k = at; - for (; k + 8 <= end; k += 8) { - uint64_t w; - std::memcpy(&w, p + k, 8); - acc |= w; - } - for (; k < end; k++) acc |= p[k]; - if (acc) found.store(true, std::memory_order_relaxed); - } - }; - if (nthreads > 1) { - std::vector th; - for (uint32_t t = 1; t < nthreads; t++) th.emplace_back(part, t); - part(0); - for (auto& x : th) x.join(); - } else { - part(0); + // ((Triforce, 11-21) only the bytes that hold data: on Linux the guest memfd's holes are left unread, guest_holes.h. Anywhere + // else, a storage outside the window, or a seek that fails: the whole storage) + std::vector extents{{0, total}}; +#ifdef __linux__ + static const bool holes = !(getenv("RX_VK_ZERO_HOLES") && atoi(getenv("RX_VK_ZERO_HOLES")) == 0); + static const int memfd = [] { + typedef int (*MemfdFn)(void); + const MemfdFn fn = (MemfdFn)dlsym(RTLD_DEFAULT, "hos_posix_guest_memfd"); + return fn ? fn() : -1; + }(); + if (holes && memfd >= 0) { + const uint8_t* base = host->guest_base ? host->guest_base() : nullptr; + const uint64_t size = host->guest_size ? host->guest_size() : 0; + std::vector data; + if (base && p >= base && total <= size && (uint64_t)(p - base) <= size - total && + guest_holes::fd_data_extents(memfd, (uint64_t)(p - base), total, &data)) + extents.swap(data); } +#endif + const uint64_t read_bytes = guest_holes::bytes(extents); + const bool found = guest_holes::any_set(p, extents, std::clamp(cpu_count() / 2, 1u, 8u)); const double ms = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); static std::atomic told{0}; - if (found.load()) { + if (found) { if (told.fetch_add(1) < 4) - logf("vk: upload: %ux%u NVN %#x (%llu MB) is not all zero (checked in %.1f ms): unswizzled as before", tex.width, - tex.height, tex.nvn_format, (unsigned long long)(total >> 20), ms); + logf("vk: upload: %ux%u NVN %#x (%llu MB) is not all zero (checked in %.1f ms, %llu MB read): unswizzled as before", + tex.width, tex.height, tex.nvn_format, (unsigned long long)(total >> 20), ms, (unsigned long long)(read_bytes >> 20)); return false; } texture_resident_flush(&tex); @@ -2026,8 +2198,8 @@ bool upload_all_zero(Texture& tex, uint64_t total, const Ranges* dirty, bool wid vkCmdClearColorImage(upload_cmd(), tex.image, VK_IMAGE_LAYOUT_GENERAL, &z, 1, &range); } if (told.fetch_add(1) < 4) - logf("vk: upload: %ux%u NVN %#x (%llu MB) is all zero: cleared on the GPU (checked in %.1f ms)", tex.width, tex.height, - tex.nvn_format, (unsigned long long)(total >> 20), ms); + logf("vk: upload: %ux%u NVN %#x (%llu MB) is all zero: cleared on the GPU (checked in %.1f ms, %llu MB read)", tex.width, + tex.height, tex.nvn_format, (unsigned long long)(total >> 20), ms, (unsigned long long)(read_bytes >> 20)); return true; } /// ((Triforce, 11-17) Whether upload_inner may take a way whose bytes are not fixed when it is recorded: the GPU unswizzle reads @@ -2045,6 +2217,90 @@ bool upload_reads_live(const Texture& tex, const Ranges* dirty) { return (!(tex.gpu < (1ull << 40) && memory_has_shadow(tex.gpu, total)) && unswizzle_gpu_takes(tex, total, vf)) || all_zero_takes(tex, total, dirty, widen_from != 0); } +// ---- ((Triforce, 11-21, the performance audit's TEX-14, after Source A's) decoded ASTC used again: astc_reuse.h. This step: +// the blocks hashed at a texture's first upload, the finalized textures listed, a hit counted. RX_VK_ASTC_REUSE=2; nothing +// is copied or let go yet, and with the switch off none of this runs ---- +astc_reuse::Mode astc_reuse_mode() { + static const astc_reuse::Mode m = astc_reuse::mode(getenv("RX_VK_ASTC_REUSE")); + return m; +} +std::mutex reuse_mu; +astc_reuse::Freed reuse_freed; // (reuse_mu) +std::atomic st_reuse_hits{0}, st_reuse_hit_bytes{0}, st_reuse_hash_ns{0}, st_reuse_hash_bytes{0}, st_reuse_hashed{0}, + st_reuse_listed{0}; +astc_reuse::Shape reuse_shape(const Texture& t) { + return {(uint32_t)t.format, t.width, t.height, t.depth, t.levels, t.layers, t.skip, (uint32_t)t.nvn_format, t.key_layout, + t.bc, t.bc_resized, t.bc1, t.bc1_no, t.detail_skip}; +} +/// (used again at its own address, or going: no longer one to take an image from) +void reuse_drop(Texture& t) { + std::lock_guard lk(reuse_mu); + if (t.freed_listed) reuse_freed.unlist(t.blocks_hash, &t), t.freed_listed = false; +} +/// The game finalized the texture at t's storage: its decoded ASTC uploaded from memory and never drawn into, of t's own +/// description, listed to serve the same blocks made again elsewhere +void reuse_released(const nvn_tex* t) { + auto [a, b] = by_gpu.equal_range(t->gpu); + for (auto it = a; it != b; ++it) { + Texture* x = it->second; + if (!x->astc || !x->blocks_hash || !x->uploaded || x->gpu_written || x->freed_listed) continue; + if (x->d.format != t->d.format || x->d.width != t->d.width || x->d.height != t->d.height || x->d.depth != t->d.depth || + x->d.levels != t->d.levels || x->d.target != t->d.target) + continue; + std::lock_guard lk(reuse_mu); + reuse_freed.list(x->blocks_hash, x); + x->freed_listed = true; + st_reuse_listed.fetch_add(1, std::memory_order_relaxed); + } +} +/// A new decoded ASTC texture's blocks hashed, at its first upload, and a finalized texture with the same blocks and shape +/// looked for: counted (mode 2). False: the caller decodes, as before +bool reuse_upload(Texture& tex, uint64_t total) { + const auto t0 = std::chrono::steady_clock::now(); + static thread_local std::vector scratch; + astc_reuse::Hasher h; + for (uint64_t at = 0; at < total; at += 64ull << 20) { + const uint64_t n = std::min(64ull << 20, total - at); + h.add(guest_bytes(tex.gpu + at, n, scratch), n); + } + tex.blocks_hash = h.done(); + st_reuse_hash_ns += std::chrono::duration_cast(std::chrono::steady_clock::now() - t0).count(); + st_reuse_hash_bytes += (int64_t)total; + st_reuse_hashed.fetch_add(1, std::memory_order_relaxed); + if (tex.sparse || tex.type != VK_IMAGE_TYPE_2D || !tex.image) return false; + Texture* from; + { + std::lock_guard lk(reuse_mu); + if (!reuse_freed.size()) return false; + from = reuse_freed.find(tex.blocks_hash, reuse_shape(tex), &tex, [](const Texture* t) { return reuse_shape(*t); }); + if (!from || !from->image || !from->uploaded || from->gpu_written) return false; + reuse_freed.unlist(from->blocks_hash, from); // (it serves one texture: counted once) + from->freed_listed = false; + } + const int64_t n = st_reuse_hits.fetch_add(1) + 1; + st_reuse_hit_bytes += (int64_t)tex.bytes; + if (n <= 8 || (n & (n - 1)) == 0) + logf("vk: ASTC texture #%u %#llx %ux%u x%u levels %u: the same blocks and shape as #%u %#llx, finalized by the game and " + "still in VRAM: a copy would do for the decode (%lld so far, %lld MB; RX_VK_ASTC_REUSE=2 counts only)", tex.id, + (unsigned long long)tex.gpu, tex.width, tex.height, tex.layers, tex.levels, from->id, (unsigned long long)from->gpu, + (long long)n, (long long)(st_reuse_hit_bytes.load() >> 20)); + return false; +} +/// (every 600 presents, texture_frame) what the hashing cost and what a reuse would have saved +void reuse_report() { + if (astc_reuse_mode() == astc_reuse::kOff || !st_reuse_hashed.load()) return; + size_t listed; + { + std::lock_guard lk(reuse_mu); + listed = reuse_freed.size(); + } + logf("vk: decoded ASTC used again (RX_VK_ASTC_REUSE=%d): %lld textures' blocks hashed so far, %lld MB in %lld ms; %lld " + "finalized ones listed so far, %zu listed now; %lld new textures had a listed one's blocks and shape, %lld MB of images", + (int)astc_reuse_mode(), (long long)st_reuse_hashed.load(), (long long)(st_reuse_hash_bytes.load() >> 20), + (long long)(st_reuse_hash_ns.load() / 1000000), (long long)st_reuse_listed.load(), listed, + (long long)st_reuse_hits.load(), (long long)(st_reuse_hit_bytes.load() >> 20)); +} + bool upload_inner(Texture& tex, const Ranges* dirty) { PROF(PROF_UPLOAD); if (!tex.gpu || tex.samples != VK_SAMPLE_COUNT_1_BIT) return false; @@ -2069,6 +2325,10 @@ bool upload_inner(Texture& tex, const Ranges* dirty) { (unsigned long long)tex.first_seen); } } + // ((Triforce, 11-21, TEX-14) a new ASTC texture's first upload: its blocks hashed once, whole or its first part + // (layer_split.h passes a range), and a finalized one with the same blocks looked for: reuse_upload above) + if (tex.astc && !tex.uploaded && !tex.blocks_hash && astc_reuse_mode() != astc_reuse::kOff && reuse_upload(tex, total)) + return up_route = UP_ASTC, true; if (tex.astc && (up_route = UP_ASTC, upload_astc_gpu(tex, total, dirty))) return true; up_route = UP_CPU; if (tex.bc) return false; // (no CPU path into BC3) @@ -2158,7 +2418,7 @@ bool upload_inner(Texture& tex, const Ranges* dirty) { // a frame's upload each, and most of them were under 16 MB static const uint64_t mt_bytes = (uint64_t)(getenv("RX_VK_UNSWIZZLE_MT_MB") ? std::max(1, atoi(getenv("RX_VK_UNSWIZZLE_MT_MB"))) : 4) << 20; const uint32_t nthreads = block_bytes >= mt_bytes && g.td == 1 && !g.tile_d - ? std::clamp(std::min(std::thread::hardware_concurrency() / 2, + ? std::clamp(std::min(cpu_count() / 2, (uint32_t)(block_bytes >> 21)), 1u, 8u) : 1u; uint64_t avail = 0; const uint8_t* direct = host->gpu_ptr(tex.gpu + from, &avail); @@ -2450,7 +2710,9 @@ bool bc1_retype(Texture& x, const char* why) { ci.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; VkImage image = VK_NULL_HANDLE; VmaAllocation allocation = nullptr; - if (!(compact_on() && create_image(ci, image, allocation, false, false, true)) && !create_image(ci, image, allocation)) { + // ((Triforce, 11-22, TEX-47) with the memory priorities on, from the textures' pool whether or not it is compacted: create()) + if (!((compact_on() || dev.memory_priority) && create_image(ci, image, allocation, false, false, true)) && + !create_image(ci, image, allocation)) { static std::atomic told{0}; if (told++ < 16) logf("vk: texture #%u %#llx %ux%u x%u kept as BC1 (%s): its BC3 image could not be made", x.id, @@ -2499,6 +2761,7 @@ bool bc1_retype(Texture& x, const char* why) { void vw_forget(const Texture& t); // (below: a destroyed texture leaves the list of GPU-written ones) void forget_gpu(Texture* tex) { + if (tex->freed_listed) reuse_drop(*tex); // ((Triforce, 11-21, TEX-14) gone: no image to take) vw_forget(*tex); auto [a, b] = by_gpu.equal_range(tex->gpu); for (auto it = a; it != b; ++it) @@ -2519,6 +2782,23 @@ bool texture_size_ok(VkImageType type, uint32_t w, uint32_t h, uint32_t z, uint3 return w <= most && h <= most && z <= lim.maxImageDimension3D && layers <= lim.maxImageArrayLayers && (samples <= 1 || samples == 2 || samples == 4 || samples == 8 || samples == 16); } +/// ((Triforce, 11-22, the performance audit's TEX-86, after Source A's) whether an image the GPU samples in the game's own +/// compressed format (BC, or ASTC where the driver samples it) is counted by its whole blocks, tex_count.h. It was counted a +/// byte a texel, and ASTC the 16 bytes of a block for every texel: 64 times an 8 x 8 texture's size, so the budget let +/// textures go for memory nothing held, and BC1 and BC4 twice. The images the renderer encodes itself (x.bc) were counted +/// right and are not asked. RX_VK_COUNT_BY_TEXELS=1: as it was. Read at the first such image, and said once) +bool count_by_blocks() { + static const bool on = [] { + const bool blocks = !tex_count::by_texels(getenv("RX_VK_COUNT_BY_TEXELS")); + if (blocks) + logf("vk: textures in the game's own BC or ASTC blocks are counted by whole blocks (RX_VK_COUNT_BY_TEXELS=1: by " + "texels, as before)"); + else + logf("vk: textures in the game's own BC or ASTC blocks are counted by texels, as before (RX_VK_COUNT_BY_TEXELS)"); + return blocks; + }(); + return on; +} Texture* create(const Key& key, const nvn_tex* t, VkFormat vf, bool astc, int bw, int bh, int bpb, bool as_target) { PROF_EXACT(); // (texture_get's slow path: PROF_TEXGET samples the rest) auto tex = std::make_unique(); @@ -2557,7 +2837,13 @@ Texture* create(const Key& key, const nvn_tex* t, VkFormat vf, bool astc, int bw static const uint32_t max_dim = getenv("RX_VK_TEX_MAX_DIM") ? (uint32_t)atoi(getenv("RX_VK_TEX_MAX_DIM")) : 2048; static const uint64_t max_bytes = (uint64_t)(getenv("RX_VK_TEX_MAX_MB") ? atoi(getenv("RX_VK_TEX_MAX_MB")) : 1536) << 20; const uint32_t texel = astc ? 4 : std::max(1, bpb / std::max(1, bw * bh)); - auto bytes_at = [&](uint32_t skip) { + // ((Triforce, 11-22, TEX-86) the game's own BC or ASTC blocks, sampled as they are (nothing is kept as BC3 yet here: + // x.format is the game's format, RGBA8 for ASTC that is decoded): by whole blocks, where it was at least a byte a texel) + const bool native_blocks = (is_bc(x.format) || is_astc(x.format)) && count_by_blocks(); + auto bytes_at = [&](uint32_t skip) -> uint64_t { + if (native_blocks) + return tex_count::bytes(x.width, x.height, x.depth, skip, x.levels, x.layers, (uint32_t)bw, (uint32_t)bh, + (uint32_t)bpb) * std::max(1, d.samples); uint64_t px = 0; for (uint32_t l = skip; l < x.levels; l++) px += (uint64_t)std::max(1u, x.width >> l) * std::max(1u, x.height >> l) * std::max(1u, x.depth >> l); @@ -2636,8 +2922,14 @@ Texture* create(const Key& key, const nvn_tex* t, VkFormat vf, bool astc, int bw static const bool all = [] { if (const char* e = getenv("RX_VK_ASTC_BC_ALL")) return e[0] == '1'; uint64_t used = 0, allowed = 0; - const bool small = vram_usage(&used, &allowed) && astc_bc_policy::all_by_vram(allowed); - if (small) logf("vk: %llu MB of VRAM allowed: all large ASTC textures kept as BC3", (unsigned long long)(allowed >> 20)); + // ((Triforce, 11-22, the performance audit's TEX-12, after Source A's) and on every integrated GPU, whose allowance + // is a share of the system's RAM: astc_bc_policy::all. An allowance that is not known counts as 0, as it did) + if (!vram_usage(&used, &allowed)) allowed = 0; + const bool integrated = dev.props.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU; + const bool small = astc_bc_policy::all(allowed, integrated); + if (small) + logf("vk: %llu MB of VRAM allowed%s: all large ASTC textures kept as BC3", (unsigned long long)(allowed >> 20), + integrated ? ", an integrated GPU" : ""); return small; }(); const uint64_t min_bytes = (bw * bh >= 36 || all) ? low : 64ull << 20; @@ -2710,8 +3002,10 @@ Texture* create(const Key& key, const nvn_tex* t, VkFormat vf, bool astc, int bw const uint64_t n = ++st_bc1_made; if (n <= 4 || (n & (n - 1)) == 0) logf("vk: texture %#llx %ux%u x%u, %u levels: every block opaque, kept as BC1 (%llu so far, %llu MB less than " - "as BC3; %llu made BC3 again)", (unsigned long long)t->gpu, x.width, x.height, x.layers, x.levels, - (unsigned long long)n, (unsigned long long)(saved >> 20), (unsigned long long)st_bc1_back.load()); + "as BC3; %llu made BC3 again; %llu blocks with a mode a partition read as opaque so far, " + "RX_VK_TEX_BC1_PARTS=0: those refused)", (unsigned long long)t->gpu, x.width, x.height, x.layers, x.levels, + (unsigned long long)n, (unsigned long long)(saved >> 20), (unsigned long long)st_bc1_back.load(), + (unsigned long long)astc_opaque::parts_passed.load(std::memory_order_relaxed)); } } } @@ -2813,6 +3107,11 @@ Texture* create(const Key& key, const nvn_tex* t, VkFormat vf, bool astc, int bw x.create_flags = ci.flags; // (hookshot, RX_VK_COMPACT) an image that is not a render target's: from the textures' own pool, the one compacted const bool in_textures = compact_on() && !x.sparse && !(as_target && !is_bc(x.format) && !is_astc(x.format)); + // ((Triforce, 11-22, the performance audit's TEX-47, after Source A's) and with the memory priorities on + // (RX_VK_MEMORY_PRIORITY and VK_EXT_memory_priority: dev.memory_priority) from that pool too when it is not compacted: + // the textures shared the other images' pool then, DLSS's among them, and could not be ranked below them. The + // compaction itself stays as RX_VK_COMPACT says (compact_step and the budget ask compact_on alone)) + const bool ranked = !in_textures && dev.memory_priority && !x.sparse && !(as_target && !is_bc(x.format) && !is_astc(x.format)); if (x.sparse) { ci.flags |= VK_IMAGE_CREATE_SPARSE_BINDING_BIT | VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT; bool ok = vkCreateImage(dev.device, &ci, nullptr, &x.image) == VK_SUCCESS; @@ -2849,6 +3148,8 @@ Texture* create(const Key& key, const nvn_tex* t, VkFormat vf, bool astc, int bw x.sparse_req.formatProperties.flags & VK_SPARSE_IMAGE_FORMAT_SINGLE_MIPTAIL_BIT ? ", one for all layers" : " a layer"); } else if (in_textures && create_image(ci, x.image, x.allocation, false, false, true)) { // (made in the textures' pool; one refused there is made as any other, below) + } else if (ranked && create_image(ci, x.image, x.allocation, false, false, true)) { + // ((Triforce, 11-22, TEX-47) the same pool, for its priority alone; one refused there is made as any other, below) } else if (!(pool_key && pool_take(pool_key, x.image, x.allocation, x.views)) && (st_pool_made += pool_key != 0, true) && !create_image(ci, x.image, x.allocation, false, as_target && !is_bc(x.format) && !is_astc(x.format)) && // (a compressed image refused: made again without its raw attachment usage, then without its raw storage @@ -2869,6 +3170,11 @@ Texture* create(const Key& key, const nvn_tex* t, VkFormat vf, bool astc, int bw px += (uint64_t)std::max(1u, x.width >> l) * std::max(1u, x.height >> l) * std::max(1u, x.depth >> l); const uint32_t texel = x.bc || is_bc(x.format) ? 1 : astc ? 4 : std::max(1, from ? (from == 3 ? 4 : from == 6 ? 8 : 16) : bpb); x.bytes = x.sparse ? 0 : px * x.layers * texel * x.samples; // (a sparse image's: as its layers are bound) + // ((Triforce, 11-22, TEX-86) the game's own BC or ASTC blocks: whole blocks (tex_count.h), not a byte or 16 a texel. The pad + // below still lifts the count to what the allocator gave) + if (!x.sparse && !x.bc && (is_bc(x.format) || is_astc(x.format)) && count_by_blocks()) + x.bytes = tex_count::bytes(x.width, x.height, x.depth, 0, x.levels, x.layers, (uint32_t)bw, (uint32_t)bh, (uint32_t)bpb) * + x.samples; if (x.bc1) x.bytes = tex_bc1::bytes(x.width, x.height, x.levels, x.layers); // (Triforce: eight bytes a block, half of BC3's) // (the budget counts what the allocator gave it: the texels' 8% short of that over a scene's 2000 textures, 3228 MB counted for 3504 // MB held, and a budget of 4035 MB took 4300. RX_VK_COUNT_BY_ESTIMATE=1: the texels) @@ -3239,6 +3545,8 @@ Texture* texture_get(const nvn_tex* t, bool for_sampling) { } } else { tex = it->second.get(); + // ((Triforce, 11-22, TEX-23) one the list's look made ahead, at its first draw: learnt, as an upload a draw makes is) + if (tex->warmed && for_sampling && !warm_looking) tex->warmed = false, warm_learn(*tex); } return texture_touch(tex, for_sampling); } @@ -3274,12 +3582,223 @@ void texture_warm_note(int slot) { std::lock_guard lk(warm_mu); if (warm_pending.size() < tex_warm::kMaxPending && warm_noted.insert(slot).second) warm_pending.push_back({slot, frame_no}); } + +// ---- (Triforce, 11-22, the performance audit's TEX-23, after Source A's) The list of what was drawn (tex_warm.h; with +// RX_VK_TEX_WARM=1 and RX_VK_TEX_WARM_LIST=1 or 2, else none of it runs) ---- +// A BC or ASTC texture's storage is noted with the time the game made it (texture_warm_born, at the texture's creation; its +// release forgets it). A draw's first upload of one made 90 s ago or less, or the first draw of one the look made ahead, puts +// its place in the list; the look itself learns nothing (warm_looking). At the look the ripe slots the list names are made +// while they have a room of their own (tex_warm::listed_room); with 1 the others wait in a second queue, made only when no +// noted slot is left ripe and the 40% rule allows; with 2 they are left to their draws. The list is read before its first use +// and saved every 300 presents when it changed (texture_frame), whole (file_whole.h) +namespace { +std::mutex warm_born_mu; +std::unordered_map warm_born; // (a BC or ASTC storage's GPU address: when the game made it; warm_born_mu) +constexpr size_t kBornMost = 1u << 18; // (storages noted at most: more are not, and are not learnt) +std::mutex warm_list_mu; +std::unordered_set warm_list; // (the places of what was drawn; warm_list_mu) +bool warm_list_changed = false; // (warm_list_mu) +std::deque warm_unlisted; // (with 1: ripe slots the list does not name; the recording thread's) +int64_t st_list_listed, st_list_noroom, st_list_unlisted, st_list_left; // (the recording thread's) +std::atomic st_list_learnt{0}; +int64_t warm_now_ns() { + return std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); +} +std::string warm_list_path() { + char exe[MAX_PATH]; + const DWORD n = game_module_path(exe, MAX_PATH); + const std::string dir(exe, n); + return dir.substr(0, dir.find_last_of("\\/") + 1) + "triforce_textures_drawn.bin"; +} +/// A texture's place: its shape and eight samples of its bytes, at its storage's address (0: its bytes could not be read) +uint64_t warm_place_of(const nvn_texdesc& d, const nvn_texlayout& L, uint64_t gpu) { + const tex_warm::Shape s{d.width, d.height, d.depth, d.levels, (uint32_t)d.format, (uint32_t)d.target, L.layers, L.size}; + uint8_t b[tex_warm::kSamples * tex_warm::kSampleBytes] = {}; + const uint64_t n = tex_warm::sample_len(L.size); + if (!gpu || !n) return 0; + for (int i = 0; i < tex_warm::kSamples; i++) + if (!host->gpu_read(gpu + tex_warm::sample_at(L.size, i), b + i * tex_warm::kSampleBytes, n)) return 0; + return tex_warm::place(tex_warm::signature(s, b), gpu); +} +} // namespace +/// RX_VK_TEX_WARM_LIST, with RX_VK_TEX_WARM on: 0 (unset: every registered slot, as before), 1 or 2 (tex_warm.h) +int texture_warm_list_mode() { + static const int mode = [] { + const int m = texture_warm_on() ? tex_warm::list_mode_of(getenv("RX_VK_TEX_WARM_LIST")) : 0; + if (m) + logf("vk: textures made ahead by the list of what was drawn (RX_VK_TEX_WARM_LIST=%d): %s; the list is " + "triforce_textures_drawn.bin beside the game", m, + m == 1 ? "the listed first, while 256 MB more fit the budget, the others after them by the 40% rule" + : "the listed alone, while 256 MB more fit the budget"); + return m; + }(); + return mode; +} +namespace { +/// The list as the file has it, once, before the first look or the first learning; a file that is not whole loads nothing +void warm_list_load_once() { + static std::once_flag once; + std::call_once(once, [] { + std::unordered_set had; + std::vector b; + bool ok = false; + if (FILE* f = fopen(warm_list_path().c_str(), "rb")) { + uint8_t chunk[65536]; + size_t n; + while (b.size() <= (tex_warm::kListMost + 8) * 8 && (n = fread(chunk, 1, sizeof chunk, f)) > 0) b.insert(b.end(), chunk, chunk + n); + fclose(f); + ok = tex_warm::decode(b.data(), b.size(), had); + } + std::lock_guard lk(warm_list_mu); + for (const uint64_t p : had) tex_warm::remember(warm_list, p); + logf("vk: the list of what was drawn: %zu textures read%s", had.size(), + b.empty() ? " (no file yet)" : ok ? "" : " (the file was not a whole one of this version: not read)"); + }); +} +/// A texture a draw used: learnt when the list is on, it is BC or ASTC, and the game made its storage 90 s ago or less +void warm_learn(const Texture& tex) { + if (warm_looking || !texture_warm_list_mode() || !tex.gpu || tex.gpu >= (1ull << 39)) return; + int64_t born = 0; + { + std::lock_guard lk(warm_born_mu); + const auto it = warm_born.find(tex.gpu); + if (it == warm_born.end()) return; + born = it->second; + } + if (!tex_warm::learnable(born, warm_now_ns())) return; + const uint64_t p = warm_place_of(tex.d, tex.L, tex.gpu); + if (!p) return; + warm_list_load_once(); + std::lock_guard lk(warm_list_mu); + if (tex_warm::remember(warm_list, p)) warm_list_changed = true, st_list_learnt.fetch_add(1, std::memory_order_relaxed); +} +/// (texture_frame, at its looks) the list saved every kSaveEvery presents when it changed +void warm_list_save_due() { + if (!texture_warm_list_mode() || frame_no % tex_warm::kSaveEvery) return; + std::vector b; + { + std::lock_guard lk(warm_list_mu); + if (!warm_list_changed) return; + warm_list_changed = false; + b = tex_warm::encode(warm_list); + } + if (!file_writers::write_whole(warm_list_path(), b.data(), b.size())) { + std::lock_guard lk(warm_list_mu); + warm_list_changed = true; // (tried again at the next) + } +} +/// (texture_warm_step, RX_VK_TEX_WARM_LIST=1 or 2) The look by the list: the ripe noted slots it names made while they have +/// their room, the others kept for later (1) or left to their draws (2); with 1, once no noted slot is left ripe, the kept +/// ones while the 40% rule allows. One budget a frame, as the look without the list +void warm_step_listed(int mode) { + warm_list_load_once(); + const struct Looking { + Looking() { warm_looking = true; } + ~Looking() { warm_looking = false; } + } looking; + const auto t0 = std::chrono::steady_clock::now(); + int64_t spent = 0; + bool any = false, more = true; + const auto make = [](const nvn_tex& t) { // (as a draw's first look; marked, so that its first draw learns it) + ParLock lk; + const size_t had = textures.size(); + Texture* tex = texture_get(&t, true); + const bool made = textures.size() != had; + if (made && tex) tex->warmed = true; + return made; + }; + const auto goes_on = [&] { + spent = std::chrono::duration_cast(std::chrono::steady_clock::now() - t0).count(); + return tex_warm::goes_on(warm_frame_ns + spent, frame_upload_ns.load(std::memory_order_relaxed)); + }; + for (;;) { + std::pair e; + { + std::lock_guard lk(warm_mu); + if (warm_pending.empty() || !tex_warm::ripe(warm_pending.front().second, frame_no)) break; + e = warm_pending.front(); + warm_pending.pop_front(); + warm_noted.erase(e.first); + } + nvn_tex t; + nvn_view v; + bool has_view = false; + if (!texture_of(e.first, &t, &v, &has_view) || !t.gpu) continue; // (registered anew as nothing since, or let go) + any = true; + st_warm_looked++; + const uint64_t p = warm_place_of(t.d, t.L, t.gpu); + bool listed; + { + std::lock_guard lk(warm_list_mu); + listed = p && warm_list.count(p); + } + if (!listed) { + if (mode == 1 && warm_unlisted.size() < tex_warm::kMaxPending) warm_unlisted.push_back(e.first), st_list_unlisted++; + else st_list_left++; // (made at its first draw, as without the list) + } else if (!tex_warm::listed_room(pool_pressed, total_bytes, warm_budget)) { + st_list_noroom++; // (the same) + } else { + st_list_listed++; + st_warm_made += make(t); + } + if (!(more = goes_on())) break; + } + // (the slots the list does not name: only when no noted slot is left ripe, and while the 40% rule allows) + while (more && mode == 1 && !warm_unlisted.empty() && + tex_warm::has_room(warm_known, warm_used, warm_allowed, total_bytes, warm_budget)) { + const int slot = warm_unlisted.front(); + warm_unlisted.pop_front(); + nvn_tex t; + nvn_view v; + bool has_view = false; + if (!texture_of(slot, &t, &v, &has_view) || !t.gpu) continue; + any = true; + st_warm_looked++; + st_warm_made += make(t); + more = goes_on(); + } + warm_frame_ns += spent; + if (!any) return; + st_warm_ns += spent, st_warm_steps++; + static int64_t told; + if (st_warm_made - told >= 256) { + told = st_warm_made; + size_t listed; + { + std::lock_guard lk(warm_list_mu); + listed = warm_list.size(); + } + logf("vk: textures made ahead by the list (RX_VK_TEX_WARM_LIST): %lld made of %lld looked up, in %lld submissions' " + "starts, %.0f ms in all; listed %lld made, %lld without room; not listed %lld kept for later (%zu waiting), %lld " + "left to their draws; %lld learnt, %zu in the list", (long long)st_warm_made, (long long)st_warm_looked, + (long long)st_warm_steps, st_warm_ns / 1e6, (long long)st_list_listed, (long long)st_list_noroom, + (long long)st_list_unlisted, warm_unlisted.size(), (long long)st_list_left, + (long long)st_list_learnt.load(std::memory_order_relaxed), listed); + } +} +} // namespace +/// When the game made a BC or ASTC texture (any thread): its storage noted with the time, for warm_learn +void texture_warm_born(const nvn_tex* t) { + if (!texture_warm_list_mode() || !t || !t->gpu || t->gpu >= (1ull << 39)) return; + int bw = 1, bh = 1; + if (nvn_block_size(t->d.format, &bw, &bh) <= 0 || bw <= 1) return; // (block-compressed: what only the game's bytes fill) + const int64_t now = warm_now_ns(); + std::lock_guard lk(warm_born_mu); + if (warm_born.size() < kBornMost || warm_born.count(t->gpu)) warm_born[t->gpu] = now; +} +/// Its release (any thread): forgotten +void texture_warm_gone(const nvn_tex* t) { + if (!texture_warm_list_mode() || !t || !t->gpu) return; + std::lock_guard lk(warm_born_mu); + warm_born.erase(t->gpu); +} /// At a submission's start, on the thread that records it, in its recording: when the frame before was light, the noted /// slots old enough are looked up as a draw's first look does (texture_get: made, and its bytes uploaded by this recording's /// upload command buffer), until the budget is taken. last_frame_work: the draws and dispatches of the frame before void texture_warm_step(int64_t last_frame_work) { if (warm_frame != frame_no) warm_frame = frame_no, warm_frame_ns = 0; if (!tex_warm::may_warm(texture_warm_on(), last_frame_work, frame_upload_ns.load(std::memory_order_relaxed), warm_frame_ns)) return; + if (const int mode = texture_warm_list_mode()) return warm_step_listed(mode); // ((Triforce, 11-22, TEX-23) by the list) if (!tex_warm::has_room(warm_known, warm_used, warm_allowed, total_bytes, warm_budget)) return; const auto t0 = std::chrono::steady_clock::now(); int64_t spent = 0; @@ -3694,6 +4213,7 @@ void texture_reupload_early() { } } Texture* texture_touch(Texture* tex, bool for_sampling) { + if (tex->freed_listed) reuse_drop(*tex); // ((Triforce, 11-21, TEX-14) used again at its own address: its image is its own) tex->last_frame = frame_no; // ((Triforce) taken back after the game let go of it: the longest such gap, for texture_sweep's keep taken: tex_pressure.h) if (const uint64_t g = tex->let_go) tex->back_gap = tex_pressure::taken_back(tex->back_gap, frame_no - g); @@ -4387,8 +4907,8 @@ bool texture_copy_into_bc(VkCommandBuffer cmd, Texture* src, uint32_t src_level, } } if (!bc_scratch_layout(cmd, dst->bc_resized)) return false; - for (uint32_t l0 = 0; l0 < layers; l0 += kBcLayers) { - const uint32_t n = std::min(kBcLayers, layers - l0); + for (uint32_t l0 = 0; l0 < layers; l0 += bc_scratch_layers()) { // ((Triforce, 11-21, TEX-10) what the scratch holds at once) + const uint32_t n = std::min(bc_scratch_layers(), layers - l0); full_barrier(cmd); const VkImageCopy cp{{VK_IMAGE_ASPECT_COLOR_BIT, src_level, src_layer + l0, n}, {sx, sy, 0}, {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, n}, {0, 0, 0}, {w, h, 1}}; @@ -4416,6 +4936,9 @@ namespace { constexpr uint64_t kFlushMax = 16ull << 20; // (the small ones: render targets that size and more are not) std::mutex vw_mu; std::map vw; // (storage start -> end of the GPU-written textures of virtual pools that may be flushed) +// ((Triforce, 11-22, the performance audit's THREAD-25, after Source A's) every page a texture listed in vw ever had its storage +// in, set before the entry and under vw_mu, read with no lock: page_marks.h, texture_gpu_written_marked) +PageMarks vw_marks; bool flushable(const Texture& t) { if (!t.guest_size || t.guest_size > kFlushMax || t.astc || t.bc || t.sparse || t.skip || t.samples != VK_SAMPLE_COUNT_1_BIT || t.aspect != VK_IMAGE_ASPECT_COLOR_BIT || t.bpb <= 0 || !t.image || @@ -4429,6 +4952,7 @@ void vw_note(Texture& t) { static const bool off = getenv("RX_VK_NO_VMAP_FLUSH") != nullptr; if (off || t.vw_listed || !flushable(t)) return; std::lock_guard lk(vw_mu); + vw_marks.mark(t.gpu, t.gpu + t.guest_size); // ((Triforce, THREAD-25) before the entry: a reader with no mark finds no entry) uint64_t& end = vw[t.gpu]; end = std::max(end, t.gpu + t.guest_size); t.vw_listed = true; @@ -4633,6 +5157,17 @@ bool flush_finish(Flush& f, bool gpu_ok) { if (f.buf) destroy_buffer(f.buf, f.alloc); return ok; } +/// ((Triforce, 11-22, the performance audit's TEX-R404, after Source A's) For each texture the eviction is about to write back: +/// the frame of the newest drawing of another image the GPU drew over its storage (evict_reason.h). One walk of the cache, +/// the candidates in the order of their starts; only in a pass that has write-backs +void drawn_others(std::vector& cands) { + const uint64_t longest = evict_reason::by_start(cands); + for (auto& kv : textures) { + const Texture* o = kv.second.get(); + if (!o->gpu_written || !o->guest_size) continue; + evict_reason::note(cands, longest, o->gpu, o->gpu + o->guest_size, o->last_written, o); + } +} /// A batch of write-backs: one submission and one wait for all of them void flush_textures(std::vector& fs) { bool any = false; @@ -4694,6 +5229,11 @@ bool texture_gpu_written_in(uint64_t lo, uint64_t size) { if (it->second > lo) return true; return false; } +/// ((Triforce, THREAD-25) no lock) whether texture_gpu_written_in can be true for the range: a texture listed in vw ever had a page +/// of it, or the range lies where the marks are not kept. False: texture_gpu_written_in is false +bool texture_gpu_written_marked(uint64_t lo, uint64_t size) { + return vw_marks.any(lo, size); +} int texture_flush_to_guest(uint64_t lo, uint64_t size) { const uint64_t hi = lo + size; @@ -4774,6 +5314,22 @@ void texture_written(Texture* tex, const char* why) { // gpu_written and released are set for this write) tex->prev_written = redrawn::prev_on_write(tex->prev_written, tex->last_written, tex->gpu_written && !tex->released, frame_no); tex->last_written = frame_no; + // ((Triforce, 11-21, the performance audit's TEX-21, after Source A's) a texture in a virtual pool that the GPU draws + // first has no page map (the draw marks it uploaded, and only a check records one): once finalized, its first check took + // the whole storage as mapped since and uploaded it over what the GPU drew. The map is recorded here, at the first write + // that says who wrote it: drawn_page_map.h. A check with generation 0 walks the mapping and asks what was ever written, + // which is not used. Under the parallel replay's lock: a segment's worker may be checking it. RX_VK_DRAWN_PAGE_MAP=0: not) + static const bool drawn_map = drawn_page_map::on(getenv("RX_VK_DRAWN_PAGE_MAP")); + if (drawn_page_map::seeds(drawn_map, why, tex->gpu, tex->guest_size, tex->page_map.size())) { + ParLock lk; + static thread_local Ranges seeded; + texture_dirty(*tex, WriteGen{}, seeded); + static std::atomic n_seeded{0}; + if (const uint64_t n = n_seeded.fetch_add(1) + 1; (n & (n - 1)) == 0) + logf("vk: textures in virtual pools the GPU drew first, their page map recorded at that draw: %llu so far (the last " + "#%u %#llx %ux%u NVN %#x by %s; RX_VK_DRAWN_PAGE_MAP=0: at their first check, as before)", (unsigned long long)n, + tex->id, (unsigned long long)tex->gpu, tex->width, tex->height, tex->nvn_format, why); + } tex->gpu_written = true; upload_same::changed(tex->same); // ((Triforce) its image's bytes are the GPU's: no kept hash of a re-upload's is theirs) tex->gpu_gen = memory_generation(); @@ -4785,6 +5341,7 @@ void texture_written(Texture* tex, const char* why) { bool texture_touch_quick(Texture* tex, bool for_sampling) { + if (tex->freed_listed) reuse_drop(*tex); // ((Triforce, 11-21, TEX-14) as texture_touch) tex->last_frame = frame_no; // (the same values whichever thread stores them) if (const uint64_t g = tex->let_go) tex->back_gap = tex_pressure::taken_back(tex->back_gap, frame_no - g); // (texture_touch) tex->let_go = 0; @@ -4821,6 +5378,9 @@ void texture_gpu_wrote(uint64_t gpu, uint64_t size) { (unsigned long long)lo, (unsigned long long)size, (unsigned long long)wlo, (unsigned long long)whi, (unsigned long long)frame_no, (unsigned long long)tl_watch_prog); } + // ((Triforce, 11-21, the performance audit's TEX-02) its regions marked before the map has it: wrote_regions.h. Whatever the + // switch says, so that it can be read at any time) + wrote_regions::mark(lo, hi); std::lock_guard lk(gpu_wrote_mu); auto it = gpu_wrote.find(lo); if (it != gpu_wrote.end() && it->second.first >= hi) { @@ -4834,9 +5394,21 @@ void texture_gpu_wrote(uint64_t gpu, uint64_t size) { } /// A finished GPU write into the texture's storage it was not uploaded after (and notes it as seen) bool gpu_wrote_into(Texture& tex) { + const uint64_t lo = tex.gpu, hi = tex.gpu + std::max(tex.guest_size, 1); + // ((Triforce, 11-21, the performance audit's TEX-02) no region of its storage was ever reached by a GPU write: no, without + // the lock and the walk below. RX_VK_WROTE_REGIONS=0: asked of the map each time, as before; 2: both, and compared) + static const int regions = getenv("RX_VK_WROTE_REGIONS") ? std::clamp(atoi(getenv("RX_VK_WROTE_REGIONS")), 0, 2) : 1; + static std::atomic st_asked{0}, st_no{0}, st_wrong{0}; + const bool none = regions && !wrote_regions::any(lo, hi); + if (regions == 2 && (st_asked.fetch_add(1, std::memory_order_relaxed) + 1) % 200000 == 0) + logf("vk: RX_VK_WROTE_REGIONS=2: %llu asked, %llu answered no by the regions' bits, %llu of those the map said yes to (must be 0)", + (unsigned long long)st_asked.load(), (unsigned long long)st_no.load(), (unsigned long long)st_wrong.load()); + if (none) { + st_no.fetch_add(1, std::memory_order_relaxed); + if (regions == 1) return false; + } std::lock_guard lk(gpu_wrote_mu); if (gpu_wrote.empty()) return false; - const uint64_t lo = tex.gpu, hi = tex.gpu + std::max(tex.guest_size, 1); uint64_t newest = 0; auto it = gpu_wrote.upper_bound(lo); if (it != gpu_wrote.begin()) --it; @@ -4844,6 +5416,9 @@ bool gpu_wrote_into(Texture& tex) { if (it->second.first > lo && it->second.second <= completed_value && it->second.second > tex.gpu_wrote_seen) newest = std::max(newest, it->second.second); if (!newest) return false; + if (none && st_wrong.fetch_add(1, std::memory_order_relaxed) < 8) // ((Triforce, 11-21, TEX-02) the check's: must not happen) + logf("vk: RX_VK_WROTE_REGIONS=2: texture %#llx (+%#llx) was written by the GPU and its regions' bits said no", + (unsigned long long)lo, (unsigned long long)(hi - lo)); tex.gpu_wrote_seen = newest; static int logged; if (logged++ < 40) @@ -4905,6 +5480,14 @@ bool sparse_take(Texture* tex, VkDeviceSize n, VkDeviceMemory& mem, VkDeviceSize VkMemoryAllocateInfo ai{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; ai.allocationSize = std::max(kChunk, n); ai.memoryTypeIndex = tex->sparse_type; + // ((Triforce, 11-22, the performance audit's TEX-47, after Source A's) a sparse texture's chunks are not the + // allocator's: with the memory priorities on they name the textures' tier themselves (Source A left them at the + // driver's 0.5). Without the extension nothing is chained) + VkMemoryPriorityAllocateInfoEXT chunk_prio{VK_STRUCTURE_TYPE_MEMORY_PRIORITY_ALLOCATE_INFO_EXT}; + if (dev.memory_priority) { + chunk_prio.priority = memory_priority::of(memory_tiers(), memory_priority::Kind::Texture); + ai.pNext = &chunk_prio; + } VkDeviceMemory m = VK_NULL_HANDLE; if (vkAllocateMemory(dev.device, &ai, nullptr, &m) != VK_SUCCESS) { static int logged; @@ -5118,6 +5701,7 @@ struct Compact { uint64_t runs = 0, all_moved = 0, all_freed = 0; // (the driver's count over 90% of what it allows, texture_frame's look every 10 presents: vram_compact.h) bool pressed = false; + uint64_t allowed = 0; // ((Triforce, 11-22, TEX-38) and what it allows at that look, 0 before one: default_min_free) } compact; const Texture* present_source = nullptr; // (what the present shows this frame: do_present holds its image and view) void compact_step(uint64_t completed) { @@ -5144,7 +5728,11 @@ void compact_step(uint64_t completed) { if (!image_compact_active()) { if (frame_no < compact.next_check) return; compact.next_check = frame_no + 120; - static const uint64_t min_free = (uint64_t)(getenv("RX_VK_COMPACT_MB") ? std::max(1, atoi(getenv("RX_VK_COMPACT_MB"))) : 512) << 20; + // ((Triforce, 11-22, the performance audit's TEX-38, after Source A's) with no switch, a sixteenth of the allowance of the + // last look, 64 to 512 MB (vram_compact.h default_min_free): 512 on a card that allows 8 GB or more, as it was. + // RX_VK_COMPACT_MB=: that size, whatever the card) + static const uint64_t asked = (uint64_t)(getenv("RX_VK_COMPACT_MB") ? std::max(1, atoi(getenv("RX_VK_COMPACT_MB"))) : 0) << 20; + const uint64_t min_free = asked ? asked : vram_compact::default_min_free(compact.allowed); uint64_t block_bytes = 0, free_bytes = 0; if (!textures_pool_usage(&block_bytes, &free_bytes) || free_bytes < vram_compact::run_from(min_free, compact.pressed)) @@ -5422,7 +6010,7 @@ bool swizzle_back_ready() { ci.stage.module = m; ci.stage.pName = "main"; ci.layout = swz_layout; - const VkResult r = vkCreateComputePipelines(dev.device, VK_NULL_HANDLE, 1, &ci, nullptr, &swz_pipe); + const VkResult r = texture_compute_pipeline(ci, &swz_pipe); vkDestroyShaderModule(dev.device, m, nullptr); if (r != VK_SUCCESS) return false; state = 1; @@ -5654,6 +6242,55 @@ struct Press { size_t step = 0; // (live textures a pass over the budget lets go of at most; 0: no step) uint64_t next = 0; // (the present of the next look while a pass stopped at its step; 0: the next tenth) } press; +// ((Triforce, 11-22, the performance audit's OS-05, after Source A's: commit_room.h) the switches, read once. +// RX_VK_TEX_COMMIT_RESERVE_MB=: the textures give back the shortfall of free commit under it (off unless set); +// RX_VK_TEX_COMMIT_GROWTH: the page files' room to grow counted as free (0 never, 1 always, unset with a reserve) +uint64_t commit_reserve() { + static const uint64_t v = commit_room::reserve_of(getenv("RX_VK_TEX_COMMIT_RESERVE_MB")); + return v; +} +bool commit_growth_on() { + static const bool v = commit_room::growth_on(getenv("RX_VK_TEX_COMMIT_GROWTH"), commit_reserve()); + return v; +} +int64_t commit_free_mb() { // (RX_VK_TEX_COMMIT_FREE_MB, tex_pressure::commit_low's: -1 for its default) + static const int64_t v = getenv("RX_VK_TEX_COMMIT_FREE_MB") ? atoll(getenv("RX_VK_TEX_COMMIT_FREE_MB")) : -1; + return v; +} +/// ((Triforce, 11-22, OS-05) The page files' room to grow, capped by their drives' free space (commit_room::growth over the +/// records Windows gives), asked at most once a second: the last answer meanwhile. 0 on any failure, and elsewhere than Windows +uint64_t page_file_growth() { +#ifdef _WIN32 + static uint64_t last = 0; + static std::chrono::steady_clock::time_point asked_at{}; + static bool asked = false; + const auto now = std::chrono::steady_clock::now(); + if (asked && now - asked_at < std::chrono::seconds(1)) return last; + asked = true, asked_at = now, last = 0; + using Query = LONG(WINAPI*)(ULONG, PVOID, ULONG, PULONG); + static const Query query = (Query)(void*)GetProcAddress(GetModuleHandleW(L"ntdll.dll"), "NtQuerySystemInformation"); + if (!query) return 0; + std::vector buf(512); // (4 KB, 8-byte aligned, as the records' names are pointers) + ULONG got = 0; + LONG status; + while ((status = query(commit_room::kPageFileClass, buf.data(), (ULONG)(buf.size() * 8), &got)) == (LONG)0xC0000004L && + buf.size() < 8192) // (STATUS_INFO_LENGTH_MISMATCH: room for more records, to 64 KB) + buf.resize(buf.size() * 2); + if (status < 0) return 0; + SYSTEM_INFO si{}; + GetSystemInfo(&si); + last = commit_room::growth((const uint8_t*)buf.data(), std::min(got, buf.size() * 8), si.dwPageSize ? si.dwPageSize : 4096, + [](char drive) -> uint64_t { + const wchar_t root[] = {(wchar_t)drive, L':', L'\\', 0}; + ULARGE_INTEGER free_bytes{}; + return GetDiskFreeSpaceExW(root, &free_bytes, nullptr, nullptr) ? free_bytes.QuadPart : 0; + }); + return last; +#else + return 0; +#endif +} +uint64_t commit_grown = 0; // ((Triforce, OS-05) the growth the last commit_now counted in, for the log) /// (Triforce) What the system may still commit and its limit: Windows charges a discrete card's memory to the commit too /// (tex_pressure.h commit_low). False where it is not known (elsewhere device memory is not charged so) bool commit_now(uint64_t* avail, uint64_t* limit) { @@ -5662,6 +6299,14 @@ bool commit_now(uint64_t* avail, uint64_t* limit) { ms.dwLength = sizeof ms; if (!GlobalMemoryStatusEx(&ms)) return false; *avail = ms.ullAvailPageFile, *limit = ms.ullTotalPageFile; + // ((Triforce, 11-22, OS-05) and, where it is counted and less is free than the reserve or commit_low's line, what the page + // files may still grow by: free commit to come, which commit_low and the reserve's cut see alike. Not counted unless one of + // RX_VK_TEX_COMMIT_RESERVE_MB and RX_VK_TEX_COMMIT_GROWTH is set: then nothing changes here) + commit_grown = 0; + if (commit_room::read_growth(commit_growth_on(), *avail, commit_reserve(), commit_room::low_line(*limit, commit_free_mb()))) { + commit_grown = page_file_growth(); + *avail += commit_grown, *limit += commit_grown; + } return true; #else (void)avail, (void)limit; @@ -5761,14 +6406,34 @@ void texture_frame(uint64_t completed) { uint64_t used = 0, allowed = 0; return (vram_usage(&used, &allowed) && allowed <= (6ull << 30) + (512ull << 20) ? 64ull : 256ull) << 20; }(); + // ((Triforce, 11-22, the performance audit's TEX-28, after Source A's) An arena is taken again once the GPU is done + // with the submission that read it, known a present later, so frames that stage in a row use two in turn; where 64 MB + // are kept the one that had its turn was freed at every second present of a load and made again the present after + // (9 to 41 ms each on a handheld). An idle arena staged into in the last RX_VK_STAGING_BURST_S seconds (2) may stay + // while the arenas add up to 256 MB, unless VRAM is over 90% of the allowance (pool_pressed, the last look's): + // staging_keep.h. Where 256 MB are kept anyway nothing changes. =0: as before) + static const int64_t burst_s = staging_keep::burst_s(getenv("RX_VK_STAGING_BURST_S")); uint64_t kept = 0; + // ((Triforce, 11-22, the performance audit's MEM-141, after Source A's) while the swap hold is on (linux_pressure.h; + // never on Windows): the arenas the GPU is done with go as the hold begins, and after 5 s unused from then on + const bool swap_hold_on = memory_swap_hold(); + static bool swap_hold_was = false; + const bool swap_hold_began = swap_hold_on && !swap_hold_was; + swap_hold_was = swap_hold_on; for (auto it = arenas.begin(); it != arenas.end();) { const bool idle = it->value <= completed && it->value != rec_value; // (and one nothing has staged into for 30 s: Kokiri Forest's gameplay stages next to nothing, and the 64 MB it kept are VRAM // behind the BAR. A load makes one again. RX_VK_STAGING_IDLE_S=: that long, 0: kept) static const int64_t idle_s = getenv("RX_VK_STAGING_IDLE_S") ? atoll(getenv("RX_VK_STAGING_IDLE_S")) : 30; - const bool stale = idle_s && std::chrono::steady_clock::now() - it->used_at > std::chrono::seconds(idle_s); - if (idle && (it->size > kArena || kept + it->size > keep || stale)) { + const int64_t idle_now = linux_pressure::arena_idle_s(swap_hold_on, idle_s); // ((Triforce, 11-22, MEM-141)) + const bool stale = idle_now && (swap_hold_began || std::chrono::steady_clock::now() - it->used_at > std::chrono::seconds(idle_now)); + const int64_t since_used_ms = + std::chrono::duration_cast(std::chrono::steady_clock::now() - it->used_at).count(); + // ((Triforce, 11-22) TEX-28's room through a load's burst, but not while MEM-141's swap hold is on) + const uint64_t allowance = + swap_hold_on ? keep : staging_keep::allowance(keep, staging_keep::recent(burst_s, pool_pressed, since_used_ms)); + if (idle && (it->size > kArena || kept + it->size > allowance || stale)) { + if (swap_hold_on) memory_swap_valve(1, it->size); destroy_buffer(it->buffer, it->allocation); it = arenas.erase(it); continue; @@ -5780,19 +6445,24 @@ void texture_frame(uint64_t completed) { PROCESS_MEMORY_COUNTERS_EX pm{}; pm.cb = sizeof pm; if (K32GetProcessMemoryInfo(GetCurrentProcess(), (PROCESS_MEMORY_COUNTERS*)&pm, sizeof pm)) - logf("vk: memory: working set %llu MB (peak %llu), private %llu MB, staging %llu MB in %zu arenas", + logf("vk: memory: working set %llu MB (peak %llu), private %llu MB, staging %llu MB in %zu arenas, %lld made so " + "far (%.0f ms)", (unsigned long long)(pm.WorkingSetSize >> 20), (unsigned long long)(pm.PeakWorkingSetSize >> 20), - (unsigned long long)(pm.PrivateUsage >> 20), (unsigned long long)(kept >> 20), arenas.size()); + (unsigned long long)(pm.PrivateUsage >> 20), (unsigned long long)(kept >> 20), arenas.size(), + (long long)st_arena_makes.load(std::memory_order_relaxed), + (double)st_arena_make_ns.load(std::memory_order_relaxed) / 1e6); #if defined(__linux__) && defined(__GLIBC__) // (glibc's heap: what its arenas hold free. The shader translation and the loads' decoding leave their garbage spread through // the threads' arenas (26 of them, 1.2 GB resident in Kokiri Forest, the heap's growth in a scene change never going back), // and a free chunk's pages are the process's till malloc_trim gives them to the system. Asked on a worker, when 192 MB or // more are free: RX_VK_MALLOC_TRIM_MB=: that many, 0: never; the "vk: malloc:" line says what the arenas hold) // (every 1800 presents: mallinfo2 walks every free chunk of every arena with its lock held) + // ((Triforce, 11-22, the performance audit's MEM-141) while the swap hold is on: every 600 presents, from 64 MB free static std::atomic busy{false}; - if (frame_no % 1800 == 0 && !busy.exchange(true)) - jobs_push([] { - static const uint64_t trim_mb = getenv("RX_VK_MALLOC_TRIM_MB") ? (uint64_t)atoll(getenv("RX_VK_MALLOC_TRIM_MB")) : 192; + if (frame_no % linux_pressure::trim_every(swap_hold_on, 1800) == 0 && !busy.exchange(true)) + jobs_push([swap_hold_on] { + static const uint64_t trim_set = getenv("RX_VK_MALLOC_TRIM_MB") ? (uint64_t)atoll(getenv("RX_VK_MALLOC_TRIM_MB")) : 192; + const uint64_t trim_mb = linux_pressure::trim_mb(swap_hold_on, trim_set); const struct mallinfo2 a = mallinfo2(); PROCESS_MEMORY_COUNTERS_EX pm0{}; pm0.cb = sizeof pm0; @@ -5820,6 +6490,12 @@ void texture_frame(uint64_t completed) { bake_writeback_step(completed); // (Triforce: what the GPU baked, into the game's memory too) bake_reupload_step(completed); // (lab: RX_VK_BAKE_REUPLOAD) compact_step(completed); // (hookshot, RX_VK_COMPACT: while its pass is open nothing of the textures' pool is freed) + // ((Triforce, 11-21, the performance audit's THREAD-21, its first step) what this loop costs a present is counted, and + // nothing else changes: dead_count.h. The port's bounded step is built only if a present spends 3 ms here) + if (frame_no % 600 == 0) reuse_report(); // ((Triforce, 11-21, TEX-14) decoded ASTC used again: what it cost and found) + static dead_count::Counts dead_ct; + uint64_t dead_destroyed = 0, dead_pooled = 0; + const auto dead_t0 = std::chrono::steady_clock::now(); for (auto it = dead.begin(); it != dead.end();) { if (compact.in_pass) break; // (hookshot, RX_VK_COMPACT: nothing is freed while a pass is open; a frame or two) if (it->value > completed) { @@ -5829,6 +6505,7 @@ void texture_frame(uint64_t completed) { if (it->pool_key) { // (a render target's image: kept for the next of its shape, above) pool_put(*it); it = dead.erase(it); + dead_pooled++; continue; } for (VkImageView v : it->views) vkDestroyImageView(dev.device, v, nullptr); @@ -5836,6 +6513,23 @@ void texture_frame(uint64_t completed) { if (it->buffer) destroy_buffer(it->buffer, it->buffer_alloc); for (VkDeviceMemory m : it->memory) vkFreeMemory(dev.device, m, nullptr); it = dead.erase(it); + dead_destroyed++; + } + { + const double dead_ms = std::chrono::duration(std::chrono::steady_clock::now() - dead_t0).count(); + if (dead_ct.note(dead_destroyed, dead_pooled, dead_ms, dead.size()) && dead_ct.over_say <= 64) + logf("vk: dead images: %llu destroyed and %llu pooled in %.1f ms in one present, %zu left waiting for the GPU", + (unsigned long long)dead_destroyed, (unsigned long long)dead_pooled, dead_ms, dead.size()); + if (dead_ct.due(frame_no)) { + if (dead_ct.destroyed || dead_ct.pooled) + logf("vk: dead images (%llu presents): %llu destroyed and %llu pooled in %.1f ms in all; the worst present %.2f ms " + "(%llu destroyed, %llu left); %llu presents of %.0f ms or more, %llu of %.0f ms or more", + (unsigned long long)dead_ct.presents, (unsigned long long)dead_ct.destroyed, (unsigned long long)dead_ct.pooled, + dead_ct.ms, dead_ct.worst_ms, (unsigned long long)dead_ct.worst_destroyed, (unsigned long long)dead_ct.worst_left, + (unsigned long long)dead_ct.over_say, dead_count::kSayMs, (unsigned long long)dead_ct.over_worth, + dead_count::kWorthMs); + dead_ct.reset(); + } } pool_trim(); // A budget: RX_VK_VRAM_MB of textures (default 4096), and less when the driver says the process is near what it @@ -5890,6 +6584,11 @@ void texture_frame(uint64_t completed) { const bool preset_budget = opts.vram_mb > 0 && !vram_env && preset_caps_memory(); uint64_t budget = preset_texture_budget((uint64_t)std::max(opts.vram_mb, 0), preset_budget, vram_env, dev.props.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU, own); + // ((Triforce, 11-22, the performance audit's MEM-141, after Source A's) an integrated GPU's textures are the PC's RAM: + // while the swap hold says the PC is short the budget is less by that much, never under 512 MB, and comes back 256 MB a + // sample (linux_pressure.h; RX_VK_VRAM_MB decides outright; 0 on Windows and without RX_VK_SWAP_RESERVE_MB) + if (const uint64_t cut = memory_swap_cut(); cut && !vram_env && dev.props.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU) + budget = linux_pressure::cut_budget(budget, cut); if (preset_budget && known) { // Safety clamp: do not exceed 75% of driver-allowed memory on constrained VRAM (prevents VK_ERROR_DEVICE_LOST) budget = std::min(budget, (allowed / 4) * 3); @@ -5925,7 +6624,7 @@ void texture_frame(uint64_t completed) { // ends the game. RX_VK_TEX_COMMIT_FREE_MB=: low under that much free, 0: never; NX2navi's default: 4 GB or a tenth) bool commit_low = false; if (press.live) { - static const int64_t want_mb = getenv("RX_VK_TEX_COMMIT_FREE_MB") ? atoll(getenv("RX_VK_TEX_COMMIT_FREE_MB")) : -1; + static const int64_t want_mb = commit_free_mb(); uint64_t avail = 0, limit = 0; commit_low = want_mb && commit_now(&avail, &limit) && tex_pressure::commit_low(avail, limit, want_mb); static bool was_low; @@ -5972,12 +6671,17 @@ void texture_frame(uint64_t completed) { } const uint64_t pressure = by_blocks ? used : used - std::min(used, reclaim); pool_pressed = known && pressure > allowed / 10 * 9; // (the target pool given back meanwhile) + pool_tier = target_pool::tier(known, pressure, allowed); // ((Triforce, 11-21, TEX-29) halved above 90%, none above 97%) compact.pressed = known && vram_compact::pressed(used, allowed); // (hookshot, RX_VK_COMPACT: by the driver's own count) + compact.allowed = known ? allowed : 0; // ((Triforce, 11-22, TEX-38) the free space a run starts from, by the card) // ((Triforce) the live line in the size's place (above, tex_pressure.h): none while under it, and once past it the textures // less the excess over 2% under it, at each look until the use is there; the fixed size while the commit is low. Before the // 90% rule, which then asks for less) press.acting = press.live && !press.held && tex_pressure::acts(pressure, allowed, press_pct, press.acting); if (press.live) budget = tex_pressure::budget(own, total_bytes, pressure, allowed, press_pct, commit_low, press.acting); + // ((Triforce) the budget as a size (the fixed one, or the live line's), before the 90% and 97% rules take their excess off + // it: rule_to below) + const uint64_t size_budget = budget; if (known && pressure > allowed / 10 * 9) { const uint64_t over = pressure - allowed / 10 * 9; budget = std::min(budget, total_bytes > over ? total_bytes - over : 0); @@ -6017,7 +6721,35 @@ void texture_frame(uint64_t completed) { budget = std::min(budget, total_bytes > over ? total_bytes - over : 0); rule_budget = std::min(rule_budget, total_bytes > over ? total_bytes - over : 0); } - const uint64_t rule_to = vram_compact::rule_target(total_bytes, rule_budget); + // ((Triforce, 11-22, the performance audit's OS-05, after Source A's: commit_room.h) Near Windows' commit limit, on any card: + // while less commit is free than RX_VK_TEX_COMMIT_RESERVE_MB (the page files' room to grow counted in), the textures at most + // what they hold less that shortfall, never under 1 GB, as the 97% rule takes its excess. Off unless the reserve is set; + // not on Linux, where commit_now knows nothing + if (const uint64_t reserve = commit_reserve()) { + uint64_t avail = 0, limit = 0; + const uint64_t cap = commit_now(&avail, &limit) ? commit_room::cap(total_bytes, avail, reserve) : UINT64_MAX; + budget = std::min(budget, cap); + rule_budget = std::min(rule_budget, cap); + static bool was_near; + static int told; + const bool near_now = cap != UINT64_MAX; + if ((near_now != was_near && told++ < 16) || (near_now && frame_no % 600 == 0)) + logf("vk: commit limit %s: %llu MB of %llu free (%llu MB of it the page files' room to grow), the reserve %llu MB: textures " + "%llu MB, %s (RX_VK_TEX_COMMIT_RESERVE_MB=0: no cut)", near_now ? "near" : "no longer near", + (unsigned long long)(avail >> 20), (unsigned long long)(limit >> 20), (unsigned long long)(commit_grown >> 20), + (unsigned long long)(reserve >> 20), (unsigned long long)(total_bytes >> 20), + near_now ? "held to what they hold less the shortfall, 1 GB at least" : "as before"); + was_near = near_now; + } + // ((Triforce) RX_VK_EVICT_EXCESS, on by default; =0 or the lab's knob noexcess: 7/8 of the rules' budget, as before) the 90% + // and 97% rules' part of the pass goes down to their budget less 64 MB, the size's to 7/8 of it: vram_compact.h + static const bool excess_env = [] { + const char* e = getenv("RX_VK_EVICT_EXCESS"); + return !e || atoi(e) != 0; + }(); + const uint64_t rule_to = excess_env && !lab_ab::on("noexcess") + ? vram_compact::rule_target(total_bytes, size_budget, rule_budget) + : vram_compact::rule_target(total_bytes, rule_budget); // (hookshot, RX_VK_COMPACT: the driver's limit asks for more than the reserve does: a pass over the budget, to 7/8 of it, as // before. Rare once the reserve is set: its line is 17 points under the limit's and counts the textures' free space out, so // this takes more than that share of the allowance free in their pool. Otherwise the pass is the hold's, and what the @@ -6030,8 +6762,9 @@ void texture_frame(uint64_t completed) { { std::lock_guard lk(pool_mu); logf("vk: render target images: %llu reused from the pool, %llu made new, %llu destroyed from it; it holds %llu " - "(%llu MB)%s", (unsigned long long)st_pool_reused, (unsigned long long)st_pool_made, + "(%llu MB of %llu, tier %d)%s", (unsigned long long)st_pool_reused, (unsigned long long)st_pool_made, (unsigned long long)st_pool_destroyed, (unsigned long long)pool_n, (unsigned long long)(pool_bytes >> 20), + (unsigned long long)(pool_cap() >> 20), pool_tier, pool_cap() ? (pool_pressed ? ", VRAM short: given back" : "") : " (RX_VK_TARGET_POOL_MB=0)"); } // ((Triforce) by live use: the budget the line sets, or while it sets none what the textures may hold before they reach @@ -6049,6 +6782,17 @@ void texture_frame(uint64_t completed) { textures.size(), live_note, known && used > allowed / 100 * 97 && room_in_blocks ? " (at the driver's limit, room in the blocks: not squeezed)" : ""); // (what takes it: decoded ASTC, the GPU's images, the rest; the largest) + // ((Triforce, 11-22, the performance audit's DESC-30, after Source B's) RX_VK_LOG_BAKED read once, its list filled + // only for it, and the largest few of each list found by a partial sort: every 600 presents this filled a list only + // that switch reads and sorted every texture to print eight. The lines' text is as it was (which of several + // textures of one size is listed was never fixed). The lab's "diagold" phases: both lists and the whole sorts + static const bool log_baked = getenv("RX_VK_LOG_BAKED") != nullptr; + const bool diag_old = lab_ab::on("diagold"); + const auto larger = [](const Texture* a, const Texture* b) { return a->bytes > b->bytes; }; + const auto largest = [&](std::vector& list, size_t n) { + if (diag_old) std::sort(list.begin(), list.end(), larger); + else std::partial_sort(list.begin(), list.begin() + (ptrdiff_t)std::min(n, list.size()), list.end(), larger); + }; uint64_t astc_b = 0, rt_b = 0, other_b = 0, recent_b = 0, baked_b = 0; size_t baked_n = 0; std::vector big, baked; @@ -6062,17 +6806,17 @@ void texture_frame(uint64_t completed) { if (!t->released) baked_n++, baked_b += t->bytes; const int k = t->bytes <= (64u << 10) ? 0 : t->bytes <= (256u << 10) ? 1 : t->bytes <= (1u << 20) ? 2 : 3; bucket_n[k]++, bucket_b[k] += t->bytes, bucket_rel[k] += t->released; - baked.push_back(t); + if (log_baked || diag_old) baked.push_back(t); } big.push_back(t); } - if (getenv("RX_VK_LOG_BAKED")) { // (what the textures drawn in one frame are: by size, finalized or not) + if (log_baked) { // (what the textures drawn in one frame are: by size, finalized or not) logf("vk: drawn in one frame: <=64 KB %zu (%llu KB, %zu finalized), <=256 KB %zu (%llu KB, %zu), <=1 MB %zu " "(%llu KB, %zu), <=4 MB %zu (%llu KB, %zu)", bucket_n[0], (unsigned long long)(bucket_b[0] >> 10), bucket_rel[0], bucket_n[1], (unsigned long long)(bucket_b[1] >> 10), bucket_rel[1], bucket_n[2], (unsigned long long)(bucket_b[2] >> 10), bucket_rel[2], bucket_n[3], (unsigned long long)(bucket_b[3] >> 10), bucket_rel[3]); - std::sort(baked.begin(), baked.end(), [](const Texture* a, const Texture* b) { return a->bytes > b->bytes; }); + largest(baked, 10); for (size_t i = 0; i < baked.size() && i < 10; i++) { const Texture* t = baked[i]; logf("vk: one frame: #%u %#llx %ux%ux%u x%u levels %u NVN %#x %llu KB, written at %llu, made at %llu, last " @@ -6082,7 +6826,7 @@ void texture_frame(uint64_t completed) { t->first_render_frame ? " rendered" : " copied or stored"); } } - std::sort(big.begin(), big.end(), [](const Texture* a, const Texture* b) { return a->bytes > b->bytes; }); + largest(big, 8); logf("vk: textures: decoded ASTC %llu MB, rendered %llu MB (%zu drawn in one frame, kept: %llu KB), other %llu MB; " "used in the last 8 presents %llu MB", (unsigned long long)(astc_b >> 20), (unsigned long long)(rt_b >> 20), baked_n, (unsigned long long)(baked_b >> 10), (unsigned long long)(other_b >> 20), @@ -6117,6 +6861,7 @@ void texture_frame(uint64_t completed) { verify_dim().second, checked, changed, missed); } warm_known = known, warm_used = used, warm_allowed = allowed, warm_budget = budget; // ((Triforce) tex_warm.h: the room there is) + warm_list_save_due(); // ((Triforce, 11-22, TEX-23) the list of what was drawn, when it changed) texture_sweep(completed, total_bytes <= budget, budget, evict_to, known, used, allowed, false, hold_to, rule_to); } @@ -6201,6 +6946,25 @@ void texture_sweep(uint64_t completed, bool sweep, uint64_t budget, uint64_t evi if (o->second != tex && o->second->last_written > tex->last_frame) return true; return false; }; + // ((Triforce, 11-21, the performance audit's TEX-76, after Source B's) and a depth target of one level and layer that a + // newer target of another size, at the same start and of the same format, was drawn over since its last use: a character's + // shadow map of the size before. It goes once unused for RX_VK_SLOT_RETIRE presents (16), not the sweep's 120; with + // nothing written back, as any drawn-over one (drawn_over is true of it). slot_retire.h has the rule; 0: as before) + static const uint64_t retire_wait = slot_retire::wait(getenv("RX_VK_SLOT_RETIRE")); + static uint64_t st_slot_retired; + const auto slot_target = [](const Texture* t) { + return slot_retire::Target{t->gpu_written, (t->aspect & VK_IMAGE_ASPECT_DEPTH_BIT) != 0, t->levels, t->layers, t->depth, + t->nvn_format, t->width, t->height, t->last_frame, t->last_written}; + }; + const auto slot_retired = [&](const Texture* tex) { + if (!retire_wait || !tex->gpu_written || frame_no - tex->last_frame < retire_wait) return false; + const slot_retire::Target old = slot_target(tex); + auto [a, e] = by_gpu.equal_range(tex->gpu); + for (auto o = a; o != e; ++o) + if (o->second != tex && slot_retire::retired_by(old, slot_target(o->second), frame_no - tex->last_frame, retire_wait)) + return true; + return false; + }; // (hookshot-amd) a render target of one level and layer whose storage another render target drew over from another start // after its last use (target_overlap.h): the transient targets above. Its texels are gone from that memory on the console too const auto drawn_over_shifted = [&](const Texture* tex) { @@ -6210,6 +6974,19 @@ void texture_sweep(uint64_t completed, bool sweep, uint64_t budget, uint64_t evi const uint64_t lo = tex->gpu, hi = tex->gpu + tex->guest_size; auto it = std::lower_bound(by_start.begin(), by_start.end(), lo > longest ? lo - longest : 0, [](const Texture* t, uint64_t g) { return t->gpu < g; }); + // ((Triforce, 11-21, the performance audit's TEX-30, after Source A's) the others' parts added up, each byte once: one + // drawn over at its head by one target and at its tail by another, each under an eighth, goes too (target_overlap.h + // ShiftedOver; by_start is in the order of the starts). RX_VK_DEAD_SUM=0: one by one, as before) + static const bool dead_sum = !(getenv("RX_VK_DEAD_SUM") && !atoi(getenv("RX_VK_DEAD_SUM"))); + if (dead_sum) { + target_overlap::ShiftedOver over{lo, hi, tex->last_frame}; + for (; it != by_start.end() && (*it)->gpu < hi; ++it) { + if (*it == tex) continue; + over.add((*it)->gpu, (*it)->gpu + (*it)->guest_size, (*it)->last_written); + if (over.an_eighth()) return true; + } + return false; + } for (; it != by_start.end() && (*it)->gpu < hi; ++it) if (*it != tex && target_overlap::shifted_over(lo, hi, tex->last_frame, (*it)->gpu, (*it)->gpu + (*it)->guest_size, (*it)->last_written)) @@ -6219,8 +6996,11 @@ void texture_sweep(uint64_t completed, bool sweep, uint64_t budget, uint64_t evi // (the transient targets: a frame's own, not sampled again, their storage drawn over by another since. 16 frames left // ~500 MB of them in Hyrule Castle Town's market at 20 fps (59 768 x 432 RGBA16F, 135 384 x 384, 41 1536 x 864 at a // time), VRAM at the driver's limit; the GPU's own use is checked below (last_value). RX_VK_DEAD_MIN_UNUSED=, 3) + // ((Triforce, 11-21, TEX-30, after Source A's) 1 when not set: of the target images made new, four in five had last + // frame's one of their shape still alive and waiting out its three presents. The one drawn over since its last use is + // dead after one; the GPU's own use (last_value) and the written-after-last-use test are as they were. =3: as before) static const uint64_t dead_min_unused = - getenv("RX_VK_DEAD_MIN_UNUSED") ? (uint64_t)std::max(1, atoi(getenv("RX_VK_DEAD_MIN_UNUSED"))) : 3; + getenv("RX_VK_DEAD_MIN_UNUSED") ? (uint64_t)std::max(1, atoi(getenv("RX_VK_DEAD_MIN_UNUSED"))) : 1; // (hookshot) a small texture the GPU drew in one frame that the game has finalized, with nothing drawn over its storage // since: the game may make a texture over that storage again and sample it without drawing it. OoT's terrain does that with its // tiles' 260 x 260 RG8 normal maps: drawn when a chunk comes in, finalized when it goes out of range, made again and sampled as @@ -6231,9 +7011,30 @@ void texture_sweep(uint64_t completed, bool sweep, uint64_t budget, uint64_t evi // may have put other things there since, and a write-back over scene data once crashed it) static const uint64_t keep_finalized_max = (uint64_t)(getenv("RX_VK_KEEP_FINALIZED_MB") ? std::max(0, atoi(getenv("RX_VK_KEEP_FINALIZED_MB"))) : 512) << 20; + // ((Triforce, 11-21, the performance audit's TEX-43, after Source A's) One of them whose storage the game has written over + // since (three quarters of it or more by the CPU since the release, finalized_obsolete.h): its image holds nothing that + // memory holds on the console. Within the budget the pass lets go of nothing the GPU did not draw over, and these were + // never drawn over by a target: the game's own writes to that memory kept every one for good, about 200 render target + // images an hour in players' logs, with the blocks they pin. It goes whatever the budget says, as one drawn over, and + // is not counted among the kept. RX_VK_KEEP_OBSOLETE=1: kept, as before) + static const bool keep_obsolete = finalized_obsolete::keep(getenv("RX_VK_KEEP_OBSOLETE")); + static uint64_t st_obsolete_n, st_obsolete_bytes; + const auto obsolete = [&](const Texture* tex) { + if (keep_obsolete || !finalized_obsolete::candidate(tex->released, tex->gpu_written, tex->gpu, tex->guest_size)) return false; + if (!tex->obsolete && finalized_obsolete::due(frame_no, tex->obsolete_checked)) { + static thread_local Ranges written; + written.clear(); + tex->obsolete_checked = frame_no; + memory_written_runs(tex->guest, tex->guest_size, tex->release_gen, written, true); // (as texture_dirty's, same_ok) + uint64_t sum = 0; + for (const auto& r : written) sum += r.second - r.first; + tex->obsolete = finalized_obsolete::rewritten(sum, tex->guest_size); + } + return tex->obsolete; + }; const auto keep_finalized = [&](const Texture* tex) { if (!keep_finalized_max || !tex->gpu_written || !tex->released || !tex->first_written || - tex->first_written != tex->last_written || tex->bytes > (4ull << 20)) + tex->first_written != tex->last_written || tex->bytes > (4ull << 20) || obsolete(tex)) return false; const auto g = gpu_owned.find(tex->gpu); // (its storage's range still the one the GPU drew it in) if (g == gpu_owned.end() || g->second.first != tex->gpu + std::max(tex->guest_size, 1)) return false; @@ -6285,8 +7086,8 @@ void texture_sweep(uint64_t completed, bool sweep, uint64_t budget, uint64_t evi if (too_young(last)) break; if (tex->last_value > completed) continue; const bool shifted = drawn_over_shifted(tex); - if (age60(frame_no - last) < need_unused && !shifted) continue; - if (shifted || drawn_over(tex)) hold_dead += tex->bytes + tex->pad; + if (age60(frame_no - last) < need_unused && !shifted && !slot_retired(tex)) continue; // ((Triforce, 11-21, TEX-76)) + if (shifted || drawn_over(tex) || obsolete(tex)) hold_dead += tex->bytes + tex->pad; } // (Triforce, 11-12) What the game has let go goes first while VRAM is short. At an area's arrival the textures of the area // left stayed until a rule asked for room, which was after the new area's had come in: those took new blocks, the old ones @@ -6328,7 +7129,14 @@ void texture_sweep(uint64_t completed, bool sweep, uint64_t budget, uint64_t evi break; const bool shifted = drawn_over_shifted(tex); const bool gone_early = let_go_short && tex->let_go && !tex->gpu_written && let_go_due(tex); - if (age60(frame_no - last) < need_unused && !shifted && !gone_early) continue; // (younger: only a dead transient target goes) + // (younger: only a dead transient target goes, (Triforce, 11-21, TEX-76) or a depth target another size replaced) + const bool young = age60(frame_no - last) < need_unused; + const bool retired = young && !shifted && !gone_early && slot_retired(tex); + if (young && !shifted && !gone_early && !retired) continue; + if (retired && (++st_slot_retired & (st_slot_retired - 1)) == 0) // (the 1st, 2nd, 4th, ...) + logf("vk: depth targets another size replaced at their start, let go %llu presents after their last use: %llu so far " + "(the last #%u %ux%u; RX_VK_SLOT_RETIRE=0: after the sweep's own wait, as before)", + (unsigned long long)retire_wait, (unsigned long long)st_slot_retired, tex->id, tex->width, tex->height); if (tex->last_value > completed) continue; // (nor what the GPU wrote and the game still has: the image is its only copy, and one made again over that // storage started empty or from stale memory: a sky or course texture drawn once, garbage from then on. @@ -6341,7 +7149,9 @@ void texture_sweep(uint64_t completed, bool sweep, uint64_t budget, uint64_t evi // RX_VK_KEEP_RENDERED_OVER=1: only past what it allows, as before) static const bool keep_over_only = getenv("RX_VK_KEEP_RENDERED_OVER") != nullptr; const uint64_t keep_rendered = keep_rendered::presents(known, used, allowed, keep_over_only); - const bool overwritten = shifted || drawn_over(tex); // (above) + // (above; (Triforce, 11-21, TEX-43) or a finalized one whose storage the CPU has rewritten: obsolete, above) + const bool stale = obsolete(tex); + const bool overwritten = shifted || drawn_over(tex) || stale; // ((Triforce) a pass over the budget lets go of press.step live textures at most (64; by live use, tex_pressure.h), the // rest at the next look, two presents on: at the Market's arrival after a tour one pass took 134 to 645 at once, each made // again within seconds. Not the render targets drawn over since: nothing comes back for them) @@ -6400,6 +7210,13 @@ void texture_sweep(uint64_t completed, bool sweep, uint64_t budget, uint64_t evi static const bool no_flush = getenv("RX_VK_EVICT_NO_FLUSH") != nullptr; evict.push_back({tex, !no_flush && !overwritten && tex->gpu_written && !tex->released && flushable(*tex), shifted}); pending_bytes += tex->bytes + tex->pad; + if (stale && !shifted && !drawn_over(tex)) { // ((Triforce, 11-21, TEX-43) what went for that alone: the 1st, 2nd, 4th, ...) + st_obsolete_bytes += tex->bytes + tex->pad; + if ((++st_obsolete_n & (st_obsolete_n - 1)) == 0) + logf("vk: finalized textures the GPU drew, their storage rewritten by the game since: %llu let go so far (%llu KB; " + "the last #%u %ux%u NVN %#x; RX_VK_KEEP_OBSOLETE=1: kept, as before)", (unsigned long long)st_obsolete_n, + (unsigned long long)(st_obsolete_bytes >> 10), tex->id, tex->width, tex->height, tex->nvn_format); + } if (by_hold && !overwritten) hold_live += tex->bytes + tex->pad; if (!overwritten) live_chosen++; if (!sweep && !overwritten && press.live && !press.held && come_back_on()) // ((Triforce) for the come-back pause, above) @@ -6413,6 +7230,31 @@ void texture_sweep(uint64_t completed, bool sweep, uint64_t budget, uint64_t evi for (uint64_t k : went_keys) come_back.went(k, frame_no); come_back_noted.store(true, std::memory_order_relaxed); } + // ((Triforce, 11-22, the performance audit's TEX-R404, after Source A's) a write-back is left out when another image the + // GPU drew overlaps the texture's storage and was drawn in a later frame than the texture's own last drawing: the game + // draws its HUD labels inside the storage of a 1920 x 1080 target it no longer uses, drawn_over sees only an image of + // the same start, and the target's old texels landed over the labels, which were then uploaded from them as noise. The + // texture goes all the same, and its bookkeeping below is as it was. evict_reason.h has the rule. + // RX_VK_EVICT_STALE_FLUSH=1: written back, as before) + static const bool stale_flush = evict_reason::flush_stale(getenv("RX_VK_EVICT_STALE_FLUSH")); + static uint64_t st_stale_skipped; + if (!stale_flush) { + std::vector cands; + for (size_t i = 0; i < evict.size(); i++) + if (evict[i].flush && evict[i].tex->guest_size) + cands.push_back({evict[i].tex->gpu, evict[i].tex->gpu + evict[i].tex->guest_size, i, evict[i].tex, 0}); + if (!cands.empty()) drawn_others(cands); + for (const evict_reason::Candidate& c : cands) { + const Texture* tex = evict[c.index].tex; + if (!evict_reason::write_back_stale(tex->last_written, c.newest)) continue; + evict[c.index].flush = false; + if (++st_stale_skipped <= 20) + logf("vk: texture #%u %#llx %ux%u the GPU last drew in frame %llu goes without its write-back: another image was " + "drawn over its storage in frame %llu (%llu so far; RX_VK_EVICT_STALE_FLUSH=1: written back, as before)", + tex->id, (unsigned long long)tex->gpu, tex->width, tex->height, (unsigned long long)tex->last_written, + (unsigned long long)c.newest, (unsigned long long)st_stale_skipped); + } + } { // (the write-backs: batches of up to 64 textures or ~512 MB of staging memory, each one submission and one wait) std::vector batch; uint64_t batch_bytes = 0; @@ -6422,7 +7264,10 @@ void texture_sweep(uint64_t completed, bool sweep, uint64_t budget, uint64_t evi for (const Flush& f : batch) if (f.ok) { static std::atomic flushed{0}; - if (++flushed % 50 == 1) logf("vk: textures the GPU drew, written back into their memory before eviction: %lld so far", (long long)flushed.load()); + if (++flushed % 50 == 1) + logf("vk: textures the GPU drew, written back into their memory before eviction: %lld so far (%llu not: " + "another image drawn over their storage since)", (long long)flushed.load(), + (unsigned long long)st_stale_skipped); } batch.clear(); batch_bytes = 0; @@ -6484,6 +7329,20 @@ void texture_sweep(uint64_t completed, bool sweep, uint64_t budget, uint64_t evi ++it; } } + // ((Triforce, 11-21, TEX-30, after Source A's) a render target's image the GPU is done with goes to the pool now, not at + // the next present's texture_frame: a target made in this present's recording takes it, where it made a new image and + // the pool got this one a present late. Not while a compaction pass is open (nothing is freed or moved then), and the + // pool's own bound is kept by its trim, next present. RX_VK_DEAD_POOL_NOW=0: at the next present, as before) + static const bool dead_pool_now = !(getenv("RX_VK_DEAD_POOL_NOW") && !atoi(getenv("RX_VK_DEAD_POOL_NOW"))); + if (dead_pool_now && !compact.in_pass) + for (auto it = dead.begin(); it != dead.end();) { + if (it->pool_key && it->value <= completed) { + pool_put(*it); + it = dead.erase(it); + } else { + ++it; + } + } if (n_dead && frame_no >= dead_logged_at + 600) { // (the dead transient targets let go so far, every 600 presents) dead_logged_at = frame_no; logf("vk: render targets drawn over from another start, let go: %llu so far (%llu MB)", (unsigned long long)n_dead, @@ -6544,7 +7403,13 @@ void texture_log_guest(const Texture* t) { } void texture_released(const nvn_tex* t) { + texture_released_at(t, memory_generation()); // ((Triforce, 11-22, MEM-85) the generation now, as before) +} +/// ((Triforce, 11-22, the performance audit's MEM-85) gen: the write generation the release counts from (release_gen): the +/// generation now, or with RX_VK_RELEASE_QUEUE the one of the game's call, the release being applied later (release_queue.h) +void texture_released_at(const nvn_tex* t, WriteGen gen) { if (!t) return; + if (astc_reuse_mode() != astc_reuse::kOff) reuse_released(t); // ((Triforce, 11-21, TEX-14) its decoded ASTC may serve again) auto [a, b] = by_gpu.equal_range(t->gpu); for (auto it = a; it != b; ++it) it->second->let_go = std::max(frame_no, 1); // (texture_sweep: first to go while VRAM is short) // (Triforce, a lab switch, off: RX_VK_RELEASE_OWN=1 finalizes only the image the released texture has, NX2navi's rule @@ -6579,7 +7444,8 @@ void texture_released(const nvn_tex* t) { } continue; } - it->second->released = true, it->second->release_gen = memory_generation(); + it->second->released = true, it->second->release_gen = gen; + it->second->obsolete = false, it->second->obsolete_checked = 0; // ((Triforce, 11-21, TEX-43) looked at afresh) watch_tex(it->second, "finalized"); static int logged; // (a big one: the course maps' kind) if (it->second->bytes >= (32ull << 20) && logged++ < 40) @@ -6694,4 +7560,25 @@ void texture_read_texels(const nvn_tex* t, unsigned level, unsigned layer0, cons } } +/// ((Triforce) RX_VK_TEX_PIPES_AHEAD) The texture converters' pipelines made now, before the game, on the caller's thread (their +/// ready functions are not thread-safe: nothing uploads while this runs). The ASTC decoder and the BC encoder only where the GPU +/// does not sample ASTC: elsewhere no upload asks for them. Logged before it starts: a driver that crashes in here is named +void texture_pipelines_warm() { + if (!tex_pipes_ahead()) return; + static bool done = false; + if (done) return; + done = true; + VkFormatProperties fp{}; + vkGetPhysicalDeviceFormatProperties(dev.physical, VK_FORMAT_ASTC_4x4_UNORM_BLOCK, &fp); + const bool astc_native = (fp.optimalTilingFeatures & VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT) != 0; + logf("vk: the texture converters' pipelines made before the game (RX_VK_TEX_PIPES_AHEAD=0: at their first upload)"); + const auto t0 = std::chrono::steady_clock::now(); + const bool astc = !astc_native && astc_ready(), bc = !astc_native && bc_ready(); + const bool unswizzle = unswizzle_ready(), back = swizzle_back_ready(); + logf("vk: texture converters ready in %.1f ms: ASTC decoder %s, BC encoder %s, unswizzle %s, swizzle back %s", + std::chrono::duration(std::chrono::steady_clock::now() - t0).count(), + astc_native ? "not needed" : astc ? "yes" : "no", astc_native ? "not needed" : bc ? "yes" : "no", unswizzle ? "yes" : "no", + back ? "yes" : "no"); +} + } // namespace mkwvk diff --git a/gpu/vk/vk_texture.h b/gpu/vk/vk_texture.h index 39d612c..9adc7da 100644 --- a/gpu/vk/vk_texture.h +++ b/gpu/vk/vk_texture.h @@ -18,6 +18,7 @@ #include "vk_internal.h" #include "upload_owed.h" #include "upload_same.h" +#include "astc_reuse.h" // ((Triforce, 11-21, TEX-14) decoded ASTC used again: the blocks' hash a texture keeps namespace mkwvk { @@ -59,6 +60,7 @@ struct Texture { int nvn_format = 0; // the format it was made for // contents bool uploaded = false; // from guest memory, as of upload_gen + bool warmed = false; // ((Triforce, 11-22, TEX-23) made ahead by the list's look, not drawn yet: its first draw learns it) WriteGen upload_gen; // (Triforce: a page write generation, 64 bits: write_gen.h) uint64_t bc_retry_frame = 0; // (hookshot) a BC3 image whose upload failed: tried again from this frame (vk_texture.cpp) upload_owed::Owed owed; // (Triforce) an upload that found no memory: what it was to bring, and when it is tried again @@ -72,11 +74,19 @@ struct Texture { bool only_cleared = false; // (hookshot) all the GPU did was clear it: what the CPU writes after still counts (a font cache's glyphs) WriteGen gpu_gen; // (the memory generation when the GPU last wrote it: CPU writes after it are taken) bool released = false; // its texture was finalized after the GPU wrote it: CPU writes count again + // ((Triforce, 11-21, TEX-14) astc_reuse.h: its ASTC blocks' hash at its first whole upload (zero: not hashed), and + // whether it is listed among the finalized ones whose image may serve the same blocks made again elsewhere) + astc_reuse::Hash blocks_hash; + bool freed_listed = false; uint64_t let_go = 0; // (Triforce) the present at which the game finalized a texture over it, 0 once it is used again: // one uploaded from memory goes first while VRAM is short (texture_sweep) uint32_t back_gap = 0; // ((Triforce) and the longest it stayed let go before it was used again, in presents: it goes // early no sooner than 8 times that after a let-go (tex_pressure.h, keep taken) WriteGen release_gen; + // ((Triforce, 11-21, TEX-43) finalized_obsolete.h: the CPU has rewritten its storage since the release, and the present + // that was last looked at (0: not yet); both start again at a release) + mutable bool obsolete = false; + mutable uint64_t obsolete_checked = 0; uint64_t gpu_wrote_seen = 0; // (the newest finished GPU write into its storage it was uploaded after) uint64_t last_written = 0; // (the frame the GPU last wrote it: a render target, each frame it is used) uint64_t prev_written = 0; // ((Triforce) the frame it was written in before last_written's, 0: none: redrawn.h) @@ -227,13 +237,19 @@ void texture_frame(uint64_t completed_value); bool texture_warm_on(); // ((Triforce) RX_VK_TEX_WARM: tex_warm.h) void texture_warm_note(int slot); void texture_warm_step(int64_t last_frame_work); +/// ((Triforce, 11-22, TEX-23) RX_VK_TEX_WARM_LIST: when the game made a BC or ASTC texture's storage, and its release; any thread) +void texture_warm_born(const nvn_tex* t); +void texture_warm_gone(const nvn_tex* t); void texture_released(const nvn_tex* t); +void texture_released_at(const nvn_tex* t, WriteGen gen); // ((Triforce, 11-22, MEM-85) from that generation: release_queue.h) /// (diagnostics: what guest memory holds of a texture's storage now, level by level, logged) void texture_log_guest(const Texture* t); /// (diagnostics, the replay's lock held) the textures the GPU wrote in [gpu, gpu + size), appended to out void texture_describe_written(uint64_t gpu, uint64_t size, std::string& out); /// (a host hook's fast check, any thread) whether a texture the GPU wrote may lie in virtual pages [gpu, gpu + size) bool texture_gpu_written_in(uint64_t gpu, uint64_t size); +/// ((Triforce, THREAD-25) page_marks.h, no lock) false: no such texture was ever listed in the range: the call above is false +bool texture_gpu_written_marked(uint64_t gpu, uint64_t size); /// Textures the GPU wrote in [gpu, gpu + size) read back and written into their memory, block linear, as the GPU /// would have left them there: before those virtual pages are unmapped or remapped (the work recorded before is /// submitted; the replay's lock held). How many were. diff --git a/gpu/vk/vk_ui.h b/gpu/vk/vk_ui.h index 178aa4d..239b45d 100644 --- a/gpu/vk/vk_ui.h +++ b/gpu/vk/vk_ui.h @@ -14,6 +14,8 @@ #include #include #include +#include +#include #include "imgui.h" #include "vk_imgui.h" @@ -115,6 +117,43 @@ inline void text_mid_centre(Ctx& c, ImFont* font, float px, float cx, float y, I inline void text_mid_right(Ctx& c, ImFont* font, float px, float x, float y, ImU32 col, const char* t) { text_mid(c, font, px, x - size_of(font, px, t).x, y, col, t); } +/// A line cut to w: as it is, or its longest whole characters with "..." that fit (a utf-8 sequence is never +/// split). out is what to draw; the width drawn (0: nothing fits) +inline float fit_text(const char* t, ImFont* font, float px, float w, std::string* out) { + out->clear(); + if (!t || !*t || w <= 0) return 0; + const float full = size_of(font, px, t).x; + if (full <= w) { *out = t; return full; } + const size_t n = strlen(t); + std::vector starts; // (every character's first byte: a cut never falls inside a utf-8 sequence) + for (size_t i = 0; i < n; i++) + if (((unsigned char)t[i] & 0xC0) != 0x80) starts.push_back(i); + // (the longest prefix that with an ellipsis fits: measured at the boundaries' halves, not one by one, since this + // runs for every line drawn every frame) + size_t lo = 0, hi = starts.size(); + while (lo < hi) { + const size_t mid = (lo + hi + 1) / 2; + *out = std::string(t, starts[mid - 1]) + "\xE2\x80\xA6"; + if (size_of(font, px, out->c_str()).x <= w) lo = mid; else hi = mid - 1; + } + if (!lo) { out->clear(); return 0; } + *out = std::string(t, starts[lo - 1]) + "\xE2\x80\xA6"; + return size_of(font, px, out->c_str()).x; +} +/// text, its one line fitted to w (an ellipsis when it does not fit) +inline float text_fit(Ctx& c, ImFont* font, float px, ImVec2 p, ImU32 col, const char* t, float w) { + std::string s; + fit_text(t, font, px, w, &s); + if (s.empty()) return 0; + return text(c, font, px, p, col, s.c_str()); +} +/// text_mid, its one line fitted to w +inline float text_mid_fit(Ctx& c, ImFont* font, float px, float x, float y, ImU32 col, const char* t, float w) { + std::string s; + fit_text(t, font, px, w, &s); + if (s.empty()) return 0; + return text_mid(c, font, px, x, y, col, s.c_str()); +} inline void text_centre(Ctx& c, ImFont* font, float px, ImVec2 p, ImU32 col, const char* t) { text(c, font, px, ImVec2(p.x - size_of(font, px, t).x * 0.5f, p.y), col, t); } diff --git a/gpu/vk/vk_vendor.h b/gpu/vk/vk_vendor.h index 642f887..1001dff 100644 --- a/gpu/vk/vk_vendor.h +++ b/gpu/vk/vk_vendor.h @@ -18,6 +18,9 @@ struct VendorFacts { VkDriverId driver_id{}; VkPhysicalDeviceType device_type = VK_PHYSICAL_DEVICE_TYPE_OTHER; const char* driver_info = ""; // (VkPhysicalDeviceDriverProperties::driverInfo) + // ((Triforce, 11-22, MEM-23) VkPhysicalDeviceVulkan11Features::storageInputOutput16. True until the device says otherwise, + // so facts made without it name no GCN card: radv_gcn) + bool storage_io16 = true; }; struct VendorFixes { @@ -47,6 +50,14 @@ inline bool amd_older_compiler(const VendorFacts& f) { return f.driver_id == VK_DRIVER_ID_AMD_PROPRIETARY && f.driver_info && std::strstr(f.driver_info, "proprietary shader compiler"); } +/// ((Triforce, 11-22, the performance audit's MEM-23 and PIPE-56, after Source A's) AMD's GCN 1 to 4 GPUs under Mesa's RADV +/// (GFX6 to GFX8: the HD 7000 cards to the RX 400 and 500 ones), told by the one thing RADV reports differently there: no +/// 16-bit storage input and output, which Vega and every later GPU has. Never true under another driver (AMD's own +/// Windows driver on the same cards is not RADV), nor for NVIDIA or Intel. It names the cards; it changes nothing by itself +inline bool radv_gcn(const VendorFacts& f) { + return f.driver_id == VK_DRIVER_ID_MESA_RADV && !f.storage_io16; +} + inline VendorFixes vendor_fixes(const VendorFacts& f) { VendorFixes x; x.fetch_bounds = f.vendor_id != 0x10DE; diff --git a/gpu/vk/vk_vma.cpp b/gpu/vk/vk_vma.cpp index 75ec978..ffa1b6e 100644 --- a/gpu/vk/vk_vma.cpp +++ b/gpu/vk/vk_vma.cpp @@ -25,13 +25,31 @@ namespace mkwvk { VmaAllocator vma; uint64_t test_card_ballast = 0; // (RX_VK_TEST_CARD_MB, a lab tool: the VRAM taken so that the card plays a smaller one; vma_init) +// ---- (Triforce, 11-22, the performance audit's TEX-47, after Source A's) VK_EXT_memory_priority ---- +// Off unless RX_VK_MEMORY_PRIORITY is set and the device has the extension (dev.memory_priority, vk_main.cpp): then the +// allocator is made with VMA_ALLOCATOR_CREATE_EXT_MEMORY_PRIORITY_BIT and gives every pool's blocks and every allocation of +// its own the priority its create info names. One that names none gets 0, below the textures, so every VmaPoolCreateInfo and +// VmaAllocationCreateInfo in this file names its tier: the render targets' pools 1.0, the textures' pool 0.25, the other +// images, the buffers and the mirrors' pieces 0.5, the lab's ballast 1.0 so that RX_VK_TEST_CARD_MB still holds the card +// (memory_priority.h; tests/test_memory_priority.py reads this file for it). Without the bit VMA ignores the field, and +// prio() is 0.5 for everything. +const memory_priority::Tiers& memory_tiers() { + static const memory_priority::Tiers t = memory_priority::active(getenv("RX_VK_MEMORY_PRIORITY"), dev.memory_priority); + return t; +} +namespace { +using PrioKind = memory_priority::Kind; +float prio(PrioKind k) { return memory_priority::of(memory_tiers(), k); } +} // namespace + bool vma_init(PFN_vkGetInstanceProcAddr gipa) { VmaVulkanFunctions f{}; f.vkGetInstanceProcAddr = gipa; f.vkGetDeviceProcAddr = vkGetDeviceProcAddr; VmaAllocatorCreateInfo ci{}; ci.flags = VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT | - (dev.memory_budget ? VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT : 0); + (dev.memory_budget ? VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT : 0) | + (dev.memory_priority ? VMA_ALLOCATOR_CREATE_EXT_MEMORY_PRIORITY_BIT : 0); // ((Triforce, 11-22, TEX-47) above) ci.physicalDevice = dev.physical; ci.device = dev.device; ci.instance = dev.instance; @@ -39,6 +57,9 @@ bool vma_init(PFN_vkGetInstanceProcAddr gipa) { ci.pVulkanFunctions = &f; ci.preferredLargeHeapBlockSize = 64ull << 20; // (64 MB blocks: Eglantine found 256 MB ones wasted VRAM) if (vmaCreateAllocator(&ci, &vma) != VK_SUCCESS) return false; + if (dev.memory_priority) + logf("vk: memory priorities on (RX_VK_MEMORY_PRIORITY, VK_EXT_memory_priority): render targets %.2f, sampled textures " + "%.2f, the rest %.2f", (double)prio(PrioKind::Target), (double)prio(PrioKind::Texture), (double)prio(PrioKind::Other)); // (Triforce, a lab tool) RX_VK_TEST_BALLAST_MB=: that much VRAM taken and held for the session, in 256 MB device buffers // of their own. RX_VK_TEST_ALLOWED_MB only changes the number the budget is told; this fills the card for real, as a card // with less VRAM or another program's use does: the driver's own behaviour at its limit (allocations of 110-170 ms in a @@ -67,6 +88,7 @@ bool vma_init(PFN_vkGetInstanceProcAddr gipa) { ai.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE; ai.requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; ai.flags = VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT; + ai.priority = prio(PrioKind::Held); // ((Triforce, 11-22, TEX-47) the ballast stays on the card) VkBuffer b = VK_NULL_HANDLE; VmaAllocation a = VK_NULL_HANDLE; if (vmaCreateBuffer(vma, &bi, &ai, &b, &a, nullptr) != VK_SUCCESS) break; // (kept: never freed) @@ -128,6 +150,7 @@ VmaPool small_targets_pool() { small_target_pool_tried = true; if (!target_images_pool()) return VK_NULL_HANDLE; // (the targets' pool's memory type: target_pool_type) VmaPoolCreateInfo pi{}; + pi.priority = prio(PrioKind::Target); // ((Triforce, 11-22, TEX-47)) pi.memoryTypeIndex = target_pool_type; pi.blockSize = (getenv("RX_VK_SMALL_TARGET_MB") ? (uint64_t)std::max(4, atoi(getenv("RX_VK_SMALL_TARGET_MB"))) : 16u) << 20; // (and a floor of blocks never given back, RX_VK_SMALL_TARGET_KEEP (8 = 128 MB): a tour on an 8 GB card played here lost @@ -159,11 +182,13 @@ VmaPool target_images_pool() { rep.samples = VK_SAMPLE_COUNT_1_BIT; rep.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; VmaAllocationCreateInfo ai{}; + ai.priority = prio(PrioKind::Target); // ((Triforce, 11-22, TEX-47)) ai.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE; ai.requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; uint32_t type = 0; if (vmaFindMemoryTypeIndexForImageInfo(vma, &rep, &ai, &type) != VK_SUCCESS) return VK_NULL_HANDLE; VmaPoolCreateInfo pi{}; + pi.priority = prio(PrioKind::Target); // ((Triforce, 11-22, TEX-47) the render targets: kept in video memory first) pi.memoryTypeIndex = type; target_pool_type = type; pi.blockSize = target_block(); @@ -197,11 +222,13 @@ VmaPool images_pool() { rep.samples = VK_SAMPLE_COUNT_1_BIT; rep.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; VmaAllocationCreateInfo ai{}; + ai.priority = prio(PrioKind::Other); // ((Triforce, 11-22, TEX-47)) ai.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE; ai.requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; uint32_t type = 0; if (vmaFindMemoryTypeIndexForImageInfo(vma, &rep, &ai, &type) != VK_SUCCESS) return VK_NULL_HANDLE; VmaPoolCreateInfo pi{}; + pi.priority = prio(PrioKind::Other); // ((Triforce, 11-22, TEX-47) the other images: the driver's own 0.5) pi.memoryTypeIndex = type; pi.blockSize = kImageBlock; if (vmaCreatePool(vma, &pi, &image_pool) != VK_SUCCESS) image_pool = VK_NULL_HANDLE; @@ -227,6 +254,7 @@ VmaPool textures_pool() { rep.samples = VK_SAMPLE_COUNT_1_BIT; rep.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; VmaAllocationCreateInfo ai{}; + ai.priority = prio(PrioKind::Texture); // ((Triforce, 11-22, TEX-47)) ai.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE; ai.requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; uint32_t type = 0; @@ -235,6 +263,7 @@ VmaPool textures_pool() { return VK_NULL_HANDLE; } VmaPoolCreateInfo pi{}; + pi.priority = prio(PrioKind::Texture); // ((Triforce, 11-22, TEX-47) the sampled textures: moved out first) pi.memoryTypeIndex = type; pi.blockSize = kImageBlock; if (vmaCreatePool(vma, &pi, &texture_pool) != VK_SUCCESS) texture_pool = VK_NULL_HANDLE; @@ -418,6 +447,7 @@ void ahead_main() { VkMemoryRequirements req{}; req.size = a.block, req.alignment = 4096, req.memoryTypeBits = ~0u; VmaAllocationCreateInfo ai{}; + ai.priority = prio(PrioKind::Other); // ((Triforce, 11-22, TEX-47) in a pool: its blocks have the pool's) ai.pool = job.first; if (dev.memory_budget) ai.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT; VmaAllocation r = VK_NULL_HANDLE; @@ -486,6 +516,9 @@ bool create_image_inner(const VkImageCreateInfo& ci, VkImage& image, VmaAllocati } bool create_image_inner_(const VkImageCreateInfo& ci, VkImage& image, VmaAllocation& allocation, bool dedicated, bool target, bool textures) { VmaAllocationCreateInfo ai{}; + // ((Triforce, 11-22, TEX-47) for an image that gets memory of its own; in a pool the pool's priority counts. The copies + // of this info below keep it) + ai.priority = prio(target ? PrioKind::Target : textures ? PrioKind::Texture : PrioKind::Other); ai.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE; // (in VRAM or not at all: an image VMA puts in system memory when VRAM is full is read over PCIe every frame, // and takes the RAM the game and everything else on the machine need) @@ -529,6 +562,7 @@ bool create_image_inner_(const VkImageCreateInfo& ci, VkImage& image, VmaAllocat // to system memory instead, slow but drawn if (getenv("RX_VK_IMAGES_VRAM_ONLY")) return false; VmaAllocationCreateInfo sai{}; + sai.priority = ai.priority; // ((Triforce, 11-22, TEX-47)) sai.usage = VMA_MEMORY_USAGE_AUTO; if (vmaCreateImage(vma, &ci, &sai, &image, &allocation, nullptr) != VK_SUCCESS) return false; static std::atomic told{0}; @@ -549,6 +583,7 @@ bool create_buffer(uint64_t size, VkBufferUsageFlags usage, bool host_visible, V bi.size = size; bi.usage = usage; VmaAllocationCreateInfo ai{}; + ai.priority = prio(PrioKind::Other); // ((Triforce, 11-22, TEX-47)) ai.usage = VMA_MEMORY_USAGE_AUTO; if (host_visible) ai.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT; // (a buffer with VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT gets VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT from VMA: @@ -568,6 +603,7 @@ bool create_buffer_cached(uint64_t size, VkBufferUsageFlags usage, VkBuffer& buf bi.size = size; bi.usage = usage; VmaAllocationCreateInfo ai{}; + ai.priority = prio(PrioKind::Other); // ((Triforce, 11-22, TEX-47)) ai.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST; ai.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT; if (dedicated) ai.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT; // (its own memory object: no part of a 64 MB block left unused) @@ -576,6 +612,31 @@ bool create_buffer_cached(uint64_t size, VkBufferUsageFlags usage, VkBuffer& buf if (mapped) *mapped = info.pMappedData; return true; } +/// ((Triforce, 11-22, the performance audit's MEM-39; lab only, RX_VK_CAP_UNCACHED=1) As create_buffer_cached, asked for as +/// memory written in order: the allocator then avoids cached types, which gives write-combined memory where there is one +bool create_buffer_uncached(uint64_t size, VkBufferUsageFlags usage, VkBuffer& buffer, VmaAllocation& allocation, + void** mapped, bool dedicated) { + VkBufferCreateInfo bi{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; + bi.size = size; + bi.usage = usage; + VmaAllocationCreateInfo ai{}; + ai.priority = prio(PrioKind::Other); // ((Triforce, 11-22, TEX-47) as create_buffer_cached's) + ai.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST; + ai.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT; + if (dedicated) ai.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT; + VmaAllocationInfo info{}; + if (vmaCreateBuffer(vma, &bi, &ai, &buffer, &allocation, &info) != VK_SUCCESS) return false; + if (mapped) *mapped = info.pMappedData; + return true; +} +/// ((Triforce, 11-22, MEM-39) An allocation's memory type and that type's property flags: for the log +uint32_t alloc_memory_type(VmaAllocation allocation, VkMemoryPropertyFlags* flags) { + VmaAllocationInfo info{}; + if (!allocation) return ~0u; + vmaGetAllocationInfo(vma, allocation, &info); + if (flags) vmaGetAllocationMemoryProperties(vma, allocation, flags); + return info.memoryType; +} bool host_cached(VmaAllocation allocation) { VkMemoryPropertyFlags f = 0; if (allocation) vmaGetAllocationMemoryProperties(vma, allocation, &f); @@ -598,6 +659,7 @@ bool create_device_buffer(uint64_t size, VkBufferUsageFlags usage, VkBuffer& buf bi.size = size; bi.usage = usage; VmaAllocationCreateInfo ai{}; + ai.priority = prio(PrioKind::Other); // ((Triforce, 11-22, TEX-47)) ai.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE; ai.requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; if (dev.memory_budget) ai.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT; @@ -611,6 +673,7 @@ bool vma_piece_alloc(uint64_t size, uint64_t alignment, uint32_t type_bits, VmaA VkMemoryRequirements req{}; req.size = size, req.alignment = alignment, req.memoryTypeBits = type_bits; VmaAllocationCreateInfo ai{}; + ai.priority = prio(PrioKind::Other); // ((Triforce, 11-22, TEX-47) the mirrors' pieces: the driver's own 0.5) ai.requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; if (dev.memory_budget) ai.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT; VmaAllocationInfo info{}; @@ -625,6 +688,7 @@ bool vma_piece_alloc(uint64_t size, uint64_t alignment, uint32_t type_bits, VmaA piece_pool_tried = true; if (vmaFindMemoryTypeIndex(vma, type_bits, &ai, &piece_type) == VK_SUCCESS) { VmaPoolCreateInfo pi{}; + pi.priority = prio(PrioKind::Other); // ((Triforce, 11-22, TEX-47)) pi.memoryTypeIndex = piece_type; pi.blockSize = kPieceBlock; if (vmaCreatePool(vma, &pi, &piece_pool) != VK_SUCCESS) piece_pool = VK_NULL_HANDLE; diff --git a/gpu/vk/vk_work.h b/gpu/vk/vk_work.h index eb6d969..f1394a6 100644 --- a/gpu/vk/vk_work.h +++ b/gpu/vk/vk_work.h @@ -13,6 +13,36 @@ #include "vk_loader.h" #include "vk_recorder.h" // (RX_VK_RECORDER: each macro below calls its packer, which calls the driver directly when the recorder is off) #include "cmd_work.h" // ((Triforce) note_work, work_begun, work_submitted) +#include "gpu_wait_kinds.h" // ((Triforce, 11-22, SYNC-27) the host's time in the calls below that block on the GPU, by kind + +// ((Triforce, 11-22, the performance audit's SYNC-27, after Source A's) The five calls that block on the GPU, each timed +// under its kind when it blocks (gpu_wait_kinds.h: a fence or a semaphore wait with a timeout above 0, a query read with the +// wait bit, an idle always). Their macros below call these after the recorder's boundary with the driver's call first, then +// its arguments, each evaluated once as before. RX_VK_GPU_WAITS=0: timed() is the call alone +namespace mkwvk::gpu_waits_vk { +template +inline VkResult queue_idle(Call&& call, VkQueue queue) { + return ::gpu_wait_kinds::timed(::gpu_wait_kinds::IDLE, true, [&] { return call(queue); }); +} +template +inline VkResult device_idle(Call&& call, VkDevice device) { + return ::gpu_wait_kinds::timed(::gpu_wait_kinds::IDLE, true, [&] { return call(device); }); +} +template +inline VkResult fences(Call&& call, VkDevice device, uint32_t count, const VkFence* list, VkBool32 all, uint64_t timeout) { + return ::gpu_wait_kinds::timed(::gpu_wait_kinds::FENCE, timeout != 0, [&] { return call(device, count, list, all, timeout); }); +} +template +inline VkResult semaphores(Call&& call, VkDevice device, const VkSemaphoreWaitInfo* info, uint64_t timeout) { + return ::gpu_wait_kinds::timed(::gpu_wait_kinds::TIMELINE, timeout != 0, [&] { return call(device, info, timeout); }); +} +template +inline VkResult query_results(Call&& call, VkDevice device, VkQueryPool pool, uint32_t first, uint32_t count, size_t size, void* data, + VkDeviceSize stride, VkQueryResultFlags flags) { + return ::gpu_wait_kinds::timed(::gpu_wait_kinds::QUERY, (flags & VK_QUERY_RESULT_WAIT_BIT) != 0, + [&] { return call(device, pool, first, count, size, data, stride, flags); }); +} +} // namespace mkwvk::gpu_waits_vk // ((Triforce) the command buffer, the first argument, counted as it is passed on: work_on returns it. __VA_OPT__: a call with no // argument after it, vkCmdEndRendering's) @@ -100,16 +130,16 @@ #define vkResetCommandPool(...) (::mkwvk::rec::boundary(::mkwvk::rec::B_vkResetCommandPool), ::vkx::vkResetCommandPool(__VA_ARGS__)) #define vkDestroyCommandPool(...) (::mkwvk::rec::boundary(::mkwvk::rec::B_vkDestroyCommandPool), ::vkx::vkDestroyCommandPool(__VA_ARGS__)) #define vkQueueSubmit2(...) (::mkwvk::rec::boundary(::mkwvk::rec::B_vkQueueSubmit2), ::vkx::vkQueueSubmit2(RX_WORK_SUBMIT(__VA_ARGS__))) -#define vkQueueWaitIdle(...) (::mkwvk::rec::boundary(::mkwvk::rec::B_vkQueueWaitIdle), ::vkx::vkQueueWaitIdle(__VA_ARGS__)) +#define vkQueueWaitIdle(...) (::mkwvk::rec::boundary(::mkwvk::rec::B_vkQueueWaitIdle), ::mkwvk::gpu_waits_vk::queue_idle(::vkx::vkQueueWaitIdle, __VA_ARGS__)) #define vkQueueBindSparse(...) (::mkwvk::rec::boundary(::mkwvk::rec::B_vkQueueBindSparse), ::vkx::vkQueueBindSparse(__VA_ARGS__)) #define vkQueuePresentKHR(...) (::mkwvk::rec::boundary(::mkwvk::rec::B_vkQueuePresentKHR), ::vkx::vkQueuePresentKHR(__VA_ARGS__)) #define vkAcquireNextImageKHR(...) (::mkwvk::rec::boundary(::mkwvk::rec::B_vkAcquireNextImageKHR), ::vkx::vkAcquireNextImageKHR(__VA_ARGS__)) -#define vkDeviceWaitIdle(...) (::mkwvk::rec::boundary(::mkwvk::rec::B_vkDeviceWaitIdle), ::vkx::vkDeviceWaitIdle(__VA_ARGS__)) -#define vkWaitForFences(...) (::mkwvk::rec::boundary(::mkwvk::rec::B_vkWaitForFences), ::vkx::vkWaitForFences(__VA_ARGS__)) +#define vkDeviceWaitIdle(...) (::mkwvk::rec::boundary(::mkwvk::rec::B_vkDeviceWaitIdle), ::mkwvk::gpu_waits_vk::device_idle(::vkx::vkDeviceWaitIdle, __VA_ARGS__)) +#define vkWaitForFences(...) (::mkwvk::rec::boundary(::mkwvk::rec::B_vkWaitForFences), ::mkwvk::gpu_waits_vk::fences(::vkx::vkWaitForFences, __VA_ARGS__)) #define vkResetFences(...) (::mkwvk::rec::boundary(::mkwvk::rec::B_vkResetFences), ::vkx::vkResetFences(__VA_ARGS__)) -#define vkWaitSemaphores(...) (::mkwvk::rec::boundary(::mkwvk::rec::B_vkWaitSemaphores), ::vkx::vkWaitSemaphores(__VA_ARGS__)) +#define vkWaitSemaphores(...) (::mkwvk::rec::boundary(::mkwvk::rec::B_vkWaitSemaphores), ::mkwvk::gpu_waits_vk::semaphores(::vkx::vkWaitSemaphores, __VA_ARGS__)) #define vkGetSemaphoreCounterValue(...) (::mkwvk::rec::boundary(::mkwvk::rec::B_vkGetSemaphoreCounterValue), ::vkx::vkGetSemaphoreCounterValue(__VA_ARGS__)) -#define vkGetQueryPoolResults(...) (::mkwvk::rec::boundary(::mkwvk::rec::B_vkGetQueryPoolResults), ::vkx::vkGetQueryPoolResults(__VA_ARGS__)) +#define vkGetQueryPoolResults(...) (::mkwvk::rec::boundary(::mkwvk::rec::B_vkGetQueryPoolResults), ::mkwvk::gpu_waits_vk::query_results(::vkx::vkGetQueryPoolResults, __VA_ARGS__)) #define vkResetQueryPool(...) (::mkwvk::rec::boundary(::mkwvk::rec::B_vkResetQueryPool), ::vkx::vkResetQueryPool(__VA_ARGS__)) #define vkGetQueueCheckpointDataNV(...) (::mkwvk::rec::boundary(::mkwvk::rec::B_vkGetQueueCheckpointDataNV), ::vkx::vkGetQueueCheckpointDataNV(__VA_ARGS__)) #define vkGetDeviceFaultInfoEXT(...) (::mkwvk::rec::boundary(::mkwvk::rec::B_vkGetDeviceFaultInfoEXT), ::vkx::vkGetDeviceFaultInfoEXT(__VA_ARGS__)) diff --git a/gpu/vk/vram_compact.h b/gpu/vk/vram_compact.h index 1c10188..7479311 100644 --- a/gpu/vk/vram_compact.h +++ b/gpu/vk/vram_compact.h @@ -25,6 +25,15 @@ inline bool pressed(uint64_t used, uint64_t allowed) { return allowed && used > /// into what a run gave back while the next one waits: under a 9.8 GB allowance the count reached 10.2 GB with 1.1-1.6 GB free inline uint64_t run_from(uint64_t min_free, bool is_pressed) { return is_pressed ? std::min(min_free, 192ull << 20) : min_free; } +/// ((Triforce, 11-22, the performance audit's TEX-38, after Source A's) That free space when RX_VK_COMPACT_MB is not set: a +/// sixteenth of what the driver allows (bytes), 64 MB at least and 512 at most; 512 while the allowance is not known. A card +/// that allows 8 GB or more starts a run from 512 MB, as it did; a 6 GB laptop that allows 5,233 MB from 327, where 512 MB +/// left free in the blocks was a tenth of the card waiting for a run +inline uint64_t default_min_free(uint64_t allowed) { + if (!allowed) return 512ull << 20; + return std::clamp(allowed / 16, 64ull << 20, 512ull << 20); +} + /// The presents between the end of a run and the next look inline uint64_t rest(bool is_pressed) { return is_pressed ? 120 : 600; } @@ -68,6 +77,18 @@ inline uint64_t reserve_pct(uint64_t pct) { return pct > kReservePoints ? pct - /// off; UINT64_MAX: nothing inline uint64_t rule_target(uint64_t total, uint64_t rule_budget) { return total > rule_budget ? rule_budget / 8 * 7 : UINT64_MAX; } +/// ((Triforce) RX_VK_EVICT_EXCESS, on by default) The same with the budget's size kept apart from what the 90% and 97% rules +/// left (rule_budget): 7/8 of the size once the textures hold more than it, as above, but for those two rules their budget less +/// kExcessSlack and no more. Their budget is "the textures less the excess", not a size: 62 MB over the 90% line on an 8 GB card +/// took 7/8 of it, some 400 MB, all on screen again within the camera's turn, made again, and over the line again. The slack +/// spaces the passes; the lower of the two when both apply; UINT64_MAX: nothing +inline constexpr uint64_t kExcessSlack = 64ull << 20; +inline uint64_t rule_target(uint64_t total, uint64_t size_budget, uint64_t rule_budget) { + uint64_t to = total > size_budget ? size_budget / 8 * 7 : UINT64_MAX; + if (rule_budget < size_budget && total > rule_budget) to = std::min(to, rule_budget - std::min(rule_budget, kExcessSlack)); + return to; +} + /// What a live texture unused for that many presents may be let go down to in the hold's pass, the lowest of the lines that /// apply to it; UINT64_MAX: it stays. Three tiers: /// - unused for need_unused (the eviction's own 60 presents): what the other rules ask for (rule_to). Over 90% of the allowance diff --git a/gpu/vk/wait_hold.h b/gpu/vk/wait_hold.h new file mode 100644 index 0000000..3e34bf9 --- /dev/null +++ b/gpu/vk/wait_hold.h @@ -0,0 +1,81 @@ +// (Triforce, 11-22, the performance audit's THREAD-11, after Source B's) The hold of a fence wait that needs stopped work. +// +// A wait of the game's for a fence whose work is stopped at an event's wait gives the other queues a moment to let it on (up to +// RX_VK_WAIT_HOLD_US, 3 ms) before it forces it: vk_replay.cpp wait(). That hold is a loop of looks: flush, the timeline's value, +// the order lock to ask whether the stopped work is still needed, and a yield of the processor (SwitchToThread). The main thread +// comes there about 15 times a second. +// +// Counted now, in the order line: the holds, their looks, the time held, the takes of the order lock, and how each hold ended +// (the fence done, the stopped work let on by another thread, the wait's own time up, or forced on at the hold's bound). +// +// RX_VK_WAIT_HOLD_PAUSE_US=, off (0) unless set, 1000 at most: between two looks a gap of PAUSEs, 16 at a time until that +// long has passed, instead of a yield a look (Source B's gap of 2 us on a like loop left the other half of the core free and +// measured 59.2 to 59.9 fps there, on a loop that runs far more often than this one); and the order lock taken only when it can +// say something new. What the lock is asked (is the oldest unfinished stopped submission at or below this fence) is state that +// changes only inside a section of that lock, and the lock counts the sections that have left. The waiter reads that count +// inside its own section; one more than that is its own section having left; while the count is still that, no other section +// has finished and the answer is the one it has. A section in progress is seen when it leaves, which is when a waiter blocked on +// the lock would have got it. The fence's own value is still read at every look, and the bound and the forced drain stay. +// A signal of the CPU's is the one thing that reaches the order without a section: it is only queued (ev_cpu_any), and acts in +// the next section that looks at the stopped queues. While one is queued the lock is taken at every look, as before, so that a +// look made inside the waiter's own section (the audit's SYNC-59 gives the hold one) is never skipped. +// +// Plain rules: tests/test_wait_hold_pause.py runs them and the loop cut out of vk_replay.cpp. +#pragma once + +#include +#include +#include +#include + +namespace mkwvk::wait_hold { + +constexpr int64_t kMostUs = 1000; +constexpr int kPauses = 16; // (PAUSEs between two looks at the clock) + +/// RX_VK_WAIT_HOLD_PAUSE_US in nanoseconds: 0 (unset, empty, no number, or 0): a yield a look and the lock at each, as before +inline int64_t pause_ns(const char* env) { + if (!env || !*env) return 0; + const long long us = std::atoll(env); + return us <= 0 ? 0 : (us > kMostUs ? kMostUs : us) * 1000; +} + +/// Whether this look takes the order lock. pause: pause_ns's; asked: the waiter has asked under the lock before, and `seen` is the +/// lock's count of sections that had left when it did (read inside its section); left: that count now; signal: a signal of the +/// CPU's is queued and no section has acted on it yet +inline bool takes_lock(int64_t pause, bool asked, uint64_t seen, uint64_t left, bool signal) { + return !pause || !asked || signal || left != seen + 1; +} + +/// The gap between two looks: pause() kPauses at a time until `ns` have passed on now() +template +inline void gap(int64_t ns, Now now, Pause pause) { + const int64_t until = now() + ns; + do { + for (int i = 0; i < kPauses; i++) pause(); + } while (now() < until); +} + +/// How a hold ended +enum End { kMet, kLetOn, kTimedOut, kForced, kEnds }; + +/// What the order line says of the holds (relaxed counts: the waiters are the game's threads, each on its own) +struct Stats { + std::atomic holds{0}, looks{0}, locks{0}, held_ns{0}, ended[kEnds] = {}; + void done(End how, int64_t looked, int64_t locked, int64_t ns) { + holds.fetch_add(1, std::memory_order_relaxed); + looks.fetch_add(looked, std::memory_order_relaxed); + locks.fetch_add(locked, std::memory_order_relaxed); + held_ns.fetch_add(ns, std::memory_order_relaxed); + ended[how].fetch_add(1, std::memory_order_relaxed); + } +}; +inline int describe(const Stats& s, int64_t pause, char* out, size_t cap) { + return snprintf(out, cap, "vk: fence waits held for stopped work: %lld holds, %lld looks, %.1f ms held, the order lock taken %lld " + "times; ended by the fence done %lld, the work let on %lld, the wait's time up %lld, forced on at the bound %lld " + "(RX_VK_WAIT_HOLD_PAUSE_US=%lld)", (long long)s.holds.load(), (long long)s.looks.load(), (double)s.held_ns.load() / 1e6, + (long long)s.locks.load(), (long long)s.ended[kMet].load(), (long long)s.ended[kLetOn].load(), + (long long)s.ended[kTimedOut].load(), (long long)s.ended[kForced].load(), (long long)(pause / 1000)); +} + +} // namespace mkwvk::wait_hold diff --git a/gpu/vk/warm_computes.h b/gpu/vk/warm_computes.h new file mode 100644 index 0000000..fa7e2fa --- /dev/null +++ b/gpu/vk/warm_computes.h @@ -0,0 +1,39 @@ +// (Triforce, the Intel logs of 11-21) Which compute programs have their driver's pipeline built by the shader preparation's +// translation pass (vk_draw.cpp prepare_run, warm_compute), apart from Vulkan so that tests/test_warm_computes.py can run it. +// +// The pass translates every compute program of the seed with its texture variants (up to 64 each) and has the driver compile +// each variant once, for the driver's cache and ours: 9681-9918 compute pipelines. Play dispatches about a tenth of them: +// the precompile after the pass builds only the programs play dispatched (triforce_computes.bin, the player's own list, 166 +// keys and 920 pipelines on an Arc A750; 847-1063 pipelines in the lab's RTX 4070 SUPER runs). On NVIDIA the whole pass is +// 170 s. On the Arc A750 of the 0.6.0 ticket (12 threads) it was 5858 s, 5175 s of it before the first graphics program, +// in the compute programs alone (about 6.4 s of a worker for each compute pipeline): nearly an hour and a half of the four +// hours that preparation took, for pipelines play never asks for. +// +// So on Intel's GPUs the pass builds the pipelines of the listed programs only, once there is a list; the others are still +// translated and kept in the program cache, and one that play does dispatch later is compiled then on a worker (one at a +// time on Intel, RX_VK_COMPUTE_SERIAL), its dispatch left out until it is ready, as for any compute pipeline the caches miss. +// With no list yet (a first start: nothing has been played) every one is built, as before. Other GPUs as before. +// RX_VK_WARM_COMPUTES=listed or all chooses on any GPU. No stamp holds it: the programs are translated as before, and the +// pipeline cache only gains fewer entries. +#pragma once + +#include +#include + +namespace warm_computes { + +inline constexpr uint32_t kIntel = 0x8086; + +/// Whether only the listed compute programs are built. asked: RX_VK_WARM_COMPUTES ("listed" or "all"; null when it is not +/// set, anything else as not set); vendor: the GPU's vendor ID +inline bool listed_only(const char* asked, uint32_t vendor) { + if (asked && !strcmp(asked, "listed")) return true; + if (asked && !strcmp(asked, "all")) return false; + return vendor == kIntel; +} + +/// Whether a compute program's pipeline is built in the pass: always unless listed_only; with it, when the list is empty (no +/// play yet) or the program is in it +inline bool build(bool listed_only, bool list_empty, bool listed) { return !listed_only || list_empty || listed; } + +} // namespace warm_computes diff --git a/gpu/vk/wb_excl_clip.h b/gpu/vk/wb_excl_clip.h new file mode 100644 index 0000000..2acaa53 --- /dev/null +++ b/gpu/vk/wb_excl_clip.h @@ -0,0 +1,53 @@ +// (Triforce, 11-22) Which part of a written binding is copied back to guest memory when the capture leaves memory out +// (vk_memory.cpp gpu_buffer_bind, RX_VK_CAP_ONLY_GPU=1; RX_VK_WB_EXCL_CLIP=0: all of it, as before). +// +// With RX_VK_CAP_ONLY_GPU=1 (0.6.0) the heap memory between the game's pools inside an imported chunk is "left out": the +// collection does not stamp the CPU's writes there, so no mirror sync copies them into the chunk's mirror, and the mirror's +// bytes there stay as they were when the mirror was made (or as a retired mirror left them). A written binding that reaches +// into that memory (a pointer table's bare entry: its first 64 bytes are marked written, vk_draw.cpp) was still written back +// from the mirror at the submission's end. The write-back diff compares the mirror with a shadow filled from guest memory, +// finds the CPU's newer words different, and copies the mirror's stale words over them. Players' 0.6.0 crashes at area +// changes (ModuleSystemWorker / sead::MainThread, guest 0x803b44e0 freeing a block whose header read 0x9ed00000001 or +// 0x1aa88000013fe): the game's heap had put a 12-byte block at exactly 0x18 past such a binding's base (0x117e70a000, +// 0x117f63a000, 0x117d35b000, 0x117e28a000 in four players' logs, each logged as "a buffer binding of +0x40 reaches +// memory the capture leaves out"), and its header at +0x10 was overwritten. +// +// A GPU address below 1 TB is valid only inside a memory pool, so the GPU cannot have written that memory on a console: the +// left-out parts are not copied back (the GPU's writes stay in the mirror, where later GPU work reads them). +#pragma once + +#include +#include +#include +#include + +namespace wb_excl_clip { + +using Range = std::pair; + +/// The parts of [lo, hi) outside every range of excl (sorted by start, not overlapping), appended to out in order. Returns +/// the bytes left out +inline uint64_t kept(uint64_t lo, uint64_t hi, const std::vector& excl, std::vector& out) { + if (hi <= lo) return 0; + // (the first range that ends past lo) + auto it = std::upper_bound(excl.begin(), excl.end(), lo, [](uint64_t v, const Range& r) { return v < r.second; }); + uint64_t at = lo, dropped = 0; + for (; it != excl.end() && it->first < hi; ++it) { + const uint64_t a = std::max(at, it->first), b = std::min(hi, it->second); + if (a > at) out.push_back({at, a}); + if (b > a) dropped += b - a; + at = std::max(at, b); + if (at >= hi) break; + } + if (at < hi) out.push_back({at, hi}); + return dropped; +} + +/// The switch: RX_VK_WB_EXCL_CLIP (nullptr or "": not set, on; "0": off), and only while the capture leaves memory out +/// (cap_only: RX_VK_CAP_ONLY_GPU's mode; with 0 nothing is left out, with 2 the capture still takes everything) +inline bool on(const char* env, int cap_only) { + if (cap_only != 1) return false; + return !(env && env[0] == '0' && !env[1]); +} + +} // namespace wb_excl_clip diff --git a/gpu/vk/wb_lazy_state.h b/gpu/vk/wb_lazy_state.h index ad8b699..4ea783e 100644 --- a/gpu/vk/wb_lazy_state.h +++ b/gpu/vk/wb_lazy_state.h @@ -110,6 +110,7 @@ class Table { return page < kPages && regions[page / kRegionPages].load(std::memory_order_acquire) != 0; } bool any() const { return protected_pages.load(std::memory_order_acquire) != 0; } + std::atomic late_retries{0}; // ((Triforce, 11-21) on_fault: faults on a page whose access was given back meanwhile /// A submission's end, for the whole pages [lo, hi) of a run it is about to write back (seq: this recording). skip gets /// the pages that are OWED: their copy is left out, and the debt is this recording's now. The others are copied as ever; @@ -165,6 +166,21 @@ class Table { return false; } + /// ((Triforce, 11-21) The fault handler's question (lazy_veh), in place of tracked() then fault(). The table gives pages their + /// access back (end_pay, end_watch) BEFORE it uncounts them, so a thread that faulted on such a page just before can get here + /// with tracked() already false and the page readable and writable again: that access runs again (RETRY), where the handler + /// sent it on as not ours and the host's crash report ended the game (a player's report on 0.6.0: about one in 100k frames). + /// A page untracked and still without access is not this table's (start_watch counts before it protects). Tracked: fault() + Fault on_fault(uint64_t page, bool write, uint64_t grow, uint64_t now_ms, std::vector& jobs) { + if (page >= kPages) return NOT_OURS; + if (!tracked(page)) { + if (!mem.accessible(page, write)) return NOT_OURS; + late_retries.fetch_add(1, std::memory_order_relaxed); + return RETRY; + } + return fault(page, write, grow, now_ms, jobs); + } + /// A fault at page. NOT_OURS: nothing of this table's. RETRY: the access can run again (a watch ended by this write, a /// payment by another thread waited for, or the page has its access back by now). PAY: jobs holds what this thread /// has to copy (the owed pages around the address, grow more at most, in PAYING now), then end_pay @@ -393,15 +409,18 @@ class Table { uint64_t b = block_end; while (b < hi && !hot(b, now_ms)) b = std::min(hi, ((b >> kHotShift) + 1) << kHotShift); if (segs.size() >= kMaxSegs) { c.refused_full++; return; } + // (a write fault on a watched page has to find the table: its region counts it, and BEFORE the protection, so + // that no fault can meet a page without access that tracked() does not know yet. 11-21, a player's report: a + // write between the two went to the crash handler) + count(a, b, +1); if (mem.set(a, b, RO)) { Seg s; s.lo = a, s.hi = b, s.src = src, s.state = WATCH, s.collects_at = e.collects_begun, s.caps_at = e.caps_made; segs.emplace(a, s); nsegs.fetch_add(1, std::memory_order_release); - // (a write fault on a watched page has to find the table: its region counts it too) - count(a, b, +1); c.watched += b - a, c.watch_pages += b - a; } else { + count(a, b, -1); c.protect_failed++; } a = b; diff --git a/gpu/vk/wb_rec_wait.h b/gpu/vk/wb_rec_wait.h index 4598ba8..a62f93f 100644 --- a/gpu/vk/wb_rec_wait.h +++ b/gpu/vk/wb_rec_wait.h @@ -81,4 +81,108 @@ struct Finished { } }; +// ---- (Triforce, 11-22, the performance audit's SYNC-122, after Source B's) The reads' side of the same held timeline ---- +// A CPU readback (vk_replay.cpp invalidate) waits for its own write-back's value. Where the recording that carries it is known +// it waits on the recordings' timeline (wait_recorded) and a stopped queue does not hold it. Where it is not known yet (the +// writer still being recorded, or stopped at an event wait) it fell back to the submissions' timeline with no bound and no +// second look: recorded and done a moment later, the read still waited until every older stopped submission had gone on. The +// reads and bindings of pages put off (vk_memory.cpp late_unresolved_read, late_deferred_bound) wait in vk_timeline_wait, whose +// held part was counted for all its callers together. +// +// RX_VK_WB_WAIT_REC is a mode now: 1 as it was (the waits end at the writer's recording), 2 the check: nothing ends earlier, +// and the readback's fallback counts what 1 would not have waited. Unset, 0 or any other text: off, the fallback is the one +// unbounded wait it was. Under 1 and 2 the fallback waits in slices of 1 ms and looks the recording up between them. +// RX_VK_TL_REC=1 (and the lab knob tlrec) ended a capture's wait at value_rec's recording, a submission's FIRST part unless +// RX_VK_REC_LAST: before the burst's write-backs are recorded. It now ends the wait as RX_VK_WB_WAIT_REC=1 does, at the +// recording the submission was finished in (Finished); RX_VK_TL_REC=2 keeps the old lookup. + +/// RX_VK_WB_WAIT_REC's modes +enum Mode { kOff = 0, kOn = 1, kCheck = 2 }; +/// Its text: "1" on, "2" the check; unset, empty, "0" and anything else (a word, another number, a number with a tail): off +inline int mode(const char* text) { + if (!text || !text[0] || text[1]) return kOff; + return text[0] == '1' ? kOn : text[0] == '2' ? kCheck : kOff; +} +/// RX_VK_TL_REC's text names the old lookup (value_rec: a first part): the number 2 +inline bool tl_rec_first_part(const char* text) { return text && std::atoi(text) == 2; } +/// Whether a capture's wait ends at the recording its submission was finished in: RX_VK_WB_WAIT_REC=1, or RX_VK_TL_REC (its +/// switch or its knob: tl_rec) unless that asks for the old lookup, which has its own wait before this one +inline bool ends_by_switch(int wb_mode, bool tl_rec, bool tl_first_part) { + return wb_mode == kOn || (tl_rec && !tl_first_part); +} + +/// The waits' callers, for the held part by caller: a capture's take, a read of a page put off, a binding of one, a readback +enum Caller { kTake = 0, kRead = 1, kBind = 2, kReadback = 3, kCallers = 4 }; +inline const char* caller_name(int who) { + static const char* const names[kCallers] = {"takes", "reads", "binds", "readbacks"}; + return who >= 0 && who < kCallers ? names[who] : "?"; +} + +/// A slice of the readback's fallback wait (ns) +constexpr uint64_t kSliceNs = 1000000ull; +/// Whether the fallback waits in slices and looks the recording up between them: the mode on or the check, and a recordings' +/// timeline to look on. Otherwise the one unbounded wait on the submissions' timeline, as before +inline bool fallback_sliced(int wb_mode, bool has_rec_timeline) { return wb_mode != kOff && has_rec_timeline; } + +/// What a fallback wait did +struct FallbackResult { + bool sliced = false; // (it waited in slices) + bool known = false; // (the writer's recording became known during the wait) + bool at_recording = false; // (it ended with the recording's wait: mode 1) + bool error = false; // (a wait gave something other than done or a timeout: a lost device; the wait ends) + int64_t rec_ns = -1; // (when the recording was seen done, from the wait's start; -1: never seen) + int64_t total_ns = 0; + int64_t slices = 0; +}; +/// The fallback wait over the calls it makes, so that a test can run it against a made-up GPU. d has: +/// void plain() the old wait: the held submits flushed, then the submissions' timeline, no bound +/// void flush() the held submits flushed (nothing when none is held) +/// uint64_t lookup() the recording the value was finished in, 0 while not known +/// int rec_wait(seq) the recordings' timeline waited for with no bound: 0 done, 2 an error +/// bool rec_done(seq) the recordings' timeline has passed seq (a poll) +/// int slice() the submissions' timeline waited for kSliceNs: 0 reached, 1 not yet, 2 an error +/// int64_t now() ns since the wait's start +/// Mode 1 ends at the recording once it is known (no overall bound: the read needs its bytes). Mode 2 ends where the old wait +/// ends, when the submissions' timeline reaches the value, and only notes when the recording was done. Mode 0: plain() +template +FallbackResult fallback_wait(int wb_mode, bool has_rec_timeline, D& d) { + FallbackResult r; + if (!fallback_sliced(wb_mode, has_rec_timeline)) { + d.plain(); + r.total_ns = d.now(); + return r; + } + r.sliced = true; + uint64_t seq = 0; + for (;;) { + d.flush(); // (a batched submit holds both timelines' signals; the writer may have been recorded and held since) + if (!seq) seq = d.lookup(); + if (seq) { + r.known = true; + if (wb_mode == kOn) { + r.error = d.rec_wait(seq) != 0; + r.at_recording = !r.error; + r.rec_ns = d.now(); + break; + } + if (r.rec_ns < 0 && d.rec_done(seq)) r.rec_ns = d.now(); + } + const int s = d.slice(); + r.slices++; + if (s == 1) continue; + r.error = s != 0; + // (the timeline cannot reach the value before the recording that finished it is done: seen now if not before) + if (!r.error && seq && r.rec_ns < 0 && d.rec_done(seq)) r.rec_ns = d.now(); + break; + } + r.total_ns = d.now(); + return r; +} +/// The part of a checked fallback wait (mode 2) after its writer's recording was seen done: what mode 1 would not have waited. +/// To a slice's grain: the recording is looked at between slices +inline int64_t fallback_held_ns(const FallbackResult& r) { + if (!r.sliced || r.at_recording || r.error || r.rec_ns < 0) return 0; + return held_ns(true, true, r.rec_ns, r.total_ns); +} + } // namespace mkwvk::wb_rec_wait diff --git a/gpu/vk/wrote_regions.h b/gpu/vk/wrote_regions.h new file mode 100644 index 0000000..f9e84b6 --- /dev/null +++ b/gpu/vk/wrote_regions.h @@ -0,0 +1,50 @@ +// (Triforce, 11-21, the performance audit's TEX-02, after Source A's) The 2 MB regions of GPU addresses a GPU buffer write +// ever reached (vk_texture.cpp texture_gpu_wrote, gpu_wrote_into). +// +// gpu_wrote_into asks, for every sampled and uploaded texture once an epoch, whether the GPU wrote into the texture's +// storage since its upload: it took gpu_wrote_mu and walked the map of written ranges each time, under the replay's ParLock +// where the quick touch failed, and nearly every answer is no. A texture whose storage lies in no region a GPU write ever +// reached was not written: that answer needs neither the lock nor the walk. One bit a 2 MB region of a 40-bit address +// space, 64 KB of atomics; a bit is never cleared, and addresses past 40 bits share the bits of those below. So a set bit +// only means "ask the map", whose answers are unchanged, and a clear one means no. A write's regions are marked before the +// map has it, and the map says yes only once the write's submission is complete, long after. +// RX_VK_WROTE_REGIONS=0: the lock and the walk each time, as before; 2: a check (a "no" asks the map too, disagreements +// counted and logged: there must be none). +// +// The bits alone, no Vulkan: tests/test_wrote_regions.py compiles them. +#pragma once +#include +#include +#include + +namespace wrote_regions { + +inline constexpr unsigned kShift = 21; // (2 MB) +inline constexpr uint64_t kRegions = 1ull << 19; // (of a 40-bit address space: 64 KB of bits) +inline std::atomic bits[kRegions / 64]; + +/// How many regions [lo, hi) touches, kRegions at the most (a longer range wraps onto every bit) +inline uint64_t span(uint64_t lo, uint64_t hi) { return std::min(((hi - 1) >> kShift) - (lo >> kShift) + 1, kRegions); } + +/// A GPU write of [lo, hi): its regions noted. Called before the write is in the map +inline void mark(uint64_t lo, uint64_t hi) { + if (hi <= lo) return; + const uint64_t a = lo >> kShift, n = span(lo, hi); + for (uint64_t i = 0; i < n; i++) { + const uint64_t r = (a + i) % kRegions; + bits[r >> 6].fetch_or(1ull << (r & 63), std::memory_order_relaxed); + } +} + +/// Whether any region of [lo, hi) was ever written +inline bool any(uint64_t lo, uint64_t hi) { + if (hi <= lo) return false; + const uint64_t a = lo >> kShift, n = span(lo, hi); + for (uint64_t i = 0; i < n; i++) { + const uint64_t r = (a + i) % kRegions; + if (bits[r >> 6].load(std::memory_order_relaxed) & (1ull << (r & 63))) return true; + } + return false; +} + +} // namespace wrote_regions diff --git a/host/achievements.c b/host/achievements.c new file mode 100644 index 0000000..a7ff58a --- /dev/null +++ b/host/achievements.c @@ -0,0 +1,176 @@ +/* (Triforce) Achievements, as a console's or a store's: things done in the game that are marked as earned, once, with a + notice on the screen and a sound when one is, and a page of the options that lists them. host/achievements.h has what + an achievement is, the list every game has, the form of a player's own list and of the file that keeps the earned + ones; this file has what is the game folder's and the screen's. + + - On by default (11-22, the owner's word); HOS_ACHIEVEMENTS=0 (triforce_env.txt) turns them off. While off nothing is + read, of the game or of the disk. + - The lists are read at the game's first present (hos_achievements_frame, host/nvn_native.c): the built-in one, then + achievements.txt in the game folder when there is one. Once the renderer answers they are named to the menu + (HOS_OPT_ACH_TEXT, _ADD, _COUNT: gpu/vk/achievement_rows.h), which lists them on the System tab's Achievements page. + - host/hos_main.c asks once a second at Link's update, on the game's thread, while a save is loaded, and hands over + how the save data is read (hos_achievements_check). + - An achievement earned is written to triforce_achievements.txt in the game folder at once (the file whole, under + another name first), said in the log, and shown: the menu's notice (HOS_OPT_ACH_EARNED) with the system menu's + decide sound (HOS_MENU_SOUND_SYS_DECIDE, at the Menu sounds row's volume; silent where that row is Off. Not the + item fanfare: most are earned as the game plays that itself). One at a time: a save that earns several at once + shows them one after the other. + - The present's thread and the game's both come here: what they share is under one lock, held for a list's naming + or a check and no longer. + - Nothing of the game's is written here, and nothing is changed in the game: the save data is only read. */ +#include "hos.h" +#include "hos_paths.h" +#include +#include +#include +#include +#include "../gpu/vk/vk_api.h" +#include "achievements.h" +#include "menu_sound.h" + +int rx_option(int id, int otherwise); +void rx_option_set(int id, int value); + +#define EARNED_FILE "triforce_achievements.txt" +#define OWN_FILE "achievements.txt" +enum { OWN_MOST = 1 << 18, SHOWN_MS = 4500 /* (a notice stays 4 s: gpu/vk/vk_menu.cpp) */ }; + +static AchList list; +static int ready; /* the lists were read */ +static int named; /* and named to the menu: it shows them, and a notice for one earned */ +static int waiting[ACH_MAX], nwaiting; /* earned, not yet shown */ +static int64_t next_shown; +static volatile char busy; /* the lock: the present's thread names the list, the game's checks it */ + +static void lock(void) { + while (__atomic_test_and_set(&busy, __ATOMIC_ACQUIRE)) {} +} +static void unlock(void) { __atomic_clear(&busy, __ATOMIC_RELEASE); } + +static const char *last_sep(const char *s) { + const char *b = strrchr(s, '\\'), *f = strrchr(s, '/'); + return b > f ? b : f; +} +/* a file of the game folder */ +static int game_file(char *out, size_t cap, const char *name) { + char exe[MAX_PATH]; + const DWORD n = hos_game_path_a(exe, sizeof exe); + const char *slash = n && n < sizeof exe ? last_sep(exe) : NULL; + if (!slash) return 0; + return snprintf(out, cap, "%.*s%s", (int)(slash + 1 - exe), exe, name) < (int)cap; +} + +/* a text file of the game folder whole, its end a 0 (freed by the caller); NULL when it is not there or over `most` */ +static char *read_text(const char *name, long most) { + char path[MAX_PATH + 40]; + if (!game_file(path, sizeof path, name)) return NULL; + FILE *f = fopen(path, "rb"); + if (!f) return NULL; + char *text = NULL; + long size = 0; + if (!fseek(f, 0, SEEK_END) && (size = ftell(f)) >= 0 && size <= most && !fseek(f, 0, SEEK_SET) && (text = (char *)malloc((size_t)size + 1))) { + const size_t got = fread(text, 1, (size_t)size, f); + text[got] = 0; + for (size_t i = 0; i < got; i++) + if (!text[i]) text[i] = ' '; /* (a 0 in the middle is not the text's end) */ + } + fclose(f); + return text; +} + +static void earned_save(void) { + static char text[ACH_MAX * (ACH_NAME + 40) + ACH_ORPHANS * ACH_LINE + 200]; + char path[MAX_PATH + 40], fresh[MAX_PATH + 48]; + const size_t n = ach_earned_write(&list, text, sizeof text); + if (!n || !game_file(path, sizeof path, EARNED_FILE)) return; + snprintf(fresh, sizeof fresh, "%s.new", path); + FILE *f = fopen(fresh, "wb"); /* (its lines end the same on Windows and Linux) */ + const int ok = f && fwrite(text, 1, n, f) == n; + if ((f && fclose(f)) || !ok || !MoveFileExA(fresh, path, MOVEFILE_REPLACE_EXISTING)) { + remove(fresh); + hos_log("achievements: %s could not be written: what was earned is kept until the game ends", path); + } +} + +static void lists_read(void) { + int refused = 0; + const int built_in = ach_parse(&list, ACH_BUILTIN, &refused); + char *own = read_text(OWN_FILE, OWN_MOST); + const int own_n = own ? ach_parse(&list, own, &refused) : 0; + free(own); + char *earned = read_text(EARNED_FILE, OWN_MOST); + const int marked = earned ? ach_earned_read(&list, earned) : 0; + free(earned); + hos_log("achievements: %d (%d of " OWN_FILE "%s), %d earned so far%s", list.n, own_n, own ? "" : ", which is not there", marked, + built_in + own_n ? "" : ": none to earn"); + if (refused) + hos_log("achievements: %d left out: a name that is empty, over %d bytes or another's, no condition, over %d of them, or a " + "line that is not about:, icon: [], item [] or flag ", refused, ACH_NAME, ACH_CONDS); +} + +/* ---- the menu's side: gpu/vk/achievement_rows.h ---- */ +static void to_menu(const char *text) { + for (const char *p = text; *p; p++) rx_option_set(HOS_OPT_ACH_TEXT, (unsigned char)*p); +} +/* the day of a unix time, as the page shows it ("" when it is not known) */ +static void day_of(int64_t when, char *out, size_t cap) { + const time_t t = (time_t)when; + const struct tm *tm = when > 0 ? localtime(&t) : NULL; + if (!tm || !strftime(out, cap, "%Y-%m-%d", tm)) out[0] = 0; +} +static void name_list(void) { + for (int i = 0; i < list.n; i++) { + const Ach *a = &list.a[i]; + char day[16]; + day_of(a->earned ? a->when : 0, day, sizeof day); + to_menu(a->name), to_menu("\n"), to_menu(a->about), to_menu("\n"), to_menu(day); + rx_option_set(HOS_OPT_ACH_ADD, (a->icon & 255) | (a->earned ? HOS_ACH_EARNED : 0)); + } + rx_option_set(HOS_OPT_ACH_COUNT, list.n); +} + +/* On unless HOS_ACHIEVEMENTS=0 (11-22: on by default). Read once */ +int hos_achievements_on(void) { + static int on = -1; + if (on < 0) { + const char *e = getenv("HOS_ACHIEVEMENTS"); + on = !(e && e[0] == '0' && !e[1]); + } + return on; +} + +/* host/nvn_native.c, at each present of the game's: the lists read at the first, and named to the menu once it answers */ +void hos_achievements_frame(void) { + static int done; /* (the present's thread alone) */ + if (done || !hos_achievements_on()) return; + lock(); + if (!ready) ready = 1, lists_read(); + if (rx_option(HOS_OPT_ACH_ASK, 0) > 0) named = done = 1, name_list(); + unlock(); +} + +/* host/hos_main.c, once a second at Link's update while a save is loaded (now_ns: hos_now_ns): what the save data has + earned since the last call is marked, kept and shown */ +void hos_achievements_check(AchProbe probe, void *ctx, int64_t now_ns) { + if (!hos_achievements_on()) return; + lock(); + if (!ready) ready = 1, lists_read(); + int fresh[ACH_MAX]; + const int got = ach_check(&list, probe, ctx, (int64_t)time(NULL), fresh, ACH_MAX); + for (int i = 0; i < got; i++) { + hos_log("achievements: earned \"%s\" (%d of %d)", list.a[fresh[i]].name, ach_earned_count(&list) - (got - 1 - i), list.n); + if (nwaiting < ACH_MAX) waiting[nwaiting++] = fresh[i]; + } + if (got) earned_save(); + if (nwaiting && named && now_ns >= next_shown) { /* (not before the menu has the list: it shows them by their place in it) */ + const int at = waiting[0]; + char day[16]; + memmove(waiting, waiting + 1, sizeof waiting[0] * (size_t)--nwaiting); + next_shown = now_ns + SHOWN_MS * 1000000LL; + day_of(list.a[at].when, day, sizeof day); + to_menu(day); + rx_option_set(HOS_OPT_ACH_EARNED, at); + hos_menu_sound_play(HOS_MENU_SOUND_SYS_DECIDE, now_ns); + } + unlock(); +} diff --git a/host/achievements.h b/host/achievements.h new file mode 100644 index 0000000..63d012a --- /dev/null +++ b/host/achievements.h @@ -0,0 +1,300 @@ +/* (Triforce) Achievements: what one is, the list's text, when one is earned, and the file that keeps the earned ones. + host/achievements.c has what is the game folder's and the screen's; host/hos_main.c asks the game. Nothing of the host is + in here: tests/test_achievements.py compiles it alone. + + An achievement has a name, a line about it, and one to ACH_CONDS conditions, all of which must hold: + item [] Link has that item, of it or more (1 without a number) + flag that flag of the game's save data is set + The list is text: + # a comment + [A Sword of Your Own] + about: Find a blade in the forest. + icon: sword green + item Weapon_Sword_Kokiri + (icon: its picture on the Achievements page and in its notice, a shape and a tone the menu draws: ach_icon below.) + ACH_BUILTIN below is the list every game has. Its conditions are items only, by the pouch names gpu/vk/oot_cheat_items.h + already has for the Inventory page: no other name of the game's is in this file, and none of its flags' names (the + Debug tab reads those from the player's own game files). A player's own list, achievements.txt beside the game, is read + after it in the same form, and may ask for flags by the names the Game flags page shows. + + An achievement is earned once, for the game folder and not for a save slot, and stays earned: triforce_achievements.txt + has a line for each, " ", the key the FNV-1a 32 of the name. A line whose achievement the lists + no longer have is kept as it is, so that a list taken away and put back loses nothing. */ +#ifndef HOS_ACHIEVEMENTS_H +#define HOS_ACHIEVEMENTS_H +#include +#include +#include +#include +#include + +enum { ACH_MAX = 128, ACH_CONDS = 16, ACH_NAME = 40 /* (a name's bytes: gpu/vk/achievement_rows.h kName holds it whole) */, ACH_ABOUT = 160, + ACH_WHAT = 96 /* (a flag's or an item's name: gpu/vk/oot_debug_link.h OOT_NAME_MAX and its end) */, ACH_ORPHANS = 64, + ACH_LINE = 256 }; +enum { ACH_ITEM = 'I', ACH_FLAG = 'F' }; + +typedef struct AchCond { + int kind; /* ACH_ITEM or ACH_FLAG */ + int least; /* ACH_ITEM: how many of it */ + char what[ACH_WHAT]; +} AchCond; +typedef struct Ach { + char name[ACH_NAME + 1], about[ACH_ABOUT + 1]; + uint32_t key; + int icon; /* its picture: a shape and a tone, ach_icon */ + AchCond conds[ACH_CONDS]; + int nconds; + int earned; + int64_t when; /* unix time it was earned, 0 not known */ +} Ach; +typedef struct AchList { + Ach a[ACH_MAX]; + int n; + char orphans[ACH_ORPHANS][ACH_LINE]; /* (the earned file's lines of achievements no list has now) */ + int norphans; +} AchList; + +/* What the game's save data has of a condition's name: an item's count, a flag's 1 or 0; -1 it has no such name */ +typedef int (*AchProbe)(void *ctx, int kind, const char *what); + +#define ACH_BUILTIN \ + "[A Sword of Your Own]\nabout: Find a blade in the forest.\nicon: sword green\nitem Weapon_Sword_Kokiri\n" \ + "[First Melody]\nabout: Learn the royal family's song.\nicon: note\nitem Item_Melody_ZeldaLullaby\n" \ + "[Forest's Thanks]\nabout: Earn the first Spiritual Stone.\nicon: gem green\nitem Item_Stone_Forest\n" \ + "[Heart of the Mountain]\nabout: Earn the second Spiritual Stone.\nicon: gem red\nitem Item_Stone_Fire\n" \ + "[Jewel of the River]\nabout: Earn the third Spiritual Stone.\nicon: gem blue\nitem Item_Stone_Water\n" \ + "[Three Stones]\nabout: Hold all three Spiritual Stones.\nicon: gem\nitem Item_Stone_Forest\nitem Item_Stone_Fire\nitem Item_Stone_Water\n" \ + "[Seven Years Later]\nabout: Draw the blade from its pedestal.\nicon: sword blue\nitem Weapon_Sword_Master\n" \ + "[Sage of Forest]\nabout: Earn the Forest Medallion.\nicon: medal green\nitem Item_Medal_Forest\n" \ + "[Sage of Fire]\nabout: Earn the Fire Medallion.\nicon: medal red\nitem Item_Medal_Fire\n" \ + "[Sage of Water]\nabout: Earn the Water Medallion.\nicon: medal blue\nitem Item_Medal_Water\n" \ + "[Sage of Shadow]\nabout: Earn the Shadow Medallion.\nicon: medal purple\nitem Item_Medal_Shadow\n" \ + "[Sage of Spirit]\nabout: Earn the Spirit Medallion.\nicon: medal orange\nitem Item_Medal_Spirit\n" \ + "[Sage of Light]\nabout: Earn the Light Medallion.\nicon: medal yellow\nitem Item_Medal_Light\n" \ + "[Six Awakened]\nabout: Hold all six medallions.\nicon: medal\nitem Item_Medal_Forest\nitem Item_Medal_Fire\nitem Item_Medal_Water\n" \ + "item Item_Medal_Shadow\nitem Item_Medal_Spirit\nitem Item_Medal_Light\n" \ + "[Full Songbook]\nabout: Learn all twelve songs.\nicon: note yellow\nitem Item_Melody_ZeldaLullaby\nitem Item_Melody_EponaSong\n" \ + "item Item_Melody_SariaSong\nitem Item_Melody_SunSong\nitem Item_Melody_TimeSong\nitem Item_Melody_StormSong\n" \ + "item Item_Melody_ForestMinuet\nitem Item_Melody_FireBolero\nitem Item_Melody_WaterSerenade\n" \ + "item Item_Melody_ShadowNocturne\nitem Item_Melody_SpiritRequiem\nitem Item_Melody_LightPrelude\n" \ + "[Spider Hunter]\nabout: Collect 10 Gold Skulltula Tokens.\nicon: token orange\nitem Item_Token_Gold 10\n" \ + "[Spider Slayer]\nabout: Collect 50 Gold Skulltula Tokens.\nicon: token silver\nitem Item_Token_Gold 50\n" \ + "[Curse Broken]\nabout: Collect all 100 Gold Skulltula Tokens.\nicon: token\nitem Item_Token_Gold 100\n" \ + "[Long Reach]\nabout: Find the Longshot.\nicon: star blue\nitem Item_Longshot\n" \ + "[Seeing Clearly]\nabout: Find the Lens of Truth.\nicon: star purple\nitem Armor_GlassTruth\n" \ + "[Honest Face]\nabout: Earn the Mask of Truth.\nicon: star orange\nitem Armor_Child_Mask_Truth\n" \ + "[A Giant's Blade]\nabout: Earn Biggoron's Sword.\nicon: sword\nitem Weapon_Sword_Biggoron\n" \ + "[Deep Pockets]\nabout: Carry the Giant's Wallet.\nicon: token green\nitem Item_Wallet_Large\n" \ + "[Light Against the Dark]\nabout: Receive the Light Arrow.\nicon: star yellow\nitem Item_Arrow_Light\n" \ + "[Twice as Tough]\nabout: Receive Double Defense.\nicon: shield silver\nitem Item_FairyBless_DefenseDouble\n" + +/* FNV-1a 32 of a name's bytes */ +static inline uint32_t ach_key(const char *name) { + uint32_t h = 2166136261u; + for (const unsigned char *p = (const unsigned char *)name; *p; p++) h = (h ^ *p) * 16777619u; + return h; +} + +/* a character of a flag's or an item's name, as the game's are: letters, digits, _ . and - */ +static inline int ach_what_char(int ch) { + return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') || ch == '_' || ch == '.' || ch == '-'; +} + +static inline int ach_find(const AchList *l, uint32_t key) { + for (int i = 0; i < l->n; i++) + if (l->a[i].key == key) return i; + return -1; +} + +/* a line's text between its blanks: its start, and its length through *n */ +static inline const char *ach_trim(const char *s, size_t *n) { + while (*n && (*s == ' ' || *s == '\t')) s++, (*n)--; + while (*n && (s[*n - 1] == ' ' || s[*n - 1] == '\t' || s[*n - 1] == '\r')) (*n)--; + return s; +} + +/* An achievement's picture, drawn by the menu and not a file (gpu/vk/achievement_rows.h has the same shapes and tones in + the same order, and draws them): a shape in the low four bits, a tone in the next four. "icon: []" in + a list; without the line, a gold star */ +#define ACH_SHAPES "star", "sword", "gem", "medal", "note", "token", "shield" +#define ACH_TONES "gold", "green", "red", "blue", "purple", "orange", "yellow", "silver" +static inline int ach_icon(int shape, int tone) { return shape | tone << 4; } + +/* the words after "icon:" read: the icon, -1 when they are not a shape and at most a tone */ +static inline int ach_icon_parse(const char *s, size_t n) { + static const char *const shapes[] = {ACH_SHAPES}, *const tones[] = {ACH_TONES}; + size_t w = 0; + while (w < n && s[w] != ' ' && s[w] != '\t') w++; + int shape = -1, tone = 0; + for (size_t i = 0; i < sizeof shapes / sizeof shapes[0]; i++) + if (strlen(shapes[i]) == w && !memcmp(shapes[i], s, w)) shape = (int)i; + if (shape < 0) return -1; + size_t rest = n - w; + const char *t = ach_trim(s + w, &rest); + if (!rest) return ach_icon(shape, 0); + tone = -1; + for (size_t i = 0; i < sizeof tones / sizeof tones[0]; i++) + if (strlen(tones[i]) == rest && !memcmp(tones[i], t, rest)) tone = (int)i; + return tone < 0 ? -1 : ach_icon(shape, tone); +} + +/* A condition's line read ("item []", "flag "): 1 when it is one */ +static inline int ach_cond_parse(const char *s, size_t n, AchCond *out) { + memset(out, 0, sizeof *out); + if (n < 6 || (memcmp(s, "item", 4) && memcmp(s, "flag", 4)) || (s[4] != ' ' && s[4] != '\t')) return 0; + out->kind = s[0] == 'i' ? ACH_ITEM : ACH_FLAG; + out->least = 1; + size_t at = 4, w = 0; + while (at < n && (s[at] == ' ' || s[at] == '\t')) at++; + while (at < n && ach_what_char((unsigned char)s[at])) { + if (w + 1 >= ACH_WHAT) return 0; + out->what[w++] = s[at++]; + } + if (!w) return 0; + if (at == n) return 1; + if (out->kind != ACH_ITEM || (s[at] != ' ' && s[at] != '\t')) return 0; + while (at < n && (s[at] == ' ' || s[at] == '\t')) at++; + int least = 0, digits = 0; + for (; at < n && s[at] >= '0' && s[at] <= '9' && digits < 5; at++, digits++) least = least * 10 + (s[at] - '0'); + if (!digits || at != n || least < 1 || least > 9999) return 0; + out->least = least; + return 1; +} + +/* A list's text added to the list. An achievement is taken whole or not at all: one with a line that is none of the + form's, without a condition, with more than ACH_CONDS, with a name that is empty, too long or another's, or past + ACH_MAX is left out, and counted in *refused (may be NULL). Returns how many were added */ +static inline int ach_parse(AchList *l, const char *text, int *refused) { + int added = 0, bad = 0, open = 0, broken = 0; + static Ach cur; /* (too large for a guest worker's stack; the lists are read by one thread, at the first check) */ + for (const char *line = text; ; ) { + const char *end = line; + while (*end && *end != '\n') end++; + size_t n = (size_t)(end - line); + const char *s = ach_trim(line, &n); + const int head = n && s[0] == '['; + if (open && (head || (!*end && !n))) { /* (the achievement before this line is whole now) */ + if (broken || !cur.nconds || ach_find(l, cur.key) >= 0 || l->n >= ACH_MAX) bad++; + else l->a[l->n++] = cur, added++; + open = 0; + } + if (head) { + memset(&cur, 0, sizeof cur); + open = 1, broken = 0; + size_t nn = s[n - 1] == ']' ? n - 2 : 0; + const char *name = ach_trim(s + 1, &nn); + if (!nn || nn > ACH_NAME) broken = 1; + else { + memcpy(cur.name, name, nn); + for (size_t i = 0; i < nn; i++) + if ((unsigned char)cur.name[i] < 32) broken = 1; + cur.key = ach_key(cur.name); + } + } else if (open && n && s[0] != '#') { /* (a line before the first name is not an achievement's) */ + AchCond c; + if (n >= 6 && !memcmp(s, "about:", 6)) { + size_t an = n - 6; + const char *a = ach_trim(s + 6, &an); + if (an > ACH_ABOUT) broken = 1; + else memcpy(cur.about, a, an), cur.about[an] = 0; + } else if (n >= 5 && !memcmp(s, "icon:", 5)) { + size_t in = n - 5; + const char *i = ach_trim(s + 5, &in); + if ((cur.icon = ach_icon_parse(i, in)) < 0) broken = 1; + } else if (cur.nconds < ACH_CONDS && ach_cond_parse(s, n, &c)) cur.conds[cur.nconds++] = c; + else broken = 1; + } + if (!*end) { + if (!n) break; + line = end; /* (the text's last line had no line feed: once more, at its end, for the achievement it closes) */ + } else line = end + 1; + } + if (refused) *refused += bad; + return added; +} + +/* 1 when every condition of the achievement holds in the save data the probe reads */ +static inline int ach_met(const Ach *a, AchProbe probe, void *ctx) { + for (int i = 0; i < a->nconds; i++) { + const AchCond *c = &a->conds[i]; + const int have = probe(ctx, c->kind, c->what); + if (have < (c->kind == ACH_ITEM ? c->least : 1)) return 0; + } + return a->nconds > 0; +} + +/* The achievements not yet earned whose conditions hold are earned now (when: unix time): their places in the list go + to fresh, cap of them at most a call (the rest at the next), and how many is returned. Nothing is asked of the probe + for an achievement that is earned already */ +static inline int ach_check(AchList *l, AchProbe probe, void *ctx, int64_t when, int *fresh, int cap) { + int got = 0; + for (int i = 0; i < l->n && got < cap; i++) { + Ach *a = &l->a[i]; + if (a->earned || !ach_met(a, probe, ctx)) continue; + a->earned = 1, a->when = when; + fresh[got++] = i; + } + return got; +} + +static inline int ach_earned_count(const AchList *l) { + int n = 0; + for (int i = 0; i < l->n; i++) n += l->a[i].earned != 0; + return n; +} + +/* The earned file's text read: " " a line. A line of an achievement the list has + marks it earned; any other line of that form is kept for the next write (ACH_ORPHANS at most); the rest is dropped. + Returns how many of the list's were marked */ +static inline int ach_earned_read(AchList *l, const char *text) { + int marked = 0; + for (const char *line = text; *line; ) { + const char *end = line; + while (*end && *end != '\n') end++; + size_t n = (size_t)(end - line); + const char *s = ach_trim(line, &n); + char *after = NULL; + const unsigned long key = n >= 10 && s[0] != '#' ? strtoul(s, &after, 16) : 0; + if (after == s + 8 && *after == ' ') { + const long long when = strtoll(after + 1, &after, 10); + const int at = ach_find(l, (uint32_t)key); + if (at >= 0) { + if (!l->a[at].earned) marked++; + l->a[at].earned = 1, l->a[at].when = when > 0 ? when : 0; + } else if (l->norphans < ACH_ORPHANS && n < ACH_LINE) { + int dup = 0; + for (int i = 0; i < l->norphans; i++) dup |= !strncmp(l->orphans[i], s, 8); + if (!dup) memcpy(l->orphans[l->norphans], s, n), l->orphans[l->norphans++][n] = 0; + } + } + line = *end ? end + 1 : end; + } + return marked; +} + +/* The earned file's text written into out: its length, 0 when cap is too small for it */ +static inline size_t ach_earned_write(const AchList *l, char *out, size_t cap) { + size_t at = 0; + int n = snprintf(out, cap, "# The achievements earned in this game folder, a line each: key, when (unix time), name.\n"); + if (n < 0 || (size_t)n >= cap) return 0; + at = (size_t)n; + for (int i = 0; i < l->n; i++) { + const Ach *a = &l->a[i]; + if (!a->earned) continue; + n = snprintf(out + at, cap - at, "%08x %lld %s\n", (unsigned)a->key, (long long)a->when, a->name); + if (n < 0 || (size_t)n >= cap - at) return 0; + at += (size_t)n; + } + for (int i = 0; i < l->norphans; i++) { + n = snprintf(out + at, cap - at, "%s\n", l->orphans[i]); + if (n < 0 || (size_t)n >= cap - at) return 0; + at += (size_t)n; + } + return at; +} + +/* host/achievements.c */ +int hos_achievements_on(void); +void hos_achievements_frame(void); +void hos_achievements_check(AchProbe probe, void *ctx, int64_t now_ns); +#endif diff --git a/host/display.c b/host/display.c index 7bffa6c..1ab61c1 100644 --- a/host/display.c +++ b/host/display.c @@ -262,6 +262,16 @@ static void typed_text(HWND w, WPARAM wp) { rx_option_set(HOS_OPT_TEXT_CHAR, (int)u); } +/* (Triforce, 11-22, after Casual Gamer's fix) A lone Alt: DefWindowProc enters the menu loop (WM_SYSCOMMAND with SC_KEYMENU) + on the window's thread, which during the start-up screen is the one waiting for the shaders, and the screen froze until a key + was pressed. The window has no menu bar: nothing to enter. Alt+Space (the system menu, lp ' ') still goes on to Windows; + Alt+Enter, F10 and Alt+F4 are WM_SYSKEYDOWN's and WM_SYSKEYUP's, below. HOS_ALT_MENU=1: as before */ +static int keymenu_dropped(WPARAM wp, LPARAM lp) { + static int before = -1; + if (before < 0) before = getenv("HOS_ALT_MENU") && atoi(getenv("HOS_ALT_MENU")) != 0; + return !before && (wp & 0xFFF0) == SC_KEYMENU && lp != ' '; +} + static LRESULT CALLBACK wndproc(HWND w, UINT msg, WPARAM wp, LPARAM lp) { switch (msg) { case WM_MOUSEMOVE: @@ -329,6 +339,7 @@ static LRESULT CALLBACK wndproc(HWND w, UINT msg, WPARAM wp, LPARAM lp) { return 0; } case WM_SYSCOMMAND: + if (keymenu_dropped(wp, lp)) return 0; /* (a lone Alt: no menu loop, keymenu_dropped) */ if ((wp & 0xFFF0) == SC_SCREENSAVE && !(wp & SCF_ISSECURE) && renderer && GetForegroundWindow() == w && !IsIconic(w)) { BOOL secure = TRUE; /* unknown or protected policy keeps Windows' normal handling */ diff --git a/host/heap_commit.h b/host/heap_commit.h new file mode 100644 index 0000000..4be9ed8 --- /dev/null +++ b/host/heap_commit.h @@ -0,0 +1,52 @@ +/* (Triforce, 11-21, the performance audit's MEM-09, after Source A's) The game's heap committed as it is touched. + + The guest window is reserved, and what the runtime hands out is committed (hos_commit): the game asks for a heap of + ~6.9 GB at its start and all of it was committed there, though the game touches a part of it. Committed memory counts + against the PC's RAM plus page file whether touched or not: on a 16 GB PC with a small page file that is where "out + of memory" came from, with GBs of RAM free. On Windows the heap is now left reserved, and the first-touch handler + that was there already for pages nothing handed out (mkw_glue.c commit_on_touch) commits it, 64 KB at a touch. + + What had to be there first, as Source A found: + - two threads touching one new 64 KB at once: the second's fault finds the pages committed, and is tried again + (heap_touch_again) where it went on to the crash report; + - a commit refused at Windows' commit limit is tried again for 5 s (mkw_glue.c commit_wait), while the renderer's + textures give commit back under their own rule: a first touch has no result to give the game; + - the kernel does not fault pages in: a file read into heap pages nothing touched yet failed without an exception. + The calls that hand guest memory to the kernel read each page of their range once first (mkw_glue.c + hos_guest_touch), so that the handler commits the reserved ones. + And a first touch commits only the reserved pages of its 64 KB (heap_touch_range): a commit over committed pages + resets their protection, the lazy write-back's and the watches' too. + + HOS_HEAP_COMMIT=1: the heap committed whole at the start, as before. HOS_LARGE_PAGES: whole, as it is reserved so. + Linux: as it was (its window does not charge untouched pages). + + The rules alone: tests/test_heap_commit_lazy.py compiles this file. */ +#pragma once + +#include +#include + +/* Whether SetMemoryHeapSize commits the new heap whole. asked: HOS_HEAP_COMMIT's value (not 0: whole); large: the heap + is in large pages; windows: 0 on Linux */ +static inline int heap_commit_whole(const char *asked, int large, int windows) { + if (!windows || large) return 1; + return asked && atoi(asked) != 0; +} + +/* Whether a fault in the guest window is tried again with nothing done: its page is committed and read-write by now, + another thread's first touch having got there between the fault and the look. Not an execute fault (kind 8): a jump + into guest memory would fault again at once, for ever; it goes on to the crash report */ +static inline int heap_touch_again(int committed, int read_write, uint64_t kind) { + return committed && read_write && kind != 8; +} + +/* The pages a first touch at `at` commits, [*lo, *hi): of the 64 KB around it, the run of reserved pages it lies in + ([run_lo, run_hi): VirtualQuery's region) */ +static inline void heap_touch_range(uint64_t at, uint64_t run_lo, uint64_t run_hi, uint64_t *lo, uint64_t *hi) { + const uint64_t c_lo = at & ~0xFFFFULL, c_hi = c_lo + 0x10000; + *lo = run_lo > c_lo ? run_lo : c_lo; + *hi = run_hi < c_hi ? run_hi : c_hi; +} + +/* Whether the n-th first touch of the heap gets a log line: the first, then each doubling from 4096 (256 MB) */ +static inline int heap_touch_said(long n) { return n == 1 || (n >= 4096 && !(n & (n - 1))); } diff --git a/host/hos.h b/host/hos.h index ed3b154..d053426 100644 --- a/host/hos.h +++ b/host/hos.h @@ -54,6 +54,10 @@ int hos_game_is_mkw(void); /* (mkw_glue.c) guest pages made usable by host APIs. (Triforce) 1: committed, 0: the commit failed (logged), -1: not in the guest window (logged) */ int hos_commit(uint64_t addr, uint64_t size); +/* ((Triforce, 11-21, MEM-09) mkw_glue.c) before guest memory goes to the kernel: each of its pages read once, so that one + only reserved is committed (the heap committed as it is touched: hos_heap_on_touch, set by SetMemoryHeapSize) */ +extern int hos_heap_on_touch; +void hos_guest_touch(uint64_t addr, uint64_t size); /* (Triforce) the guest memory census (hos_mem.c, host/guest_census.h): each place that commits, maps or lets go of guest pages says what it did. COMMITTED: handed out and committed (before the commit itself); TOUCHED: committed on a first touch; PAGES, MAPPED: the game's own pages and physical maps; DECOMMITTED: let go and decommitted; ZEROED: let go and @@ -84,6 +88,18 @@ void web_mark_written(const void *p, size_t n); /* (hos/web/win32.c: guest mem #else #define HOS_WRITTEN(p, n) ((void)0) #endif +/* ((Triforce, 11-22, the performance audit's MEM-13, after Source A's; HOS_MEM_ZERO_MARK=1, off unless set) Guest memory + filled by host code: the freed pages a Vulkan import holds, which are zeroed in place and not discarded. The renderer + does not see a bare memset (the soft write watch knows what guest code and HOS_WRITTEN mark), so its mirror, its + captures and the write-back's shadow kept the freed bytes, and a later write-back could put them back over the zeros. + With the switch the fill is marked written: the memset first and the mark after it, since a capture takes the mark + before it copies the page. Without it: a plain memset, as before. Off until it is measured at arrivals: the marked + zeros are pages the captures and the mirror syncs then copy */ +int hos_mem_zero_mark(void); /* (hos_mem.c: HOS_MEM_ZERO_MARK, read once) */ +static inline void hos_guest_fill(uint64_t a, int v, uint64_t n) { + memset(GP(a), v, (size_t)n); + if (hos_mem_zero_mark()) HOS_WRITTEN(GP(a), (size_t)n); +} #ifdef A64_MEM_ZERO extern uint64_t hos_space_end; /* web: the guest window ends where the host's own data starts (hos_main.c) */ #define HOS_SPACE_END hos_space_end @@ -128,6 +144,7 @@ void hos_thread_join(HosThread *t); void hos_thread_destroy(HosThread *t); void hos_thread_set_name(HosThread *t, const char *name); void hos_thread_pin(void *h, const char *name); /* HOS_PIN: host CPU affinity by thread name */ +void hos_sim_cpu_apply(void); /* ((Triforce, 11-22, OS-49) HOS_SIM_CPU, a lab switch: hos_sim_cpu.h; before any thread */ int hos_game_cores(void); /* totk_cores.c: HOS_GAME_CORES, or 0 */ const char *hos_thread_name(HosThread *t); void hos_thread_affinity(HosThread *t, int ideal, uint64_t mask); @@ -142,6 +159,7 @@ int hos_vcore_away(void); void hos_vcore_back(void); void hos_vcore_checkpoint(void); void hos_vcore_yield(void); +int hos_vcore_asked(void); /* a waiter asked this thread for its core (vc_kick) */ int hos_cv_sleep(void *cv, void *srw_lock, unsigned ms); /* SleepConditionVariableSRW (exclusive) for guest threads */ void hos_thread_boost(int level); void hos_thread_stack(HosThread *t, uint64_t *addr, uint64_t *size); @@ -152,6 +170,7 @@ Ctx *hos_boot_main_thread(uint64_t stack_size); /* the calling host th uint32_t hos_thread_handle(HosThread *t); HosThread *hos_thread_by_handle(uint32_t h); int hos_thread_prio(HosThread *t); +int hos_thread_serial(HosThread *t); /* ((Triforce, OS-33) a thread the frame waits on: host/short_sleep.h) */ void hos_thread_wait(HosThread *t, int64_t timeout_ns, int *timed_out); /* < 0: forever */ void hos_thread_wake(HosThread *t); diff --git a/host/hos_main.c b/host/hos_main.c index 3dc18be..29e076d 100644 --- a/host/hos_main.c +++ b/host/hos_main.c @@ -15,6 +15,7 @@ #include "../gpu/vk/vk_api.h" #include "../gpu/vk/oot_cheat_items.h" #include "../gpu/vk/oot_debug_link.h" /* ((Triforce) the Game flags and Cutscenes pages' lines) */ +#include "achievements.h" /* ((Triforce) what an achievement asks of the save data: host/achievements.c) */ #include "../gpu/vk/ultrawide_rows.h" /* ((Triforce) the Ultrawide row's shapes: ultrawide_ratio) */ #include #include @@ -34,6 +35,7 @@ const char *hos_version(void) { return HOOKSHOT_VERSION; } /* (discord_rpc.c's #ifdef __linux__ #include #include +#include /* ((Triforce) driver_env_choose) */ #endif #include #include @@ -183,24 +185,30 @@ static int reserve_guest(void) { /* (only where /dev/udmabuf can be opened: a shared mapping's pages cannot be host-pointer imports (RADV: VK_ERROR_INVALID_EXTERNAL_HANDLE), so a pool udmabuf refused would be no import at all. A pool larger than udmabuf's size_limit_mb (64 MB unless raised) is imported as several dma-bufs bound into one sparse buffer, so no setting is - needed. HOS_GUEST_MEMFD=1: the memfd all the same) */ + needed. HOS_GUEST_MEMFD=1: the memfd all the same; =0: anonymous memory. + (Triforce, 11-21) A size_limit_mb that cannot be read counts as udmabuf's default, 64 MB, as the renderer takes it + (gpu/vk/vk_memory.cpp import_dma_buf): Flatpak and other sandboxes hide /sys/module while /dev/udmabuf opens, and the + window stayed anonymous there, 15-45 ms a submission on AMD. Only a limit that reads 0 (udmabuf then refuses every + dma-buf) keeps it anonymous) */ const char *e = getenv("HOS_GUEST_MEMFD"); int usable = 0; - unsigned long long limit_mb = 0; { const int u = open("/dev/udmabuf", O_RDWR | O_CLOEXEC); if (u >= 0) { close(u); + usable = 1; FILE *f = fopen("/sys/module/udmabuf/parameters/size_limit_mb", "r"); if (f) { - if (fscanf(f, "%llu", &limit_mb) != 1) limit_mb = 0; + unsigned long long limit_mb = 0; + if (fscanf(f, "%llu", &limit_mb) == 1 && limit_mb == 0) usable = 0; fclose(f); + } else { + hos_log("mem: udmabuf's size_limit_mb cannot be read (in a container?): its default, 64 MB, assumed"); } - usable = limit_mb > 0; } if (!usable && !e) hos_log("mem: guest memory stays host-pointer imports for the GPU (slow submissions on AMD): %s", - u < 0 ? "/dev/udmabuf cannot be opened" : "udmabuf's size_limit_mb cannot be read"); + u < 0 ? "/dev/udmabuf cannot be opened" : "udmabuf's size_limit_mb is 0"); } if (e ? atoi(e) != 0 : usable) { const int fd = (int)syscall(SYS_memfd_create, "hookshot-guest", 1u /* MFD_CLOEXEC */ | 2u /* MFD_ALLOW_SEALING */); @@ -467,6 +475,96 @@ static void load_hookshot_env(void) { fclose(f); } +#ifdef __linux__ +/* (Triforce, 11-21, Linux) Steam's Game Mode gives every game RADV_FORCE_VRS_CONFIG_FILE (its Half Rate Shading). While that + file exists, RADV on RDNA 2 and newer puts a forced shading rate into every pipeline's key, whatever the file holds, 1x1 too: + the pipelines prepared in Desktop Mode, or by the builder, missed the driver's caches and were all compiled again (on a Deck, + over 25 minutes to the title, against under one). The variable goes before Vulkan is loaded, when it names a file that + exists, unless HOS_HALF_RATE_SHADING=1 (triforce_env.txt can set it) keeps it; RADV_FORCE_VRS is left alone. What was done is + said on stderr now and in the log with the boot: lines (driver_env_note). The slider is worth little here: RADV keeps fragment + programs that read their position at full rate, and nearly all of this game's do */ +static char driver_env_note[640]; +static void driver_env_choose(void) { + const char *v = getenv("RADV_FORCE_VRS_CONFIG_FILE"); + struct stat st; + if (!v || !*v || stat(v, &st) != 0) return; + const char *keep = getenv("HOS_HALF_RATE_SHADING"); + if (keep && !strcmp(keep, "1")) { + snprintf(driver_env_note, sizeof driver_env_note, + "boot: RADV_FORCE_VRS_CONFIG_FILE=%s kept (HOS_HALF_RATE_SHADING=1: Steam's Half Rate Shading changes every " + "pipeline's key, so the driver's caches prepared without it miss)", v); + } else { + snprintf(driver_env_note, sizeof driver_env_note, + "boot: RADV_FORCE_VRS_CONFIG_FILE=%s dropped (Steam's Half Rate Shading changes every pipeline's key; " + "HOS_HALF_RATE_SHADING=1 keeps it)", v); + unsetenv("RADV_FORCE_VRS_CONFIG_FILE"); + } + fprintf(stderr, "%s\n", driver_env_note); +} + +/* (Triforce, 11-22, the performance audit's PIPE-97, after Source A's and Source B's) Room in the driver's own shader + cache for the pipelines the game prepares. The ~41,400 predicted pipelines are built into the driver's disk cache + alone, and NVIDIA's and Mesa's caches hold 1 GiB unless told otherwise and trim themselves to it: Source A saw a + full preparation stop at 1,026 MB with about 45% of it dropped, to be compiled again in play. Both drivers read + their size from the environment when they load, so the Linux main sets 10 GiB for each before Vulkan loads, in the + game's run and the builder's preparation alike, without overwriting: a size the player set stays. The cache only + grows by what is built. The env file cannot set these two names (env_line_allowed). Not set here: + __GL_SHADER_DISK_CACHE_SKIP_CLEANUP (the cache would then grow with every update's preparation). + HOS_DRIVER_CACHE_ROOM=0, in the environment or triforce_env.txt: neither is set, as before. Windows' NVIDIA driver + takes its size from its control panel: nothing is done there. One line for the log, as driver_env_choose's */ +static char driver_room_line[320]; +static void driver_cache_room(void) { + const char *off = getenv("HOS_DRIVER_CACHE_ROOM"); + if (off && !strcmp(off, "0")) { + snprintf(driver_room_line, sizeof driver_room_line, + "boot: the drivers' shader cache sizes left as they are (HOS_DRIVER_CACHE_ROOM=0)"); + } else { + setenv("__GL_SHADER_DISK_CACHE_SIZE", "10737418240", 0); /* (NVIDIA's, in bytes: 10 GiB) */ + setenv("MESA_SHADER_CACHE_MAX_SIZE", "10G", 0); /* (Mesa's: RADV, the Steam Deck) */ + const char *nv = getenv("__GL_SHADER_DISK_CACHE_SIZE"), *mesa = getenv("MESA_SHADER_CACHE_MAX_SIZE"); + snprintf(driver_room_line, sizeof driver_room_line, + "boot: room in the drivers' shader caches: __GL_SHADER_DISK_CACHE_SIZE=%.64s, MESA_SHADER_CACHE_MAX_SIZE=%.64s " + "(a size set before the game stays; HOS_DRIVER_CACHE_ROOM=0 sets neither)", nv ? nv : "", mesa ? mesa : ""); + } + fprintf(stderr, "%s\n", driver_room_line); +} + +/* (Triforce, 11-22, the performance audit's PIPE-98, after Source A's) NVIDIA's Linux driver keeps no shader cache at + all when __GL_SHADER_DISK_CACHE_PATH names a folder that is not there: it neither makes it nor falls back to + ~/.cache/nvidia, and every one of the game's prepared pipelines is compiled again at each start (Source A: a + preparation of 830.7 s against 51.0 s from a filled cache). So before Vulkan loads the Linux main makes the folder + the variable names, with its parents, for the user alone (0700). The value is never changed, and nothing is made + when the variable is not set: only a folder the player or a launcher named. One line when it cannot be made (a + file in the way, no permission), on stderr and kept for the log. HOS_DRIVER_CACHE_FOLDER=0, in the environment or + triforce_env.txt: nothing is made, as before. The driver files its cache by the program's file name, so the Linux + binary's name stays "triforce" (nx2/build.py EXE; tests/test_driver_cache_folder.py pins it), and + __GL_SHADER_DISK_CACHE_APP_NAME is never set: every player's cache would start over */ +static char driver_folder_line[1200]; +static void driver_cache_folder(void) { + const char *off = getenv("HOS_DRIVER_CACHE_FOLDER"); + const char *path = getenv("__GL_SHADER_DISK_CACHE_PATH"); + char dir[1024]; + struct stat st; + if ((off && !strcmp(off, "0")) || !path || !*path) return; + const size_t len = strlen(path); + if (len < sizeof dir) { + memcpy(dir, path, len + 1); + for (size_t i = 1; i < len; i++) { /* (each parent in turn; one that is there already is no failure) */ + if (dir[i] != '/') continue; + dir[i] = 0; + mkdir(dir, 0700); + dir[i] = '/'; + } + mkdir(dir, 0700); + } + if (len < sizeof dir && stat(path, &st) == 0 && S_ISDIR(st.st_mode)) return; /* (made, or there already) */ + snprintf(driver_folder_line, sizeof driver_folder_line, + "boot: __GL_SHADER_DISK_CACHE_PATH=%.900s is not a folder and cannot be made one: NVIDIA's driver keeps no " + "shader cache without it", path); + fprintf(stderr, "%s\n", driver_folder_line); +} +#endif + int hos_oot_protect_hearts; volatile int hos_oot_live_action; /* (test runs) the live input file's "!cheat ": host/nn_hid.c */ @@ -1237,6 +1335,24 @@ static void oot_link_tick(Ctx *c, uint64_t actor, int64_t now) { oot_link_say(OOT_SAID_FLAGS, states); } +/* ---- (Triforce) Achievements (host/achievements.c, on unless HOS_ACHIEVEMENTS=0): what the save data has of a + condition's name, read as the pages above and the Inventory page read it, and nothing written. Asked once a second at + Link's update, and only while a save is loaded: before that GameData holds a new game's values */ +static int oot_achievement_probe(void *ctx, int kind, const char *what) { + Ctx *c = (Ctx *)ctx; + if (kind == ACH_ITEM) return oot_item_get(c, what); + const char state = oot_flag_state(c, 1, oot_name_hash(what, strlen(what))); + return state == '1' ? 1 : state == '0' ? 0 : -1; +} + +static void oot_achievements_tick(Ctx *c, int64_t now) { + static int64_t next; + if (!hos_achievements_on() || now < next) return; + next = now + 1000000000LL; + if (oot_pouch_count(c, oot_item_names[0]) < 0) return; + hos_achievements_check(oot_achievement_probe, c, now); +} + static uint64_t oot_gameplay_view(Ctx *c) { const uint64_t singleton = recomp_load64(c, 0x84ca6bc0ULL); const uint64_t manager = singleton ? recomp_load64(c, singleton) : 0; @@ -1292,6 +1408,7 @@ static void oot_update_player_cheats(Ctx *c, uint64_t actor, uint64_t components oot_bench_tick(c, actor, now); /* ((Triforce, 11-19) benchmark mode: its tour, host/hos_bench.c */ oot_inventory_refresh(c, now); oot_link_tick(c, actor, now); /* ((Triforce) the Game flags and Cutscenes pages) */ + oot_achievements_tick(c, now); /* ((Triforce) host/achievements.c) */ if (action >= OOT_CHEAT_ITEM_ACTION && action < OOT_CHEAT_ITEM_ACTION + 256 * (int)OOT_ITEM_COUNT) { oot_inventory_edit(c, action); } else if (action == OOT_AGE_ACTION) { @@ -1843,7 +1960,13 @@ static int game_main(int argc, char **argv) { } #endif HANDLE err = GetStdHandle(STD_ERROR_HANDLE); - if (!err || err == INVALID_HANDLE_VALUE || GetFileType(err) == FILE_TYPE_UNKNOWN) { + /* ((Triforce, 11-21, ticket "Linux: games started with the launcher's Play write no triforce_log.txt") HOS_LOG_FILE=1: + the log file whatever stderr is. The launcher's Play on Linux gives the game pipes it reads and drops (so a full pipe + never blocks it), and a pipe is a known stderr: the whole log went into it and no triforce_log.txt was written, so a + session started from the launcher left nothing to send with a report, nor with an F8 capture) */ + const char *lf = getenv("HOS_LOG_FILE"); + const int log_file = lf && *lf && *lf != '0'; + if (log_file || !err || err == INVALID_HANDLE_VALUE || GetFileType(err) == FILE_TYPE_UNKNOWN) { char path[MAX_PATH + 32]; DWORD n = hos_game_path_a(path, MAX_PATH); while (n && path[n - 1] != 92 && path[n - 1] != 47) n--; @@ -1861,6 +1984,10 @@ static int game_main(int argc, char **argv) { } } hos_log("triforce %s", HOOKSHOT_VERSION); /* (the log's first line: which build a report comes from) */ +#ifdef __linux__ + if (driver_room_line[0]) hos_log("%s", driver_room_line); /* ((Triforce, 11-22) the performance audit's PIPE-97: decided in main, before this log was open */ + if (driver_folder_line[0]) hos_log("%s", driver_folder_line); /* ((Triforce, 11-22) the performance audit's PIPE-98: likewise */ +#endif #ifdef _WIN32 if (instance_held >= 0) hos_log("instance: %s", instance_held ? "this game folder is this run's (a second copy started on it says so and ends)" @@ -1876,6 +2003,7 @@ static int game_main(int argc, char **argv) { : "Windows' throttling of a window not in front left as it is (HOS_POWER_THROTTLING=1)"); } #endif + hos_sim_cpu_apply(); /* ((Triforce, 11-22, the performance audit's OS-49) HOS_SIM_CPU, a lab switch: before any thread */ const char *image = NULL; for (int i = 1; i < argc; i++) { if (!strcmp(argv[i], "--romfs") && i + 1 < argc) hos_romfs_dir = argv[++i]; @@ -2039,6 +2167,9 @@ static int game_main(int argc, char **argv) { (unsigned long long)(hos_total_memory >> 20)); #ifndef A64_MEM_ZERO memory_notice(); +#endif +#ifdef __linux__ + if (driver_env_note[0]) hos_log("%s", driver_env_note); /* ((Triforce) driver_env_choose's line) */ #endif void hos_mods_init(const char *image_path); hos_mods_init(image); /* mods.c: \mods (RomFS files) */ @@ -2090,6 +2221,11 @@ int main(int argc, char **argv) { int hos_display_init(void); void hos_display_loop(void); load_hookshot_env(); +#ifdef __linux__ + driver_env_choose(); /* ((Triforce) after the env file, which may set HOS_HALF_RATE_SHADING; before Vulkan, the builder's run too) */ + driver_cache_room(); /* ((Triforce, 11-22) the performance audit's PIPE-97: 10 GiB for the driver's shader cache, likewise */ + driver_cache_folder(); /* ((Triforce, 11-22) the performance audit's PIPE-98: the folder __GL_SHADER_DISK_CACHE_PATH names, made */ +#endif game_argc = argc, game_argv = argv; int prepare_shaders = 0; for (int i = 1; i < argc; i++) { diff --git a/host/hos_mem.c b/host/hos_mem.c index cc8d328..6cf4fd9 100644 --- a/host/hos_mem.c +++ b/host/hos_mem.c @@ -1,6 +1,8 @@ /* Guest memory map: the system region (our own objects), the heap, address regions. The whole 4 GB window is committed up front (hos_main.c), so "mapping" here is only bookkeeping and zeroing. */ #include "hos.h" +#include "heap_commit.h" /* ((Triforce, 11-21, MEM-09) the heap committed as it is touched) */ +#include "stall_host.h" /* ((Triforce, 11-22, OS-51) the host's part of a long frame, counted where it is spent) */ #include #include #include @@ -166,6 +168,57 @@ static void pages_mark_used_locked(uint64_t addr, uint64_t size) { for (uint64_t p = addr >> 12; p < (addr + size + 0xFFF) >> 12 && p < (1ULL << 20); p++) pages_used[p >> 6] |= 1ULL << (p & 63); } +/* ((Triforce, 11-22, the performance audit's MEM-13) HOS_MEM_ZERO_MARK=1: hos_guest_fill (hos.h) marks what it fills as + written. Read once, off unless set */ +int hos_mem_zero_mark(void) { + static int on = -1; + if (on < 0) { + const char *e = getenv("HOS_MEM_ZERO_MARK"); + const int v = e && *e && atoi(e) != 0; + if (v) hos_log("mem: freed pages zeroed under an import are marked written for the renderer (HOS_MEM_ZERO_MARK=1)"); + on = v; + } + return on; +} + +/* (Triforce, 11-22, the performance audit's OS-40, after Source A's) Whether a thread keeps its virtual core while it + discards. A discard is milliseconds in the kernel for a region of some MB, and before it the lazy write-back's pages + there are paid and pages a GPU import holds are zeroed; all the while the thread runs no guest code and the game + threads allowed only on its core wait. Source A gives the core up before a freed region's discard and takes it back + once, after the region is unlisted: at its Market the core waits of 50 ms and more went from 287 to 19 (275 of them + loader threads'), the longest from 177 to 45 ms, with the frames unchanged; the freeing thread once waited 124 ms for + its core back, in a load. + HOS_DISCARD_KEEP_CORE, read once: + - not set: as before. The core is given up for the discard's own kernel call, and only with the limit on + (HOS_MEM_LIMIT), where that was measured; hos_region_free itself gives nothing up; + - 0: Source A's. hos_region_free gives the core up before the whole of its discard (the write-back's payment and the + import's zero-fill too) and takes it back once the region is unlisted, never under the regions' lock; the discard's + own call gives it up as well, limit or not, which is what FreeMemoryPages' discard then has (under a region's + give-up it finds the core gone already and does nothing); + - 1 (any other number): the core kept through every discard, with the limit too. + Not the default: it reverses the choice made here to stay on upstream's path without the limit, it changes when a game + thread waits, and it was not measured here */ +enum { DISCARD_CORE_AS_BEFORE, DISCARD_CORE_GIVEN_UP, DISCARD_CORE_KEPT }; +static int discard_core_rule(const char *env) { + if (!env || !*env) return DISCARD_CORE_AS_BEFORE; + return atoi(env) != 0 ? DISCARD_CORE_KEPT : DISCARD_CORE_GIVEN_UP; +} +/* (a freed region's give-up around its whole discard; and the discard call's own) */ +static int discard_core_region_away(int mode) { return mode == DISCARD_CORE_GIVEN_UP; } +static int discard_core_call_away(int mode, int limit_on) { + return mode == DISCARD_CORE_GIVEN_UP || (mode == DISCARD_CORE_AS_BEFORE && limit_on); +} +static int discard_core(void) { + static int mode = -1; + if (mode < 0) { + mode = discard_core_rule(getenv("HOS_DISCARD_KEEP_CORE")); + if (mode != DISCARD_CORE_AS_BEFORE) + hos_log("mem: a thread %s while it discards freed memory (HOS_DISCARD_KEEP_CORE)", + mode == DISCARD_CORE_GIVEN_UP ? "gives its virtual core up for the whole of a freed region's discard" : "keeps its virtual core"); + } + return mode; +} + static void discard_pages(uint64_t addr, uint64_t size) { if (!hos_guest_range(addr, size)) return; /* (hookshot: guest pages only; Linux's discard zeroes what it is given) */ const uint64_t lo = (addr + 0xFFF) & ~0xFFFULL, hi = (addr + size) & ~0xFFFULL; @@ -181,12 +234,16 @@ static void discard_pages(uint64_t addr, uint64_t size) { (QueryMemoryInfo above), with the game's frames waiting on it (Hyrule Field 34 fps, Castle Town 15). Zeroed: the pages read as zero, as freed pages; the census keeps them for the give-back once no import holds them) */ extern int rx_guest_imported(uint64_t lo, uint64_t hi); + const int64_t stall0 = stall_host_begin(); /* ((Triforce, OS-51) the host's part of a long frame: stall_host.h) */ { const int io = hos_guest_io_begin(GP(lo), hi - lo); const int held = rx_guest_imported(lo, hi); - if (held) memset(GP(lo), 0, (size_t)(hi - lo)), hos_mem_note(HOS_MEM_ZEROED, lo, hi - lo); + if (held) hos_guest_fill(lo, 0, hi - lo), hos_mem_note(HOS_MEM_ZEROED, lo, hi - lo); /* ((Triforce, 11-22, MEM-13) marked with HOS_MEM_ZERO_MARK=1) */ hos_guest_io_end(io); - if (held) return; /* (lab tour: 5.2 GB zeroed in 0.47 s in all) */ + if (held) { /* (lab tour: 5.2 GB zeroed in 0.47 s in all) */ + stall_host_end(STALL_HOST_ZERO, stall0, hi - lo); + return; + } } /* (Triforce, the census: let go and still committed. Not the large pages above, left as they were, nor Linux's below, noted zeroed or decommitted there) */ @@ -203,9 +260,11 @@ static void discard_pages(uint64_t addr, uint64_t size) { in which the thread runs no guest code and cannot see a better thread's ask for its core (hos_thread.c vc_kick). Held to its memory (HOS_MEM_LIMIT) the game frees regions by the hundred while an area loads, on its Resource Memory thread. Only with the limit on, where it was measured: without it the game frees few regions and the path is upstream's) */ - const int away = hos_mem_limit() ? hos_vcore_away() : 0; + /* ((Triforce, 11-22, OS-40) HOS_DISCARD_KEEP_CORE: 0 gives it up without the limit too, 1 never: discard_core above) */ + const int away = discard_core_call_away(discard_core(), hos_mem_limit() != 0) ? hos_vcore_away() : 0; if (!discard || discard(GP(lo), (SIZE_T)(hi - lo)) != ERROR_SUCCESS) VirtualAlloc(GP(lo), (SIZE_T)(hi - lo), MEM_RESET, PAGE_READWRITE); + stall_host_end(STALL_HOST_DISCARD, stall0, hi - lo); /* ((Triforce, OS-51) with its write-back wait, not the wait for the core) */ if (away) hos_vcore_back(); hos_guest_io_end(io); #elif !defined(__EMSCRIPTEN__) @@ -213,15 +272,18 @@ static void discard_pages(uint64_t addr, uint64_t size) { extern int rx_guest_imported(uint64_t lo, uint64_t hi); /* (pages a dma-buf import holds stay where they are, zeroed: a hole punched under one would leave the GPU on the old pages and the game on new ones) */ - if (rx_guest_imported(lo, hi)) memset(GP(lo), 0, (size_t)(hi - lo)), hos_mem_note(HOS_MEM_ZEROED, lo, hi - lo); + if (rx_guest_imported(lo, hi)) hos_guest_fill(lo, 0, hi - lo), hos_mem_note(HOS_MEM_ZEROED, lo, hi - lo); /* ((Triforce, 11-22, MEM-13) as above) */ else hos_posix_guest_discard(GP(lo), (size_t)(hi - lo)), hos_mem_note(HOS_MEM_DECOMMITTED, lo, hi - lo); #endif } static int64_t region_frees_bare; /* ((Triforce) regions freed without a discard: hos_region_discard_needed; the refusal line's) */ +/* ((Triforce, 11-22, OS-40) the frees that gave their core up for the discard (HOS_DISCARD_KEEP_CORE=0), and how long the + longest waited for its core back; a line every 1,000) */ +static int64_t region_frees_away, region_back_longest_ns; void hos_region_free(uint64_t addr) { uint64_t freed_size = 0; - int game = 0; + int game = 0, away = 0; LOCK(); for (int i = 0; i < nregions; i++) { if (regions[i].addr == addr && !regions[i].freeing) { @@ -244,6 +306,9 @@ void hos_region_free(uint64_t addr) { region is discarded, as upstream does) */ const uint64_t game_pages = addr + freed_size <= GC_SPACE ? gc_count(&census, GC_GAME, addr, addr + freed_size) : ~0ULL; if (hos_region_discard_needed(game && hos_mem_limit(), hos_mem_census_on(), game_pages)) { + /* ((Triforce, 11-22, OS-40) HOS_DISCARD_KEEP_CORE=0: the core given up from here, with the region still listed and + marked, and taken back below once it is unlisted. A bare free, which makes no kernel call, keeps it) */ + if (discard_core_region_away(discard_core())) away = hos_vcore_away(); discard_pages(addr, freed_size); } else { hos_mem_note(HOS_MEM_DISCARDED, addr, freed_size); @@ -258,6 +323,20 @@ void hos_region_free(uint64_t addr) { } } UNLOCK(); + /* ((Triforce, 11-22, OS-40) the core back, after the lock is let go: a thread that queued for its core with the lock + held could wait on one that holds a core and waits for the lock) */ + if (away) { + const int64_t t0 = hos_now_ns(); + hos_vcore_back(); + const int64_t waited = hos_now_ns() - t0; + int64_t longest = __atomic_load_n(®ion_back_longest_ns, __ATOMIC_RELAXED); + while (waited > longest && + !__atomic_compare_exchange_n(®ion_back_longest_ns, &longest, waited, 1, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) {} + const int64_t n = __atomic_add_fetch(®ion_frees_away, 1, __ATOMIC_RELAXED); + if (n % 1000 == 0) + hos_log("mem: %lld freed regions discarded with the thread's virtual core given up; the longest waited %.1f ms for it " + "back (HOS_DISCARD_KEEP_CORE)", (long long)n, __atomic_exchange_n(®ion_back_longest_ns, 0, __ATOMIC_RELAXED) / 1e6); + } } /* ---- nn::os memory -------------------------------------------------------------------------------------- */ @@ -299,7 +378,24 @@ HLE(_ZN2nn2os17SetMemoryHeapSizeEm) { for (uint64_t a = HOS_HEAP_BASE + hos_heap_size; a < HOS_HEAP_BASE + size; a += 4096) *(volatile uint8_t *)GP(a) = 0; hos_log("heap: %llu MB prefaulted in %.0f ms", (unsigned long long)((size - hos_heap_size) >> 20), (hos_now_ns() - t0) / 1e6); } - if (size > hos_heap_size) hos_commit(HOS_HEAP_BASE + hos_heap_size, size - hos_heap_size); + /* ((Triforce, 11-21, the performance audit's MEM-09, after Source A's) Windows: the heap is left reserved and committed + as it is touched (mkw_glue.c commit_on_touch), not all at once: the game asks for ~6.9 GB and touches a part of it, and + committed, all of it counted against RAM plus the page file. heap_commit.h has the rule and what it needed first. + HOS_HEAP_COMMIT=1: whole, as before; and in large pages (HOS_LARGE_PAGES), reserved committed) */ + { + extern uint64_t hos_large_hi; +#if defined(_WIN32) + const int windows = 1; +#else + const int windows = 0; +#endif + const int whole = heap_commit_whole(getenv("HOS_HEAP_COMMIT"), hos_large_hi != 0, windows); + if (size > hos_heap_size && whole) hos_commit(HOS_HEAP_BASE + hos_heap_size, size - hos_heap_size); + if (size > hos_heap_size && !whole) { + hos_heap_on_touch = 1; /* (the kernel calls on guest memory touch their buffers first: hos_guest_touch) */ + hos_log("heap: left reserved, committed as it is touched (HOS_HEAP_COMMIT=1: all of it now, as before)"); + } + } hos_heap_size = size; hos_log("heap: %llu MB at %#llx", (unsigned long long)(size >> 20), (unsigned long long)HOS_HEAP_BASE); RET(0); @@ -433,9 +529,11 @@ HLE(_ZN2nn2os15FreeMemoryPagesEmm) { still maps them, and decommitted pages under an import faulted the device. Read as zero, as freed pages) */ extern int rx_guest_imported(uint64_t lo, uint64_t hi); /* (Triforce: zeroed or decommitted: the lazy write-back's pages there first, rx_backend.c) */ + const int64_t stall0 = hi > lo ? stall_host_begin() : 0; /* ((Triforce, OS-51) the host's part of a long frame) */ const int io = hi > lo ? hos_guest_io_begin(GP(lo), hi - lo) : 0; /* ((Triforce) the census: zeroed pages stay committed, kept for the give-back once no import holds them) */ - if (hi > lo && rx_guest_imported(lo, hi)) memset(GP(lo), 0, (size_t)(hi - lo)), hos_mem_note(HOS_MEM_ZEROED, lo, hi - lo); + /* ((Triforce, 11-22, MEM-13) hos_guest_fill: the zeros marked written with HOS_MEM_ZERO_MARK=1) */ + if (hi > lo && rx_guest_imported(lo, hi)) hos_guest_fill(lo, 0, hi - lo), hos_mem_note(HOS_MEM_ZEROED, lo, hi - lo); #if defined(_WIN32) else if (hi > lo) VirtualFree(GP(lo), (SIZE_T)(hi - lo), MEM_DECOMMIT), hos_mem_note(HOS_MEM_DECOMMITTED, lo, hi - lo); #else @@ -446,6 +544,7 @@ HLE(_ZN2nn2os15FreeMemoryPagesEmm) { } #endif hos_guest_io_end(io); + stall_host_end(STALL_HOST_DECOMMIT, stall0, hi > lo ? hi - lo : 0); RET(0); return; } @@ -736,7 +835,9 @@ int64_t hos_mem_give_back(uint64_t max_bytes) { uint64_t runs = 0, refused = 0; /* (from 4 GB up: below it AllocateMemoryPages zeroes pages without committing them, and a host call handed one that was given back would fail on it without a fault) */ + const int64_t stall0 = stall_host_begin(); /* ((Triforce, OS-51) the host's part of a long frame: stall_host.h) */ const uint64_t given = gc_give_back(&census, 1ULL << 32, &cursor, 16ULL << 20, max_bytes, 64, census_offer, NULL, &runs, &refused); + stall_host_end(STALL_HOST_GIVE_BACK, stall0, given); census_given += given, census_given_runs += runs, census_refused += refused; return census_unsupported && !given ? -1 : (int64_t)given; } diff --git a/host/hos_sim_cpu.h b/host/hos_sim_cpu.h new file mode 100644 index 0000000..658b25a --- /dev/null +++ b/host/hos_sim_cpu.h @@ -0,0 +1,75 @@ +/* (Triforce, 11-22, the performance audit's OS-49, after Source B's) A lab switch: the process held to fewer of the PC's + cores, so that a lab PC plays as one with fewer does: the game's threads, and the renderer's pools. + + HOS_SIM_CPU=/ ("6/12"; "6" alone: as many threads as cores). The process's affinity becomes of + the PC's physical cores (hos_sim_cpu_mask): the ones on the die with the largest L3 first where there is one (hos_x3d.h), + the fastest class first, the last ones first, so that the core kept for the replay (the last of the fastest class, + hos_thread.c last_fastest_core) is among them; at least one of the PC's cores always stays out. The renderer sizes its + pools by (gpu/vk/cpu_count.h): an affinity mask alone does not change what std::thread::hardware_concurrency + answers. Set in host/hos_main.c before any thread (hos_thread.c hos_sim_cpu_apply). Windows only. Unset, or a value that + cannot be read or asks for all of the PC's cores: nothing changes, and the variable is cleared so the renderer agrees. + + The rules alone, plain C: tests/test_sim_cores.py compiles them. */ +#pragma once +#include +#include + +typedef struct { + int cores, threads; +} HosSimCpu; + +/* 1 and *out for "/" or "": each 1 to 256, threads not under cores; else 0 */ +static inline int hos_sim_cpu_parse(const char *e, HosSimCpu *out) { + if (!e || !*e) return 0; + char *end = NULL; + const long c = strtol(e, &end, 10); + if (end == e) return 0; + long t = c; + if (*end == '/') { + const char *s = end + 1; + t = strtol(s, &end, 10); + if (end == s) return 0; + } + if (*end || c < 1 || c > 256 || t < c || t > 256) return 0; + out->cores = (int)c, out->threads = (int)t; + return 1; +} + +/* The process's mask: `want` of the n cores listed (masks[i]: a core's logical processors; classes[i]: its efficiency class, + higher is faster), the ones on `die` first when die is not 0, then the fastest class, then the last ones (the highest + processors). 0 when want is not from 1 to n - 1: one of the PC's cores always stays out */ +static inline uint64_t hos_sim_cpu_mask(const uint64_t *masks, const unsigned char *classes, int n, uint64_t die, int want) { + if (n > 256 || want < 1 || want > n - 1) return 0; + unsigned char taken[256] = {0}; + uint64_t mask = 0; + for (int k = 0; k < want; k++) { + int best = -1; + for (int i = 0; i < n; i++) { + if (taken[i]) continue; + if (best < 0) { + best = i; + continue; + } + const int on_i = die && (masks[i] & die), on_best = die && (masks[best] & die); + if (on_i != on_best) { + if (on_i) best = i; + continue; + } + if (classes[i] != classes[best]) { + if (classes[i] > classes[best]) best = i; + continue; + } + if (masks[i] > masks[best]) best = i; + } + taken[best] = 1; + mask |= masks[best]; + } + return mask; +} + +/* The count of logical processors in a mask (vc_spin_max sizes by it while the switch holds the process) */ +static inline int hos_sim_cpu_count(uint64_t mask) { + int n = 0; + for (; mask; mask &= mask - 1) n++; + return n; +} diff --git a/host/hos_svc.c b/host/hos_svc.c index 2e61c08..ec0cc28 100644 --- a/host/hos_svc.c +++ b/host/hos_svc.c @@ -357,6 +357,8 @@ static void query_memory(uint64_t out, uint64_t addr) { #define CUR_THREAD 0xFFFF8000u static HosThread *thread_handle(uint32_t h) { return h == CUR_THREAD ? hos_self() : hos_thread_by_handle(h); } +int hos_core_count_on(void); +void hos_core_count_svc(void); void a64_svc(Ctx *c, uint32_t imm) { hos_vcore_checkpoint(); switch (imm) { @@ -445,6 +447,7 @@ void a64_svc(Ctx *c, uint32_t imm) { return; } case 0x10: /* GetCurrentProcessorNumber */ + if (hos_core_count_on()) hos_core_count_svc(); /* (TICK-062 A10, HOS_CORE_COUNT) */ c->x[0] = (uint32_t)hos_thread_core(hos_self()); return; case 0x1E: /* GetSystemTick */ @@ -503,8 +506,8 @@ void a64_svc(Ctx *c, uint32_t imm) { extern int rx_guest_imported(uint64_t lo, uint64_t hi); const int io = hos_guest_io_begin(GP(c->x[0]), c->x[1]); /* (Triforce: the lazy write-back's pages first: rx_backend.c) */ /* ((Triforce) the census: zeroed pages stay committed, kept for the give-back once no import holds them) */ - if (rx_guest_imported(c->x[0], c->x[0] + c->x[1])) - memset(GP(c->x[0]), 0, (size_t)c->x[1]), hos_mem_note(HOS_MEM_ZEROED, c->x[0], c->x[1]); + if (rx_guest_imported(c->x[0], c->x[0] + c->x[1])) /* ((Triforce, 11-22, MEM-13) the zeros marked written with HOS_MEM_ZERO_MARK=1) */ + hos_guest_fill(c->x[0], 0, c->x[1]), hos_mem_note(HOS_MEM_ZEROED, c->x[0], c->x[1]); else VirtualFree(GP(c->x[0]), (SIZE_T)c->x[1], MEM_DECOMMIT), hos_mem_note(HOS_MEM_DECOMMITTED, c->x[0], c->x[1]); hos_guest_io_end(io); } diff --git a/host/hos_thread.c b/host/hos_thread.c index d5c1f0a..5efe516 100644 --- a/host/hos_thread.c +++ b/host/hos_thread.c @@ -6,7 +6,9 @@ +0x1F8, so lifted SDK code that reads it inline finds the same thread the HLE knows. */ #include "hos.h" #include "hos_x3d.h" +#include "hos_sim_cpu.h" /* ((Triforce, 11-22, OS-49) HOS_SIM_CPU, a lab switch: fewer of the PC's cores */ #include "pace_wait.h" /* ((Triforce) hos_sleep_until_ns: the frame wait to its time) */ +#include "short_sleep.h" /* ((Triforce, 11-21) how sleeps under a millisecond wait, by the PC's processors) */ #include #include #define WIN32_LEAN_AND_MEAN @@ -29,6 +31,7 @@ struct HosThread { uint64_t core_mask; uint64_t tls_slots[HOS_TLS_SLOTS]; char name[48]; + int serial; /* ((Triforce, OS-33) a thread the frame waits on, by its name: short_sleep.h short_sleep_serial_name) */ HANDLE handle; int started; int bg_kind; int64_t bg_since; /* (HOS_BG_THROTTLE: bg_throttle) */ @@ -40,6 +43,8 @@ struct HosThread { int vc_held, vc_last, vc_queued; volatile int vc_kick; int64_t vc_since, vc_grant_at; + /* (HOS_CORE_COUNT, TICK-062 A10) calls that ask for the core number (HLE, svc 0x10), waits for a virtual core and their time */ + int64_t cc_hle, cc_svc, cc_vcw, cc_vcw_ns; CONDITION_VARIABLE vc_cv; /* (hookshot) fibers: the context running now (NULL: ctx, the thread's own), the thread's host fiber once it switched to a guest fiber */ @@ -49,6 +54,7 @@ struct HosThread { /* The SDK's own mutexes (InternalCriticalSection) tag a lock word with the owner's handle, read from ThreadType+0x1B0 through TLR+0x1F8 (sdk+0x13186c). Every thread gets a unique handle there. */ +static void thread_gen_bump(void); #define TT_HANDLE 0x1B0 static volatile LONG next_handle = 0x10000; uint32_t hos_thread_handle(HosThread *t) { return t->kernel_handle; } @@ -59,10 +65,13 @@ void hos_thread_adopt_guest(HosThread *t) { uint64_t tt = G64(t->tlr + 0x1F8); if (tt && tt != t->guest_tt) { t->guest_tt = tt; + thread_gen_bump(); G32(tt + TT_HANDLE) = t->kernel_handle; } } int hos_thread_prio(HosThread *t) { return t->priority; } +/* ((Triforce, 11-22, the performance audit's OS-33) set where the thread is named, read by its own short sleeps) */ +int hos_thread_serial(HosThread *t) { return __atomic_load_n(&t->serial, __ATOMIC_RELAXED); } int hos_trace; extern const uint64_t hos_thread_start_hook, hos_thread_exit_hook; /* gen/hos_image.c */ @@ -136,7 +145,12 @@ int hos_thread_priority(HosThread *t, int set, int value) { return old; } -#if defined(_WIN32) && !defined(__EMSCRIPTEN__) +/* (Triforce, 11-22, the performance audit's THREAD-56, after Source A's) The log's writer thread on Linux too: it was + built for Windows only, so on Linux and the Steam Deck every hos_log wrote stderr in the thread that logged, a game + thread with its virtual core held (Source A saw a 717 ms frame: the audio thread waited 321 ms for a core whose + thread sat in such a write). Everything the writer calls is in the POSIX shim (host/posix/windows.h). Only the web + build has no writer. HOS_LOG_SYNC=1: each line written by the thread that logs it, as Linux did */ +#if !defined(__EMSCRIPTEN__) static SRWLOCK log_lock = SRWLOCK_INIT, log_write_lock = SRWLOCK_INIT; static CONDITION_VARIABLE log_cv = CONDITION_VARIABLE_INIT; static char *log_q[2048]; @@ -271,6 +285,10 @@ static int log_queue(const char *line) { if (on && !CreateThread(NULL, 0, log_main, NULL, 0, NULL)) on = 0; if (on) log_prev_filter = SetUnhandledExceptionFilter(log_crash_filter); if (on) atexit(log_flush); +#ifndef _WIN32 + /* ((Triforce, 11-22) THREAD-56: the shim's ExitProcess is quick_exit, which runs these and not atexit's) */ + if (on) at_quick_exit(log_flush); +#endif log_async = on; } ReleaseSRWLockExclusive(&log_lock); @@ -398,6 +416,7 @@ static void setup_guest(HosThread *t) { static void track(HosThread *t) { AcquireSRWLockExclusive(&threads_lock); if (nthreads < 1024) threads[nthreads++] = t; + thread_gen_bump(); ReleaseSRWLockExclusive(&threads_lock); } @@ -483,7 +502,15 @@ void hos_thread_wait(HosThread *t, int64_t timeout_ns, int *timed_out) { } void hos_thread_wake(HosThread *t) { SetEvent(t->wake); } -HosThread *hos_thread_of(uint64_t guest_tt) { +/* TICK-062 B2: hos_thread_of is a linear scan of every thread under a shared lock. HOS_THREAD_OF_CACHE=1: each thread keeps + a few (ThreadType, thread) pairs, valid while thread_gen has not changed. thread_gen is bumped by everything that can + change what the scan returns: a thread added (track) or removed (hos_thread_destroy), or a guest_tt rewritten (adopt). + A reader takes the generation BEFORE the scan and stores the pair under it: a change during the scan leaves the pair + stale at once, never served. The answer is then the scan's own (the first matching thread in the table), and a cached + pointer to a destroyed thread cannot be served (destroy bumps the generation before the free). */ +static volatile LONG thread_gen; +static void thread_gen_bump(void) { InterlockedIncrement(&thread_gen); } +static HosThread *thread_of_scan(uint64_t guest_tt) { HosThread *r = NULL; AcquireSRWLockShared(&threads_lock); for (int i = 0; i < nthreads; i++) @@ -491,6 +518,18 @@ HosThread *hos_thread_of(uint64_t guest_tt) { ReleaseSRWLockShared(&threads_lock); return r; } +HosThread *hos_thread_of(uint64_t guest_tt) { + static int on = -1; + if (on < 0) on = !(getenv("HOS_THREAD_OF_CACHE") && atoi(getenv("HOS_THREAD_OF_CACHE")) == 0); /* ((Triforce) on by default since 11-21, lab +5.8%; =0: off */ + if (!on) return thread_of_scan(guest_tt); + static _Thread_local struct { uint64_t tt; HosThread *t; LONG gen; } cache[8]; + const unsigned i = (unsigned)((guest_tt >> 4) * 0x9E3779B1u >> 7) & 7; + const LONG g = *(volatile LONG *)&thread_gen; + if (cache[i].tt == guest_tt && cache[i].gen == g && cache[i].t) return cache[i].t; + HosThread *r = thread_of_scan(guest_tt); + cache[i].tt = guest_tt, cache[i].t = r, cache[i].gen = g; + return r; +} Ctx *hos_boot_main_thread(uint64_t stack_size) { HosThread *t = (HosThread *)calloc(1, sizeof *t); @@ -502,6 +541,7 @@ Ctx *hos_boot_main_thread(uint64_t stack_size) { t->ideal_core = 0; t->started = 1; snprintf(t->name, sizeof t->name, "MainThread"); + __atomic_store_n(&t->serial, short_sleep_serial_name(t->name), __ATOMIC_RELAXED); DuplicateHandle(GetCurrentProcess(), GetCurrentThread(), GetCurrentProcess(), &t->handle, 0, FALSE, DUPLICATE_SAME_ACCESS); /* a real handle, for the profiler */ SetThreadDescription(GetCurrentThread(), L"MainThread"); /* (HOS_SAMPLE finds threads by it) */ @@ -604,6 +644,7 @@ void hos_thread_destroy(HosThread *t) { AcquireSRWLockExclusive(&threads_lock); for (int i = 0; i < nthreads; i++) if (threads[i] == t) { threads[i] = threads[--nthreads]; break; } + thread_gen_bump(); ReleaseSRWLockExclusive(&threads_lock); if (t->handle) CloseHandle(t->handle); if (t->wake) CloseHandle(t->wake); @@ -612,6 +653,7 @@ void hos_thread_destroy(HosThread *t) { void hos_thread_set_name(HosThread *t, const char *name) { snprintf(t->name, sizeof t->name, "%s", name); + __atomic_store_n(&t->serial, short_sleep_serial_name(t->name), __ATOMIC_RELAXED); if (t->handle) { wchar_t w[48]; MultiByteToWideChar(CP_UTF8, 0, t->name, -1, w, 48); @@ -680,7 +722,10 @@ static uint64_t pick_reserved_core(const uint64_t *masks, const unsigned char *c finds such a die; everywhere else 0, and nothing changes. HOS_X3D=0: off there too. Other threads of the process (the driver's, the shader workers) stay where Windows puts them. Windows only: Linux as before. NX2navi keeps the whole process on the die and reserves no core there; here the replay's core stays, on the die, as - the replay is pinned to it by default (gpu/vk/vk_replay.cpp ReplayPin). HOS_RESERVE_CORE=0 keeps none, as ever */ + the replay is pinned to it by default (gpu/vk/vk_replay.cpp ReplayPin). HOS_RESERVE_CORE=0 keeps none, as ever. + (Triforce, 11-22, the performance audit's THREAD-49, after Source A's) That holds where the die's cores have two + logical processors each. With SMT off no core of the die is kept unless HOS_RESERVE_CORE=1 asks for one or the replay + has a thread of its own (RX_VK_ASYNC=1): hos_x3d.h hos_x3d_keep_replay_core has the rule and Source A's numbers */ #if defined(_WIN32) /* (hos_x3d_pick over what Windows lists for the calling thread's processor group) */ static uint64_t x3d_detect(HosX3d *out) { @@ -738,7 +783,7 @@ static uint64_t hos_x3d_cpus(void) { (unsigned long long)x.die, (unsigned long long)(x.big >> 20), (unsigned long long)(x.small >> 20), x.cores, x.why == HOS_X3D_FEW ? "fewer than 8, so the game's threads are not kept there" : off ? "not used (HOS_X3D=0)" - : "the game's threads and the replay's core are kept there (HOS_X3D=0: as before)"); + : "the game's threads are kept there, and the replay's core when one is kept (HOS_X3D=0: as before)"); #endif } const uint64_t m = mask; @@ -746,6 +791,8 @@ static uint64_t hos_x3d_cpus(void) { return m; } +/* ((Triforce, 11-22, the performance audit's OS-49) the mask HOS_SIM_CPU holds the process to, 0: none (hos_sim_cpu.h) */ +static uint64_t sim_cpu_mask; #if defined(_WIN32) /* (pick_reserved_core over the calling thread's group's cores; *cores: how many, 0 when they could not be read) */ static uint64_t last_fastest_core(int *cores) { @@ -765,7 +812,8 @@ static uint64_t last_fastest_core(int *cores) { const SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *e = (const SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *)(buf + at); if (!e->Size) break; if (e->Relationship == RelationProcessorCore && e->Processor.GroupCount && e->Processor.GroupMask[0].Group == group && - (!die || ((uint64_t)e->Processor.GroupMask[0].Mask & die))) { + (!die || ((uint64_t)e->Processor.GroupMask[0].Mask & die)) && + (!sim_cpu_mask || ((uint64_t)e->Processor.GroupMask[0].Mask & sim_cpu_mask))) { /* ((Triforce, OS-49) inside it) */ masks[n] = (uint64_t)e->Processor.GroupMask[0].Mask; classes[n] = e->Processor.EfficiencyClass; n++; @@ -777,15 +825,72 @@ static uint64_t last_fastest_core(int *cores) { return pick_reserved_core(masks, classes, n); } #endif +int rx_async_replay(void); /* (host/rx_backend.c: the replay on a thread of its own, RX_VK_ASYNC=1) */ +/* ((Triforce, 11-22, the performance audit's OS-49, after Source B's) HOS_SIM_CPU=/, a lab switch: the process + held to that many of the PC's cores (hos_sim_cpu.h), from host/hos_main.c before any thread. Not applied (unset, a value that + cannot be read, as many cores as the PC has, or the call refused): the variable is cleared, so that the renderer's pools + (gpu/vk/cpu_count.h) size by the whole PC too. Windows only: elsewhere the variable is left and the renderer reads it there + as nothing */ +void hos_sim_cpu_apply(void) { +#if defined(_WIN32) + const char *e = getenv("HOS_SIM_CPU"); + if (!e || !*e) return; + HosSimCpu s; + uint64_t mask = 0; + int n = 0; + if (hos_sim_cpu_parse(e, &s)) { + static uint64_t masks[256]; + static unsigned char classes[256]; + GROUP_AFFINITY ga; + const WORD group = GetThreadGroupAffinity(GetCurrentThread(), &ga) ? ga.Group : 0; + DWORD len = 0; + if (!GetLogicalProcessorInformationEx(RelationProcessorCore, NULL, &len) && GetLastError() == ERROR_INSUFFICIENT_BUFFER) { + char *buf = (char *)malloc(len); + if (buf && GetLogicalProcessorInformationEx(RelationProcessorCore, (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *)buf, &len)) + for (DWORD at = 0; at < len && n < 256;) { + const SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *x = (const SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *)(buf + at); + if (!x->Size) break; + if (x->Relationship == RelationProcessorCore && x->Processor.GroupCount && x->Processor.GroupMask[0].Group == group) { + masks[n] = (uint64_t)x->Processor.GroupMask[0].Mask; + classes[n] = x->Processor.EfficiencyClass; + n++; + } + at += x->Size; + } + free(buf); + } + mask = hos_sim_cpu_mask(masks, classes, n, hos_x3d_cpus(), s.cores); + if (mask && !SetProcessAffinityMask(GetCurrentProcess(), (DWORD_PTR)mask)) mask = 0; + } + if (!mask) { + hos_log("cpu: HOS_SIM_CPU=%s not applied (/, fewer cores than the %d here): the whole PC, as without it", e, n); + SetEnvironmentVariableA("HOS_SIM_CPU", NULL); + _putenv("HOS_SIM_CPU="); + return; + } + sim_cpu_mask = mask; + hos_log("cpu: HOS_SIM_CPU=%s (a lab switch): the process held to logical processors %#llx, %d of the %d cores; the renderer's " + "pools sized for %d threads", e, (unsigned long long)mask, s.cores, n, s.threads); +#endif +} uint64_t hos_reserved_cpus(void) { static uint64_t mask = ~0ull; if (mask != ~0ull) return mask; mask = 0; - if (getenv("HOS_RESERVE_CORE") && !atoi(getenv("HOS_RESERVE_CORE"))) return mask; + const char *e = getenv("HOS_RESERVE_CORE"); + const int asked = e ? atoi(e) != 0 : -1; /* ((Triforce, THREAD-49) -1: not set) */ + if (!asked) return mask; #if defined(_WIN32) { int cores = 0; const uint64_t fast = last_fastest_core(&cores); + /* ((Triforce, 11-22, the performance audit's THREAD-49, after Source A's) on the cache die of a two-die X3D with + SMT off none is kept: hos_x3d.h. cores: the die's, where there is one) */ + const uint64_t die = hos_x3d_cpus(); + if (cores && fast && !hos_x3d_reserved(fast, die, cores, asked, rx_async_replay())) { + hos_log("cpu: no core kept for the replay on the cache die (HOS_RESERVE_CORE=1: one is)"); + return mask; + } if (cores) return mask = fast; } #endif @@ -853,19 +958,38 @@ int64_t hos_now_ns(void) { return (int64_t)((now.QuadPart / freq.QuadPart) * 1000000000LL + (now.QuadPart % freq.QuadPart) * 1000000000LL / freq.QuadPart); } -static void sleep_ns(int64_t ns); -/* (virtual cores: a sleep gives the core up, SleepThread(0) offers it to a waiter as good) */ -void hos_sleep_ns(int64_t ns) { +static void sleep_for(int64_t ns, int guest); +/* ((Triforce, 11-22, the performance audit's OS-33) the game threads yielding through a short sleep now, and the short + sleeps that took the timer because the cap's worth already were: short_sleep.h, HOS_SHORT_SLEEP_POLLERS) */ +static int short_pollers; +static int64_t short_capped; +/* ((Triforce, 11-21, the performance audit's OS-31) the logical processors this process may run on (its affinity mask, as + the game starts with it): Windows with one processor group; 0, not known (more than one group, where the mask is one + group's and not the process's, or elsewhere) */ +static int process_threads(void) { + int threads = 0; +#if defined(_WIN32) + DWORD_PTR proc = 0, sys = 0; + if (GetActiveProcessorGroupCount() == 1 && GetProcessAffinityMask(GetCurrentProcess(), &proc, &sys)) + threads = __builtin_popcountll((uint64_t)proc); +#endif + return threads; +} +/* (virtual cores: a sleep gives the core up, SleepThread(0) offers it to a waiter as good) guest: the game's own sleep */ +static void sleep_away(int64_t ns, int guest) { if (ns <= 0) { hos_vcore_yield(); SwitchToThread(); return; } const int away = hos_vcore_away(); - sleep_ns(ns); + sleep_for(ns, guest); if (away) hos_vcore_back(); } -static void sleep_ns(int64_t ns) { +void hos_sleep_ns(int64_t ns) { sleep_away(ns, 0); } +/* (the host's own sleep, as pace_wait_until takes it) */ +static void sleep_ns(int64_t ns) { sleep_for(ns, 0); } +static void sleep_for(int64_t ns, int guest) { /* A high-resolution waitable timer (one per thread) sleeps to within ~0.5 ms without spinning; Sleep() rounds to the timer resolution, so without one the last millisecond is spun (threads that sleep in short steps, the vibration thread for one, used 15% of a core spinning). */ @@ -893,13 +1017,57 @@ static void sleep_ns(int64_t ns) { return; #endif /* Under a millisecond: yielded away until the time (the timer's ~0.5 ms granularity made TOTK's 20 us waits for - its job workers 25x longer, ~2000 a second on the frame's critical path). HOS_SHORT_SLEEP_TIMER=1: as before */ - static int short_timer = -1; - if (short_timer < 0) short_timer = getenv("HOS_SHORT_SLEEP_TIMER") != NULL; - if (ns < 1000000 && !short_timer) { - while (hos_now_ns() < until) - if (!SwitchToThread()) YieldProcessor(); - return; + its job workers 25x longer, ~2000 a second on the frame's critical path). ((Triforce, 11-21, OS-31) Not the game's + own sleeps where the processors are few: short_sleep.h. HOS_SHORT_SLEEP_TIMER=1: the timer always; 0: never; not + set: by the processors on Windows. Chosen once. (Triforce, 11-22, OS-33) The same word holds the cap on the game + threads that yield at once, HOS_SHORT_SLEEP_POLLERS: 0, no limit, unless that switch asks for one) */ + static volatile LONG how_now = -1; + int word = (int)InterlockedCompareExchange(&how_now, -1, -1); + if (word < 0) { + const int threads = process_threads(); + /* ((Triforce, 11-22, the performance audit's OS-32, after Source A's) off Windows the processors are not counted: + with the switch not set, short_sleep.h's SHORT_SLEEP_ELSEWHERE (all yield, as before, until a Steam Deck A/B). + HOS_SHORT_SLEEP_TIMER=game: the game's own on the timer at any count, on any system) */ +#if defined(_WIN32) + const int windows = 1; +#else + const int windows = 0; +#endif + const int how = short_sleep_timer_os(getenv("HOS_SHORT_SLEEP_TIMER"), threads, windows, SHORT_SLEEP_ELSEWHERE); + const int cap = short_sleep_pollers(getenv("HOS_SHORT_SLEEP_POLLERS"), getenv("HOS_SHORT_SLEEP_TIMER"), how, threads); + word = short_sleep_word(how, cap); + if (InterlockedCompareExchange(&how_now, word, -1) == -1) { + hos_log("sleep: sleeps under 1 ms: %s (%d logical processors for this process%s; HOS_SHORT_SLEEP_TIMER=1: all on the " + "timer, game: the game's own on the timer, 0: all yield%s)", how == SHORT_SLEEP_ALL_TIMER ? "all on the timer" + : how == SHORT_SLEEP_GUEST_TIMER ? "the game's own on the timer, the host's yield" : "all yield until their time", + threads, threads ? "" : ": not known", + windows ? "" : SHORT_SLEEP_ELSEWHERE == SHORT_SLEEP_GUEST_TIMER ? "; not set, off Windows: the game's own on the timer" + : "; not set, off Windows: all yield"); + if (cap) + hos_log("sleep: at most %d game threads yield through a short sleep at once, the next waits on the timer; the " + "main and presentation threads always yield (HOS_SHORT_SLEEP_POLLERS)", cap); + } else word = (int)InterlockedCompareExchange(&how_now, -1, -1); /* (another thread chose first: its choice) */ + } + const int how = short_sleep_how(word); + if (short_sleep_yields(ns, how, guest)) { + /* ((Triforce, OS-33) the cap's gate: a counted sleeper over the cap takes the timer below, its count given back) */ + int took; + if (short_sleep_poll_enter(&short_pollers, short_sleep_cap(word), guest, cur ? hos_thread_serial(cur) : 0, &took)) { + /* TICK-062 B4, HOS_SHORT_SLEEP_PAUSE=1: one yield, then a PAUSE loop to the deadline (SwitchToThread in a loop + keeps calling into the kernel and bounces the SMT sibling's work) */ + static int pause_spin = -1; + if (pause_spin < 0) pause_spin = !(getenv("HOS_SHORT_SLEEP_PAUSE") && atoi(getenv("HOS_SHORT_SLEEP_PAUSE")) == 0); /* ((Triforce) on by default since 11-21, lab +5.8%; =0: off */ + if (pause_spin) { + if (hos_now_ns() < until) SwitchToThread(); + while (hos_now_ns() < until) { YieldProcessor(); YieldProcessor(); YieldProcessor(); YieldProcessor(); } + } else { + while (hos_now_ns() < until) + if (!SwitchToThread()) YieldProcessor(); + } + short_sleep_poll_leave(&short_pollers, took); + return; + } + __atomic_add_fetch(&short_capped, 1, __ATOMIC_RELAXED); } if (!tried) { tried = 1; @@ -988,13 +1156,56 @@ HLE(_ZN2nn2os20ChangeThreadPriorityEPNS0_10ThreadTypeEi) { RET((uint32_t)hos_thread_priority(thread_arg(c, ARG(0)), 1, (int)ARG(1))); } +/* (Triforce, 11-22, the performance audit's GAME-22, after Source B's) What the game asks for, in the log: a mask that + names two or more cores, which is what HOS_VCORE_WIDE=1 widens (vc_allowed), always; every mask when that switch is set + to anything or with the trace. Whether the switch can do anything in a game is read here: with no such line no thread + has a mask of more than one core and it changes nothing. The first 64 */ +static void core_mask_note(HosThread *t, int ideal, uint64_t mask) { + static volatile LONG logged; + static int every = -1; + if (every < 0) every = getenv("HOS_VCORE_WIDE") != NULL; + const int cores = hos_cores() > 8 ? 8 : hos_cores(); + const int named = __builtin_popcountll(mask & ((1ull << cores) - 1)); + if (named < 2 && !every && !hos_trace) return; + if (InterlockedIncrement(&logged) > 64) return; + hos_log("vcores: %s (priority %d) asks for ideal core %d and core mask %#llx, %d of the %d cores%s", t->name, t->priority, + ideal, (unsigned long long)mask, named, cores, named > 1 ? " (HOS_VCORE_WIDE=1: any core)" : ""); +} /* nn::os::SetThreadCoreMask(ThreadType*, int idealCore, unsigned long mask) */ HLE(_ZN2nn2os17SetThreadCoreMaskEPNS0_10ThreadTypeEim) { HosThread *t = thread_arg(c, ARG(0)); hos_thread_affinity(t, (int)ARG(1), ARG(2)); + core_mask_note(t, (int)ARG(1), ARG(2)); } -HLE(_ZN2nn2os20GetCurrentCoreNumberEv) { RET((uint32_t)hos_thread_core(cur)); } +/* TICK-062 A10, HOS_CORE_COUNT=1 (diagnostic, no behaviour change): per guest thread, the calls that ask for the core number + (nn::os::GetCurrentCoreNumber, svc 0x10) and the waits for a virtual core, logged every 5 s (cumulative) */ +int hos_core_count_on(void) { + static int on = -1; + if (on < 0) on = getenv("HOS_CORE_COUNT") != NULL; + return on; +} +void hos_core_count_svc(void) { if (cur) cur->cc_svc++; } +static void core_count_log(void) { + static volatile LONG64 last; + const int64_t now = hos_now_ns(); + const int64_t l = last; + if (now - l < 5000000000LL || InterlockedCompareExchange64(&last, now, l) != l) return; + if (!l) return; + AcquireSRWLockShared(&threads_lock); + for (int i = 0; i < nthreads; i++) { + const HosThread *t = threads[i]; + if (t->cc_hle + t->cc_svc + t->cc_vcw) + hos_log("corecount: %-24s core-number calls %lld (hle %lld, svc %lld), virtual-core waits %lld (%.1f ms)", t->name, + (long long)(t->cc_hle + t->cc_svc), (long long)t->cc_hle, (long long)t->cc_svc, (long long)t->cc_vcw, + t->cc_vcw_ns / 1e6); + } + ReleaseSRWLockShared(&threads_lock); +} +HLE(_ZN2nn2os20GetCurrentCoreNumberEv) { + if (hos_core_count_on()) { cur->cc_hle++; core_count_log(); } + RET((uint32_t)hos_thread_core(cur)); +} HLE(_ZN2nn2os26GetThreadAvailableCoreMaskEv) { RET((1u << hos_cores()) - 1); } /* cores 0-2: what an application gets */ HLE(_ZN2nn2os19GetCurrentStackInfoEPmS1_) { @@ -1002,9 +1213,28 @@ HLE(_ZN2nn2os19GetCurrentStackInfoEPmS1_) { if (ARG(1)) G64(ARG(1)) = cur->stack_size; } +/* (Triforce, 11-22, the performance audit's THREAD-41, after Source A's) The game's job system yields after every job + and actor callback, thousands of times a second. A yield is answered twice: hos_vcore_yield hands the virtual core to + a waiting game thread of equal or better priority, which is the console's yield, and SwitchToThread then gives the + processor up as well, a call into the kernel each time. Source A, with the game held to 4 processors, found half of + its six executors' CPU time inside that call and dropped it (mean frame 0.26 to 0.34 ms shorter there, nothing with + 16); Windows still takes the processor from a thread at the end of its time slice. + HOS_OS_YIELD=0 drops the call here. Off unless asked for: not set or 1, both answers as before. It changes how the + game's threads wait and was not measured here, and a thread that yields in a loop for one waiting on its processor + then keeps it to the end of its slice, the stall to watch with 2 to 4 processors. Without virtual cores (HOS_VCORES=0) + the processor's yield is the only one, so it stays whatever the switch says. SleepThread(0) (sleep_away) and the + short sleeps' yields are not this and keep theirs. Read once */ +static int os_yield(void) { + static int on = -1; + if (on < 0) { + const char *e = getenv("HOS_OS_YIELD"); + on = e && *e ? atoi(e) != 0 : 1; + } + return on; +} HLE(_ZN2nn2os11YieldThreadEv) { hos_vcore_yield(); - SwitchToThread(); + if (os_yield() || !hos_vcores_on()) SwitchToThread(); } /* HOS_SLEEP_LOG=1 (diagnostic): guest sleeps by caller (LR) every 5 s: count, asked, slept */ void hos_sleep_logged(int64_t ns, uint64_t lr) { @@ -1020,9 +1250,9 @@ void hos_sleep_logged(int64_t ns, uint64_t lr) { if (poll_cap && ns > poll_cap && ns <= 100000) ns = poll_cap; static int on = -1; if (on < 0) on = getenv("HOS_SLEEP_LOG") != NULL; - if (!on) { hos_sleep_ns(ns); return; } + if (!on) { sleep_away(ns, 1); return; } /* ((Triforce, 11-21, OS-31) the game's own sleep: short_sleep.h) */ const int64_t t0 = hos_now_ns(); - hos_sleep_ns(ns); + sleep_away(ns, 1); const int64_t took = hos_now_ns() - t0; static SRWLOCK lk = SRWLOCK_INIT; static struct { uint64_t lr; int64_t n, asked, slept; } tab[64]; @@ -1044,6 +1274,11 @@ void hos_sleep_logged(int64_t ns, uint64_t lr) { hos_log("sleep: lr %#llx: %lld calls, asked %.2f ms, slept %.2f ms (%.1f us each)", (unsigned long long)tab[k].lr, (long long)tab[k].n, tab[k].asked / 1e6, tab[k].slept / 1e6, tab[k].slept / 1e3 / tab[k].n); + /* ((Triforce, OS-33) the short sleeps that took the timer over the cap, HOS_SHORT_SLEEP_POLLERS) */ + const int64_t capped = __atomic_exchange_n(&short_capped, 0, __ATOMIC_RELAXED); + if (capped) + hos_log("sleep: %lld short sleeps took the timer, the cap's worth of game threads yielding already " + "(HOS_SHORT_SLEEP_POLLERS)", (long long)capped); memset(tab, 0, sizeof tab); } ReleaseSRWLockExclusive(&lk); @@ -1342,6 +1577,10 @@ static struct { sleep (NX2navi's "core grant to run", runtime/kernel.c grant_at) */ int64_t spun, slept, spin_ns, spin_capped, run_spin_ns, run_spin_n, run_sleep_ns, run_sleep_n, run_max_ns; } vc_st; +/* ((Triforce, 11-22, the performance audit's OS-51) the waits since the last present, beside vc_st's 5 s: their time summed + over the threads, how many, the longest. Added to with vc_lock held, taken by the present's look without it + (hos_vcore_waits_take; host/stall_host.h has the line) */ +static struct { int64_t ns, n, longest; } vc_gap; int hos_vcores_on(void) { static int on = -1; @@ -1364,7 +1603,17 @@ static unsigned vc_allowed(HosThread *t) { static int any = -1; if (any < 0) any = getenv("HOS_VCORE_ANY") && atoi(getenv("HOS_VCORE_ANY")) != 0; if (any) return all; + /* (Triforce, 11-22, the performance audit's GAME-22, after Source B's) HOS_VCORE_WIDE=1: a thread whose mask names two + or more cores takes any free core. Source B's reason: the game's two main threads may run on two of three cores + only, where per-core job workers of better priority preempt them while the other cores stand idle a third of the + time. A one-core mask, which is what guards the game's lock-free per-core data, stays as it is, and so does a + thread with no mask. Off unless set: a widened thread runs under core numbers the console never gives it (the + risk HOS_VCORE_ANY has for every thread), Source B never measured it, and neither was it here. Whether any thread + has such a mask is in the log (core_mask_note). Read once */ + static int wide = -1; + if (wide < 0) wide = getenv("HOS_VCORE_WIDE") && atoi(getenv("HOS_VCORE_WIDE")) != 0; unsigned m = (unsigned)(t->core_mask & all); + if (wide && __builtin_popcount(m) > 1) return all; if (!m) m = 1u << hos_thread_core(t); return m; } @@ -1449,7 +1698,8 @@ static int vc_spin_max(void) { const char *e = getenv("HOS_VCORE_SPIN_MAX"); SYSTEM_INFO si; GetSystemInfo(&si); - const int n = e ? atoi(e) : vc_spin_most((int)si.dwNumberOfProcessors); + /* ((Triforce, 11-22, OS-49) by the processors HOS_SIM_CPU leaves the process, when it holds it) */ + const int n = e ? atoi(e) : vc_spin_most(sim_cpu_mask ? hos_sim_cpu_count(sim_cpu_mask) : (int)si.dwNumberOfProcessors); most = n < 0 ? 0 : n; } return most; @@ -1568,6 +1818,8 @@ static void vc_acquire(HosThread *t) { const int64_t now = hos_now_ns(), waited = now - t0; vc_st.wait_ns += waited; if (waited > vc_st.max_ns) vc_st.max_ns = waited; + __atomic_add_fetch(&vc_gap.ns, waited, __ATOMIC_RELAXED), __atomic_add_fetch(&vc_gap.n, 1, __ATOMIC_RELAXED); /* ((Triforce, OS-51)) */ + if (waited > __atomic_load_n(&vc_gap.longest, __ATOMIC_RELAXED)) __atomic_store_n(&vc_gap.longest, waited, __ATOMIC_RELAXED); if (ended) vc_st.spun++; /* ((Triforce) every wait ends in the spin or after a sleep) */ else if (slept) vc_st.slept++; if (t->vc_grant_at) { /* ((Triforce) granted by a release: from the grant until it runs) */ @@ -1577,6 +1829,7 @@ static void vc_acquire(HosThread *t) { if (run > vc_st.run_max_ns) vc_st.run_max_ns = run; t->vc_grant_at = 0; } + t->cc_vcw++, t->cc_vcw_ns += waited; /* (HOS_CORE_COUNT) */ vc_stats_locked(now); ReleaseSRWLockExclusive(&vc_lock); } @@ -1623,6 +1876,13 @@ void hos_vcore_back(void) { HosThread *t = cur; if (hos_vcores_on() && t && !t->vc_held) vc_acquire(t); } +/* ((Triforce, 11-22, the performance audit's OS-51) the waits for a core since the last take: their summed time, how many, + the longest; all left at zero. No lock: a wait that ends meanwhile is in this take or the next) */ +void hos_vcore_waits_take(int64_t *ns, int64_t *count, int64_t *longest) { + *ns = __atomic_exchange_n(&vc_gap.ns, 0, __ATOMIC_RELAXED); + *count = __atomic_exchange_n(&vc_gap.n, 0, __ATOMIC_RELAXED); + *longest = __atomic_exchange_n(&vc_gap.longest, 0, __ATOMIC_RELAXED); +} /* Lifted code calls this from its interrupt polls with the loop's guest registers still in host registers (a64_poll_cur, mkw_glue.c: the poll hands over no context, so the compiler need not spill them). It, and everything it calls, must never read or write the thread's guest context (Ctx) or run guest code: what it would see there may be older than the loop. A @@ -1662,6 +1922,15 @@ void hos_vcore_yield(void) { HosThread *t = cur; if (hos_vcores_on() && t) vc_yield(t, 0); } +/* ((Triforce, 11-21) a waiter asked the calling thread for its core (vc_kick): nn_os_sync.c's spins before a sleep stop at once + then, where they spun on with the core held for up to HOS_SPIN_US (20 us) while the waiter could not run. HOS_SPIN_YIELD=0: + the spins never look, as before */ +int hos_vcore_asked(void) { + static int on = -1; + if (on < 0) on = !(getenv("HOS_SPIN_YIELD") && atoi(getenv("HOS_SPIN_YIELD")) == 0); + HosThread *t = cur; + return on && t && t->vc_held && t->vc_kick; +} /* SleepConditionVariableSRW for guest threads: the core given up for the wait; after it the lock is let go while a core is taken again (waiting for a core while holding it could stall the thread that will wake the next waiter), so callers see a spurious wake-up and look at their condition again, as they do anyway */ @@ -1679,6 +1948,7 @@ int hos_cv_sleep(void *cv, void *lk, unsigned ms) { int hos_vcores_on(void) { return 0; } int hos_vcore_away(void) { return 0; } void hos_vcore_back(void) {} +void hos_vcore_waits_take(int64_t *ns, int64_t *count, int64_t *longest) { *ns = *count = *longest = 0; } void hos_vcore_checkpoint(void) {} void hos_vcore_yield(void) {} int hos_cv_sleep(void *cv, void *lk, unsigned ms) { diff --git a/host/hos_x3d.h b/host/hos_x3d.h index 5be390f..bd02e79 100644 --- a/host/hos_x3d.h +++ b/host/hos_x3d.h @@ -5,6 +5,11 @@ none of them crosses dies. Windows' own placement of games on that die (Game Mode with AMD's chipset driver) depends on its taking the program for a game. + (Triforce, 11-22, the performance audit's THREAD-49, after Source A's) The replay's core is kept on the die only where + the die's cores have more than one logical processor each (SMT on). With SMT off a kept core left the game 7 of the + die's 8 cores: the replay ran faster and the frame slower (hos_x3d_keep_replay_core below). HOS_RESERVE_CORE=1 keeps + one there too, as before. + Nothing is done: - with every L3 cache the same size: one die (a 7800X3D), two dies alike (a 7950X, a part with the cache on both), an Intel processor of two kinds of core, whose cores differ in their L2 caches and share one L3; @@ -96,4 +101,24 @@ static inline uint64_t hos_x3d_thread_mask(uint64_t process, uint64_t die, uint6 if (on & ~reserved) return reserved || on != process ? on & ~reserved : 0; return on != process ? on : 0; } + +/* (Triforce, 11-22, the performance audit's THREAD-49, after Source A's) Whether a core of the die is kept for the replay. + die: hos_x3d_pick's die, 0 for none; cores: the cores in it; asked: HOS_RESERVE_CORE, -1 when it is not set, else 0 or + 1; own_thread: the replay has a thread of its own (RX_VK_ASYNC=1), which is pinned to the kept core. + Source A measured a 9950X3D with SMT off: with one of the die's 8 cores kept the replay took 9.8-10.0 ms a frame and + the game ran at 56.9-57.4 fps; with none kept, 10.1-10.5 ms and 59.8-59.9 fps. The game runs six cores' worth of + threads beside the replay's, and the eighth core was worth more to them than to the replay alone. With SMT on (16 + logical processors on the die's 8 cores) it was never measured, so the core stays kept there, as before; so it does + with no such die, where this rule changes nothing. The switch, when set, wins either way */ +static inline int hos_x3d_keep_replay_core(uint64_t die, int cores, int asked, int own_thread) { + if (asked >= 0) return asked != 0; + if (!die || cores <= 0 || own_thread) return 1; + return __builtin_popcountll(die) > cores; +} + +/* The core kept for the replay (hos_reserved_cpus): fast, the last core of the fastest kind among the die's cores when + there is a die and among all otherwise (pick_reserved_core's, 0 for none), unless the rule above keeps none */ +static inline uint64_t hos_x3d_reserved(uint64_t fast, uint64_t die, int cores, int asked, int own_thread) { + return hos_x3d_keep_replay_core(die, cores, asked, own_thread) ? fast : 0; +} #endif diff --git a/host/hos_zstd.c b/host/hos_zstd.c index d5447c5..39efd8c 100644 --- a/host/hos_zstd.c +++ b/host/hos_zstd.c @@ -3,12 +3,16 @@ calls the lifted code makes at the head of those functions (nx2/oot_patches.py patch_host_zstd). HOS_HOST_ZSTD, read once at the first call: - unset or 0 the game's own decoder, exactly as before (the calls answer 0 and its code runs; after the first call + unset or 1 the host decodes what host_zstd.h takes, the game's code the rest + 0 the game's own decoder, exactly as before (the calls answer 0 and its code runs; after the first call the lifted code no longer makes them) - 1 the host decodes what host_zstd.h takes, the game's code the rest 2 check: the game's code decodes everything and its results stand; the host decodes beside it and every - difference is logged ("zstd check: ... MISMATCH"). The log's "zstd:" lines count what was compared - Off by default: nobody has run it in Triforce's game. A wrong answer here damages every file the game loads. */ + difference is logged ("zstd check: ... MISMATCH", "wrote past the end"). The log's "zstd:" lines count + what was compared + ((Triforce, 11-21, the performance audit's IO-01, after Source A's) On by default since the host is given the + content's exact size to write into and check mode ran clean in the game: host_zstd.h hz_room. A wrong answer here + damages every file the game loads, so 0 stays, and 2 is the way to look. HOS_HOST_ZSTD_AWAY (0, 1, 2): which decodes + give their virtual core up, host_zstd.h hz_away_mode.) */ #include "hos.h" #define WIN32_LEAN_AND_MEAN #include @@ -37,8 +41,11 @@ static int mode(void) { LONG v = m; if (v < 0) { const char *e = getenv("HOS_HOST_ZSTD"); - v = e && *e ? atoi(e) : 0; + v = e && *e ? atoi(e) : 1; /* ((Triforce, 11-21, IO-01) not set: on */ v = v == 1 || v == 2 ? v : 0; + const char *a = getenv("HOS_HOST_ZSTD_AWAY"); + const int away = a && *a ? atoi(a) : 1; + hz_away_mode = away == 0 || away == 2 ? away : 1; /* (only the build the addresses are for: nx2/oot_patches.py puts the calls into no other, and this says so twice) */ if (v && strncmp(hos_main_build_id, "e0f3ccfa", 8)) { hos_log("zstd: HOS_HOST_ZSTD is for game build e0f3ccfa, this is %.8s: the game's own decoder", hos_main_build_id); @@ -50,8 +57,11 @@ static int mode(void) { } if (InterlockedCompareExchange(&m, v, -1) == -1 && v) hos_log(v == 2 ? "zstd: check mode (HOS_HOST_ZSTD=2): the game's decoder decodes, the host's beside it, differences are logged" - : "zstd: the game's files are decoded by the host's zstd %s (HOS_HOST_ZSTD=1; =2 checks it against the game's)", - ZSTD_versionString()); + : "zstd: the game's files are decoded by the host's zstd %s (HOS_HOST_ZSTD=0: the game's own decoder; =2 " + "checks the host's against it); whole buffers %s (HOS_HOST_ZSTD_AWAY)", + v == 2 ? "" : ZSTD_versionString(), + hz_away_mode == 1 ? "of 128 KB give their virtual core up, streams keep it" + : hz_away_mode == 2 ? "and stream feeds of 64 KB give their virtual core up" : "and streams keep their virtual core"); v = m; hos_host_zstd = (int)v; } diff --git a/host/host_alloc.h b/host/host_alloc.h new file mode 100644 index 0000000..3a1a670 --- /dev/null +++ b/host/host_alloc.h @@ -0,0 +1,48 @@ +/* (Triforce, 11-22, the performance audit's OS-04, after Source A's) The wait at Windows' commit limit. + + When RAM and the page file are full Windows refuses a commit (error 1455), and a refusal often ends in moments: a + page file set to grow grows, another program lets memory go, the renderer gives its textures' commit back. The + game's own commits already wait for that (mkw_glue.c commit_wait, since the heap is committed as it is touched). The + host's own tables did not: a malloc that failed there ended the game at once. Now those allocations wait the same + way, tried again every 50 ms for up to 5 s, and only then end the game as before, after the out-of-memory notice. + The renderer's failed `new` waits too (gpu/vk/new_wait.h). + + HOS_COMMIT_WAIT_MS=: how long all three wait, 5000 unless set, 60000 at most; 0: none, a refusal fails at once. + + The numbers and the loop alone, no system calls: tests/test_commit_limit.py compiles them. */ +#ifndef HOST_ALLOC_H +#define HOST_ALLOC_H +#include +#include + +enum { HOST_ALLOC_STEP_MS = 50, HOST_ALLOC_WAIT_MS = 5000, HOST_ALLOC_WAIT_MOST_MS = 60000 }; + +/* HOS_COMMIT_WAIT_MS's value in ms: not set (or empty) 5000; a number that many, 0 and less none, 60000 at most */ +static inline int host_alloc_wait_ms(const char *env) { + if (!env || !*env) return HOST_ALLOC_WAIT_MS; + const long v = atol(env); + return v <= 0 ? 0 : v > HOST_ALLOC_WAIT_MOST_MS ? HOST_ALLOC_WAIT_MOST_MS : (int)v; +} +/* How often a refusal is tried again: once a step for the wait (5000 ms: 100 times; under a step: never) */ +static inline int host_alloc_tries(int wait_ms) { return wait_ms > 0 ? wait_ms / HOST_ALLOC_STEP_MS : 0; } + +/* After a first refusal: a step slept and try_again(ctx) called, up to `tries` times, until it gives something. What it + gave (NULL: still refused), and in *waited_ms how long that took */ +static inline void *host_alloc_retry(void *(*try_again)(void *ctx), void *ctx, int tries, void (*sleep_ms)(int ms), int *waited_ms) { + void *p = NULL; + *waited_ms = 0; + for (int i = 0; i < tries && !p; i++) { + sleep_ms(HOST_ALLOC_STEP_MS); + *waited_ms += HOST_ALLOC_STEP_MS; + p = try_again(ctx); + } + return p; +} + +/* malloc, calloc and realloc that wait at the limit before they answer NULL (mkw_glue.c); a size of 0 is answered as the + C library answers it. The caller still ends the game on NULL, as before */ +void *hos_xmalloc(size_t n); +void *hos_xcalloc(size_t count, size_t size); +void *hos_xrealloc(void *p, size_t n); +int hos_commit_wait_ms(void); /* (HOS_COMMIT_WAIT_MS, read once) */ +#endif diff --git a/host/host_zstd.h b/host/host_zstd.h index 4a1e9c4..3f66d40 100644 --- a/host/host_zstd.h +++ b/host/host_zstd.h @@ -66,7 +66,23 @@ #include "../third_party/zstd/lib/zstd.h" #include "../third_party/zstd/lib/zstd_errors.h" -#define HZ_AWAY_BYTES (64u << 10) /* inputs from this size give the virtual core up while they decode */ +#define HZ_AWAY_BYTES (64u << 10) /* inputs from this size give the virtual core up while they decode (hz_away_mode 2) */ +/* ((Triforce, 11-21, the performance audit's IO-01, after Source A's) Who gives the game's virtual core up for a decode: + HOS_HOST_ZSTD_AWAY, set by the includer before the first call. + 1 (the default) a whole buffer whose input or output (the content's size where the frame says it, else the + caller's room) is HZ_AWAY_WHOLE or more; a stream's feed never. A feed is a + chunk the loading thread just read, a millisecond or two of decoding, and taking a core again + after each one cost more than it gave: Source A's loads froze in 7 of 10 runs on a 22-thread + laptop with the core given up there, in none of 3 with it kept; + 0 never; + 2 as before this: whole buffers and feeds alike from HZ_AWAY_BYTES of input */ +#define HZ_AWAY_WHOLE (128u << 10) +static int hz_away_mode = 1; +static inline int hz_away_whole(uint64_t size, uint64_t cap) { + return hz_away_mode == 2 ? size >= HZ_AWAY_BYTES : hz_away_mode == 1 ? size >= HZ_AWAY_WHOLE || cap >= HZ_AWAY_WHOLE : 0; +} +static inline int hz_away_feed(uint64_t size) { return hz_away_mode == 2 && size >= HZ_AWAY_BYTES; } +#define HZ_CANARY 64 /* (check mode: bytes after the host's own output that must stay as they were set) */ #define HZ_MAX_DICT (64u << 20) #define HZ_NDICT 32 #define HZ_NSTREAM 16 @@ -90,6 +106,7 @@ static _Thread_local uint64_t hz_caller; static struct { uint64_t calls, in, out, ns, game, inplace, away, checked, differ; /* one-shot */ uint64_t streams, feeds, s_in, s_out, s_ns, s_game, s_checked, s_differ; + uint64_t tail; /* ((Triforce, IO-01) check mode: decodes that wrote past their content's end) */ } hz_stats; static inline uint32_t hz_ld32(uint64_t a) { uint32_t v; memcpy(&v, HZ_PTR(a), 4); return v; } @@ -210,7 +227,7 @@ static size_t hz_decode(void *out, uint64_t cap, uint64_t src, uint64_t size, ui int slot; ZSTD_DCtx *d = hz_dctx_take(&slot); if (!d) return 0; - const int away = size >= HZ_AWAY_BYTES ? HZ_AWAY() : 0; + const int away = hz_away_whole(size, cap) ? HZ_AWAY() : 0; if (away) hz_add(&hz_stats.away, 1); *ok = 1; const size_t r = dd ? ZSTD_decompress_usingDDict(d, out, (size_t)cap, HZ_PTR(src), (size_t)size, dd) @@ -223,12 +240,12 @@ static size_t hz_decode(void *out, uint64_t cap, uint64_t src, uint64_t size, ui static void hz_count(uint64_t in, uint64_t out, uint64_t ns) { const uint64_t n = __atomic_add_fetch(&hz_stats.calls, 1, __ATOMIC_RELAXED); hz_add(&hz_stats.in, in), hz_add(&hz_stats.out, out), hz_add(&hz_stats.ns, ns); - if (n == 1 || !(n & 0x3fff)) + if (!(n & (n - 1)) || !(n & 0x3fff)) /* ((Triforce, IO-01) the 1st, 2nd, 4th, ... and every 16384th) */ HZ_LOG("zstd: %llu calls decoded on the host, %.1f MB in, %.1f MB out, %.0f ms; %llu left to the game's decoder (%llu of " - "them in place), %llu with the core given up; check: %llu compared, %llu differ", (unsigned long long)n, - hz_stats.in / 1048576.0, hz_stats.out / 1048576.0, hz_stats.ns / 1e6, (unsigned long long)hz_stats.game, - (unsigned long long)hz_stats.inplace, (unsigned long long)hz_stats.away, (unsigned long long)hz_stats.checked, - (unsigned long long)hz_stats.differ); + "them in place), %llu with the core given up; check: %llu compared, %llu differ, %llu wrote past the end", + (unsigned long long)n, hz_stats.in / 1048576.0, hz_stats.out / 1048576.0, hz_stats.ns / 1e6, + (unsigned long long)hz_stats.game, (unsigned long long)hz_stats.inplace, (unsigned long long)hz_stats.away, + (unsigned long long)hz_stats.checked, (unsigned long long)hz_stats.differ, (unsigned long long)hz_stats.tail); } /* (the input inside the output: the result depends on how the decoder's writes overtake its reads, which libzstd does @@ -238,6 +255,21 @@ static int hz_in_place(uint64_t dst, uint64_t cap, uint64_t src, uint64_t size) return cap && src < dst + cap && dst < src + size; } +/* ((Triforce, 11-21, the performance audit's IO-01, after Source A's) The room libzstd is given for a whole buffer: the + content's size when the frames say it and it fits, not the caller's whole room. libzstd may park a block's literals in + spare room past where the block ends (zstd_decompress_block.c, the "split literals" buffer); the game's decoder, zstd + 1.4, never writes past the content, and what lies in the game's memory after it is not the decoder's to touch. *exact: + the room is the content's. Else (a frame without its content size, or a size that does not fit: libzstd then refuses + and the game's decoder decides) the caller's room, all of which may have been written */ +static uint64_t hz_room(uint64_t cap, uint64_t src, uint64_t size, int *exact) { + *exact = 0; + if (size < 5 || !src || !HZ_OK(src, size)) return cap; + const unsigned long long n = ZSTD_findDecompressedSize(HZ_PTR(src), (size_t)size); + if (n == ZSTD_CONTENTSIZE_UNKNOWN || n == ZSTD_CONTENTSIZE_ERROR || n > cap) return cap; + *exact = 1; + return (uint64_t)n; +} + /* A call of the game's ZSTD_decompressMultiFrame. 1: decoded here, *ret its result (the decoded size). 0: the game's own function is to run, with its arguments as they are */ static int hz_oneshot(uint64_t dst, uint64_t cap, uint64_t src, uint64_t size, uint64_t dict, uint64_t dict_size, @@ -246,9 +278,10 @@ static int hz_oneshot(uint64_t dst, uint64_t cap, uint64_t src, uint64_t size, u hz_add(&hz_stats.inplace, 1), hz_add(&hz_stats.game, 1); return 0; } - int ok = 0; + int ok = 0, exact = 0; const uint64_t t0 = (uint64_t)HZ_NOW_NS(); - const size_t r = (cap && !dst) || !HZ_OK(dst, cap) ? 0 : hz_decode(HZ_PTR(dst), cap, src, size, dict, dict_size, ddict, &ok); + const uint64_t room = hz_room(cap, src, size, &exact); /* ((Triforce, IO-01) the content's size, when known) */ + const size_t r = (cap && !dst) || !HZ_OK(dst, cap) ? 0 : hz_decode(HZ_PTR(dst), room, src, size, dict, dict_size, ddict, &ok); if (!ok || ZSTD_isError(r)) { /* (the game's decoder decides, and gives its own error code) */ static int logged; if (__atomic_fetch_add(&logged, 1, __ATOMIC_RELAXED) < 8) @@ -257,7 +290,8 @@ static int hz_oneshot(uint64_t dst, uint64_t cap, uint64_t src, uint64_t size, u hz_add(&hz_stats.game, 1); return 0; } - if (r) HZ_WRITTEN(dst, r); + /* ((Triforce, IO-01) a room that was not the content's own: all of it may have been written, and is told so) */ + if (exact ? r != 0 : cap != 0) HZ_WRITTEN(dst, exact ? r : cap); *ret = r; hz_count(size, r, (uint64_t)HZ_NOW_NS() - t0); return 1; @@ -271,10 +305,14 @@ static uint64_t hz_oneshot_check(uint64_t dst, uint64_t cap, uint64_t src, uint6 hz_add(&hz_stats.inplace, 1), hz_add(&hz_stats.game, 1); return game(user); } - uint8_t *mine = cap ? (uint8_t *)malloc((size_t)cap) : NULL; - int ok = 0; + /* ((Triforce, IO-01) the host's output with the room hz_oneshot would give it, every byte set first: what is not the + content must come back as it was, or the decode wrote past its content's end (counted, "wrote past the end") */ + uint8_t *mine = cap ? (uint8_t *)malloc((size_t)cap + HZ_CANARY) : NULL; + int ok = 0, exact = 0; + const uint64_t room = hz_room(cap, src, size, &exact); + if (mine) memset(mine, 0xA5, (size_t)cap + HZ_CANARY); const uint64_t t0 = (uint64_t)HZ_NOW_NS(); - const size_t r = cap && !mine ? 0 : hz_decode(mine, cap, src, size, dict, dict_size, ddict, &ok); + const size_t r = cap && !mine ? 0 : hz_decode(mine, room, src, size, dict, dict_size, ddict, &ok); const uint64_t ns = (uint64_t)HZ_NOW_NS() - t0; const uint64_t g = game(user); const int host_ok = ok && !ZSTD_isError(r); @@ -287,6 +325,17 @@ static uint64_t hz_oneshot_check(uint64_t dst, uint64_t cap, uint64_t src, uint6 !ok ? "inputs not taken" : ZSTD_getErrorName(r)); } else { hz_add(&hz_stats.checked, 1); + for (uint64_t i = r; mine && i < cap + HZ_CANARY; i++) + if (mine[i] != 0xA5) { + hz_add(&hz_stats.tail, 1); + static int told; + if (__atomic_fetch_add(&told, 1, __ATOMIC_RELAXED) < 8) + HZ_LOG("zstd check: the host wrote past the end: %llu bytes of content, byte %llu of a room of %llu changed " + "(%s; called from %#llx)", (unsigned long long)r, (unsigned long long)i, (unsigned long long)cap, + exact ? "the room was the content's: libzstd wrote past its capacity" : "no content size in the frame", + (unsigned long long)hz_caller); + break; + } if (g != r || (r && (!HZ_OK(dst, r) || memcmp(mine, HZ_PTR(dst), r)))) { hz_add(&hz_stats.differ, 1); static int logged; @@ -394,7 +443,7 @@ static ZSTD_DCtx *hz_stream_dctx(void) { /* Feeds in to the stream's context until the input is used up, the frame ends (0), an error, or nothing moves. With the virtual core given up for large chunks, as hz_decode */ static size_t hz_stream_run(HzStream *st, ZSTD_outBuffer *out, ZSTD_inBuffer *in) { - const int away = in->size >= HZ_AWAY_BYTES ? HZ_AWAY() : 0; + const int away = hz_away_feed(in->size) ? HZ_AWAY() : 0; /* ((Triforce, IO-01) a feed keeps its core: hz_away_mode) */ if (away) hz_add(&hz_stats.away, 1); size_t r = 1; while (in->pos < in->size) { @@ -409,11 +458,12 @@ static size_t hz_stream_run(HzStream *st, ZSTD_outBuffer *out, ZSTD_inBuffer *in static void hz_stream_count(const HzStream *st, uint64_t out) { const uint64_t n = __atomic_add_fetch(&hz_stats.streams, 1, __ATOMIC_RELAXED); hz_add(&hz_stats.s_in, st->in), hz_add(&hz_stats.s_out, out); - if (n == 1 || !(n & 0x3ff)) + if (!(n & (n - 1)) || !(n & 0x3ff)) /* ((Triforce, IO-01) the 1st, 2nd, 4th, ... and every 1024th) */ HZ_LOG("zstd: %llu streams decoded on the host in %llu feeds, %.1f MB in, %.1f MB out, %.0f ms; %llu streams left to the " - "game's decoder; check: %llu compared, %llu differ", (unsigned long long)n, (unsigned long long)hz_stats.feeds, - hz_stats.s_in / 1048576.0, hz_stats.s_out / 1048576.0, hz_stats.s_ns / 1e6, (unsigned long long)hz_stats.s_game, - (unsigned long long)hz_stats.s_checked, (unsigned long long)hz_stats.s_differ); + "game's decoder; check: %llu compared, %llu differ, %llu wrote past the end", (unsigned long long)n, + (unsigned long long)hz_stats.feeds, hz_stats.s_in / 1048576.0, hz_stats.s_out / 1048576.0, hz_stats.s_ns / 1e6, + (unsigned long long)hz_stats.s_game, (unsigned long long)hz_stats.s_checked, (unsigned long long)hz_stats.s_differ, + (unsigned long long)hz_stats.tail); } /* A stream at its begin (phase 1): its slot, the host context ready for the frame, or NULL when the game's feed is to @@ -441,10 +491,11 @@ static HzStream *hz_stream_begin(uint64_t s, uint64_t src, int32_t size, uint64_ if ((!st->dctx && !(st->dctx = hz_stream_dctx())) || ZSTD_isError(ZSTD_DCtx_reset(st->dctx, ZSTD_reset_session_only)) || ZSTD_isError(ZSTD_DCtx_refDDict(st->dctx, dd)) || - (check && !(st->shadow = (uint8_t *)malloc((size_t)st->content)))) { + (check && !(st->shadow = (uint8_t *)malloc((size_t)st->content + HZ_CANARY)))) { hz_stream_drop(st); return NULL; } + if (check) memset(st->shadow + st->content, 0xA5, HZ_CANARY); /* ((Triforce, IO-01) what must stay: hz_stream_check) */ if (!check) { /* what the game's begin leaves in its state */ hz_st32(s + 0x14, (uint32_t)st->content); hz_st64(s + 0x30, ws); @@ -474,7 +525,9 @@ static uint64_t hz_stream_feed(HzStream *st, uint64_t s, uint64_t src, int32_t s return 0xffffffffu; } ZSTD_inBuffer in = {HZ_PTR(src), (size_t)size, 0}; - ZSTD_outBuffer out = {HZ_PTR(st->dst0), (size_t)st->cap0, (size_t)st->pos}; + /* ((Triforce, IO-01) the content's own size, not the game's whole room: libzstd may park literals in room past a + block's end, where the game's decoder never writes. The begin took the stream only with room for the content) */ + ZSTD_outBuffer out = {HZ_PTR(st->dst0), (size_t)st->content, (size_t)st->pos}; const size_t r = hz_stream_run(st, &out, &in); const uint32_t wrote = (uint32_t)(out.pos - st->pos); st->pos = out.pos, st->in += in.pos; @@ -545,6 +598,15 @@ static uint64_t hz_stream_check(HzStream *st, uint64_t s, uint64_t src, int32_t const int out_all = ph == 2 && !g; /* the job feeds no more, with or without the frame's last bytes */ if (ph != 3 && ph != 4 && !out_all) return g; hz_add(&hz_stats.s_checked, 1); + for (int i = 0; i < HZ_CANARY; i++) /* ((Triforce, IO-01) the bytes after the host's own output: as they were set) */ + if (st->shadow[st->content + i] != 0xA5) { + hz_add(&hz_stats.tail, 1); + static int told; + if (__atomic_fetch_add(&told, 1, __ATOMIC_RELAXED) < 8) + HZ_LOG("zstd check: a host stream wrote past the end of its %llu bytes of content (called from %#llx)", + (unsigned long long)st->content, (unsigned long long)hz_caller); + break; + } const int same = ph == 3 ? st->bad || st->done != 1 /* the game failed: the host must not have a clean frame */ : !st->bad && !st->moved && st->done != 2 && (ph != 4 || st->done == 1) && st->pos == st->content && HZ_OK(st->dst0, st->content) && diff --git a/host/menu_sound.c b/host/menu_sound.c index d2ef8df..005c573 100644 --- a/host/menu_sound.c +++ b/host/menu_sound.c @@ -154,14 +154,33 @@ static float *to_mix_rate(float *pcm, uint32_t *frames, uint32_t rate) { return out; } -float *hos_menu_sound_decode(const uint8_t *b, uint64_t size, uint32_t *frames_out) { +/* ((Triforce, 11-20) the samples of a channel in the whole DSP-ADPCM frames the file holds, `frames` at most. A tune's last + frame may be cut short at the file's end: the shop theme's second channel ends 4 bytes into its last frame of 8 (the 5 + samples it still counts are there, the frame is not), which decode_channel refuses, as it must for an interface sound. + The tune is cut to the whole frames: 5 samples of a minute) */ +static uint32_t frames_held(const uint8_t *info, uint64_t size, uint32_t frames) { + const uint16_t codec = u16(info); + const uint32_t off = u32(info + 0x30); + if (off >= size) return 0; + const uint64_t have = size - off; + uint64_t n = 0; + if (codec == 0) n = have / 2; + else if (codec == 1) n = have / 8 * 14; + return n < frames ? (uint32_t)n : frames; +} + +/* (the decoder, for a file of at most `seconds`: the interface sounds' ten, the Mods screen's tune's more) */ +static float *decode_most(const uint8_t *b, uint64_t size, uint32_t *frames_out, uint32_t seconds) { if (size < 0x10 || memcmp(b, "BWAV", 4)) return NULL; const uint16_t nch = u16(b + 0x0e); if (!nch || nch > 2 || 0x10 + (uint64_t)nch * 0x4c > size) return NULL; const uint32_t rate = u32(b + 0x10 + 4); uint32_t frames = u32(b + 0x10 + 8); /* (the interface sounds are 48 kHz and short: at most ten seconds of a rate a sound file has) */ - if (rate < 8000 || rate > 192000 || !frames || frames > rate * 10) return NULL; + if (rate < 8000 || rate > 192000 || !frames || frames > (uint64_t)rate * seconds) return NULL; + if (seconds > 10) /* (a tune: as long as the samples its channels hold, the last frame's cut-off ones left out) */ + for (uint16_t c = 0; c < nch; c++) frames = frames_held(b + 0x10 + c * 0x4c, size, frames); + if (!frames) return NULL; float *pcm = (float *)calloc((size_t)frames * 2, sizeof *pcm); int ok = pcm != NULL; for (uint16_t c = 0; ok && c < nch; c++) ok = decode_channel(b, size, b + 0x10 + c * 0x4c, pcm, frames); @@ -176,6 +195,12 @@ float *hos_menu_sound_decode(const uint8_t *b, uint64_t size, uint32_t *frames_o return pcm; } +float *hos_menu_sound_decode(const uint8_t *b, uint64_t size, uint32_t *frames_out) { return decode_most(b, size, frames_out, 10); } +/* ((Triforce, 11-20) a tune: the Mods screen's music, host/mods_music.c) */ +float *hos_menu_sound_decode_long(const uint8_t *b, uint64_t size, uint32_t *frames_out, uint32_t seconds) { + return decode_most(b, size, frames_out, seconds); +} + static void load_all(void) { /* (once: hos_menu_sound_frame's store of 1 into loaded starts it) */ int n = 0; for (int id = 1; id < SOUNDS; id++) { diff --git a/host/menu_sound.h b/host/menu_sound.h index 649abed..ba6e54c 100644 --- a/host/menu_sound.h +++ b/host/menu_sound.h @@ -9,6 +9,12 @@ extern "C" { /* A BWAV file's samples as interleaved stereo floats at 48 kHz, *frames of them (freed by the caller); NULL when the bytes are not a file this plays. Nothing outside [file, file + size) is read */ float *hos_menu_sound_decode(const uint8_t *file, uint64_t size, uint32_t *frames); +/* ((Triforce, 11-20) the same for a file of up to `seconds`: a tune (host/mods_music.c, the Mods screen's music) */ +float *hos_menu_sound_decode_long(const uint8_t *file, uint64_t size, uint32_t *frames, uint32_t seconds); +/* The Mods screen's music (host/mods_music.c), at each block the sound thread mixes, after the game's own mix and its + limiter and before the options' sounds: while the Mods screen is open from the title the game's nch channels are + turned down and the game's shop theme is added to the front pair, crossing over in 0.6 s each way */ +void hos_mods_music_frame(float *const *channels, int nch, uint32_t frames); /* The Menu sounds row's percent as a gain: 0 at 0 and under, 1 at 100 and over */ float hos_menu_sound_gain(int percent); /* Starts the sound of a HOS_MENU_SOUND_* (gpu/vk/vk_api.h) on a free voice, else on the oldest. No lock is taken: it diff --git a/host/mkw_glue.c b/host/mkw_glue.c index 10df91c..83b7ddf 100644 --- a/host/mkw_glue.c +++ b/host/mkw_glue.c @@ -2,6 +2,9 @@ commit (TOTK commits its whole 4 GB window up front; MKW's 512 GB window is only reserved), and stand-ins for the hooks of files not taken into host/ (the renderer and display, the profiler, mods, TOTK's own patches). */ #include "hos.h" +#include "heap_commit.h" /* ((Triforce, 11-21, MEM-09) the heap committed as it is touched: its rules) */ +#include "stall_host.h" /* ((Triforce, 11-22, OS-51) the host's part of a long frame, counted where it is spent) */ +#include "host_alloc.h" /* ((Triforce, 11-22, OS-04) the wait at Windows' commit limit: its length, and the host's own allocations) */ #include #include #define WIN32_LEAN_AND_MEAN @@ -15,6 +18,83 @@ The runtime commits what it hands out (image, system region, heap, address regions: hos_commit), because host APIs given guest buffers (ReadFile into a game buffer) fail on reserved pages without raising an exception. A guest or host access to a page nothing committed is committed on first touch, 64 KB at a time, by the handler. */ +/* ((Triforce, 11-21, the performance audit's MEM-09, after Source A's) A commit refused at Windows' commit limit (1455: RAM + and page file full) is tried again for up to 5 s, 100 times 50 ms apart: the renderer gives its textures' commit back + when the system's runs low (vk_texture.cpp, RX_VK_TEX_COMMIT_FREE_MB), and a page file set to grow does. Before, the + commit failed at once and the game's next write into those pages ended it. With the heap committed as it is touched + (heap_commit.h) a refusal can come at any first touch. 1: committed after waiting; 0: not */ +/* ((Triforce, 11-22, the performance audit's OS-04) HOS_COMMIT_WAIT_MS, read once: how long a refusal at the commit limit + is waited out, here and in hos_xmalloc below: 5000 ms unless set, 0 for no wait at all (host_alloc.h) */ +int hos_commit_wait_ms(void) { + static int ms = -1; + if (ms < 0) ms = host_alloc_wait_ms(getenv("HOS_COMMIT_WAIT_MS")); + return ms; +} +static int commit_wait(void *p, SIZE_T size) { + static volatile LONG made, given_up; + const int tries = host_alloc_tries(hos_commit_wait_ms()); /* ((Triforce, OS-04) 100 for the 5 s; 0: the refusal stands at once) */ + for (int i = 0; i < tries; i++) { + Sleep(50); + if (VirtualAlloc(p, size, MEM_COMMIT, PAGE_READWRITE)) { + const LONG n = InterlockedIncrement(&made); + if (n <= 8 || (n & 255) == 0) + hos_log("mem: commit of %#llx bytes refused at the commit limit, made after %d ms (%ld so far)", + (unsigned long long)size, (i + 1) * 50, (long)n); + return 1; + } + if (GetLastError() != ERROR_COMMITMENT_LIMIT) return 0; + } + if (tries && InterlockedIncrement(&given_up) <= 8) + hos_log("mem: commit of %#llx bytes still refused after %d ms at the commit limit", (unsigned long long)size, + tries * HOST_ALLOC_STEP_MS); + return 0; +} +/* ((Triforce, 11-22, the performance audit's OS-04, after Source A's) The host's own tables at the commit limit: malloc, + calloc and realloc that are tried again every 50 ms for up to HOS_COMMIT_WAIT_MS before they answer NULL, on which + their callers end the game as before (host_alloc.h has why). The thread keeps its virtual core while it waits: some of + the callers hold a lock of the host's, and a thread that queued for its core again under one could wait on a thread + that holds a core and waits for that lock. The first 8 rescues and the first 8 refusals are logged, and a refusal + that stands brings the out-of-memory notice, once */ +typedef struct { int kind; void *p; size_t count, n; } HostAllocAsk; +static void *host_alloc_try(void *ctx) { + const HostAllocAsk *a = (const HostAllocAsk *)ctx; + return a->kind == 0 ? malloc(a->n) : a->kind == 1 ? calloc(a->count, a->n) : realloc(a->p, a->n); +} +static void host_alloc_sleep(int ms) { Sleep((DWORD)ms); } +DWORD WINAPI hos_out_of_memory_box(LPVOID); +static void *host_alloc(HostAllocAsk *a) { + void *p = host_alloc_try(a); + if (p || !a->n || (a->kind == 1 && !a->count)) return p; /* (a size of 0: the C library's own answer) */ + static volatile LONG made, given_up, said; + int waited = 0; + p = host_alloc_retry(host_alloc_try, a, host_alloc_tries(hos_commit_wait_ms()), host_alloc_sleep, &waited); + const unsigned long long bytes = (unsigned long long)a->n * (a->kind == 1 ? a->count : 1); + if (p) { + const LONG n = InterlockedIncrement(&made); + if (n <= 8 || (n & 255) == 0) + hos_log("mem: a host allocation of %llu bytes refused, made after %d ms (%ld so far)", bytes, waited, (long)n); + return p; + } + if (InterlockedIncrement(&given_up) <= 8) + hos_log("mem: a host allocation of %llu bytes still refused after %d ms", bytes, waited); + if (InterlockedExchange(&said, 1) == 0) { + hos_log("mem: Windows is out of memory (RAM and page file): close other programs, or let Windows manage the page file size"); + CloseHandle(CreateThread(NULL, 0, hos_out_of_memory_box, NULL, 0, NULL)); + } + return NULL; +} +void *hos_xmalloc(size_t n) { + HostAllocAsk a = {0, NULL, 1, n}; + return host_alloc(&a); +} +void *hos_xcalloc(size_t count, size_t size) { + HostAllocAsk a = {1, NULL, count, size}; + return host_alloc(&a); +} +void *hos_xrealloc(void *p, size_t n) { + HostAllocAsk a = {2, p, 1, n}; + return host_alloc(&a); +} void *hos_mem_commit_begin(uint64_t addr, uint64_t size); /* ((Triforce) hos_mem.c: the census's notes around the commit) */ void hos_mem_commit_end(void *had, uint64_t addr, uint64_t size, int committed); int hos_commit(uint64_t addr, uint64_t size) { @@ -40,8 +120,10 @@ int hos_commit(uint64_t addr, uint64_t size) { void *const census_had = hos_mem_commit_begin(lo, hi - lo); /* (Triforce: a commit of committed pages resets their protection, the lazy write-back's too: rx_backend.c) */ const int io = hos_guest_io_begin(GP(lo), hi - lo); - const int committed = VirtualAlloc(GP(lo), (SIZE_T)(hi - lo), MEM_COMMIT, PAGE_READWRITE) != NULL; - const DWORD commit_err = committed ? 0 : GetLastError(); + int committed = VirtualAlloc(GP(lo), (SIZE_T)(hi - lo), MEM_COMMIT, PAGE_READWRITE) != NULL; + DWORD commit_err = committed ? 0 : GetLastError(); + /* ((Triforce, 11-21, MEM-09) refused at the commit limit: tried again for up to 5 s, commit_wait above) */ + if (!committed && commit_err == ERROR_COMMITMENT_LIMIT && commit_wait(GP(lo), (SIZE_T)(hi - lo))) committed = 1, commit_err = 0; hos_guest_io_end(io); hos_mem_commit_end(census_had, lo, hi - lo, committed); if (!committed) { @@ -61,6 +143,20 @@ int hos_commit(uint64_t addr, uint64_t size) { return 1; } +/* ((Triforce, 11-21, MEM-09) Guest memory about to go to the kernel (a file read into it, a write from it): with the heap + committed as it is touched (hos_heap_on_touch, hos_mem.c), each page of [addr, addr + size) is read once first, so that + one still only reserved is committed by the handler below, as the CPU's own first touch of it would. The kernel does + not fault a page in: ReadFile into heap pages nothing touched yet failed with no exception. A byte read a 4 KB page, + nothing written: 30 microseconds for a 100 MB buffer in use. Not VirtualQuery for which pages are reserved: it walks + every page of the committed region it is asked in, GBs of them, and the title ran 105 to 141 fps for 240 with it in + each read. With the heap committed whole (HOS_HEAP_COMMIT=1, large pages, Linux) nothing is done, as before */ +int hos_heap_on_touch; +void hos_guest_touch(uint64_t addr, uint64_t size) { + if (!hos_heap_on_touch || !size || !hos_guest_range(addr, size)) return; + const uint64_t end = addr + size; + for (uint64_t a = addr; a < end; a = (a | 0xFFF) + 1) (void)*(volatile const uint8_t *)GP(a); +} + DWORD WINAPI hos_out_of_memory_box(LPVOID unused) { (void)unused; MessageBoxA(NULL, @@ -80,12 +176,38 @@ static LONG CALLBACK commit_on_touch(EXCEPTION_POINTERS *info) { const uint64_t guest = (uint64_t)(p - A64_MEM); if (guest < 0x100000) return EXCEPTION_CONTINUE_SEARCH; /* (a null pointer: the game's own fault) */ MEMORY_BASIC_INFORMATION mbi; - if (!VirtualQuery(p, &mbi, sizeof mbi) || mbi.State != MEM_RESERVE) return EXCEPTION_CONTINUE_SEARCH; - static volatile LONG touched; - if (InterlockedIncrement(&touched) <= 16) + if (!VirtualQuery(p, &mbi, sizeof mbi)) return EXCEPTION_CONTINUE_SEARCH; +#ifdef _WIN32 + /* ((Triforce, 11-21, MEM-09) another thread committed this 64 KB between the fault and the look: the access is tried + again. With the heap committed as it is touched, two threads touching one new chunk at once is common, and the second + went on to the crash report. Read-write only: the lazy write-back's pages and the watches' are their own handlers'. + Windows only: Linux's VirtualQuery stand-in calls every guest page committed and read-write, and every fault in the + window would be tried again for ever) */ + if (heap_touch_again(mbi.State == MEM_COMMIT, mbi.Protect == PAGE_READWRITE, rec->ExceptionInformation[0])) + return EXCEPTION_CONTINUE_EXECUTION; +#endif + if (mbi.State != MEM_RESERVE) return EXCEPTION_CONTINUE_SEARCH; + /* ((Triforce, 11-21, MEM-09) the heap's own first touches are how it is committed now (hos_mem.c SetMemoryHeapSize): + counted apart, and not among the 16 lines for pages nothing handed out) */ + const int heap = guest >= HOS_HEAP_BASE && guest - HOS_HEAP_BASE < hos_heap_size; + static volatile LONG touched, heap_touched; + if (!heap && InterlockedIncrement(&touched) <= 16) hos_log("mem: %#llx committed on first touch (nothing handed it out)", (unsigned long long)guest); - if (!VirtualAlloc(GP(guest & ~0xFFFFULL), 0x10000, MEM_COMMIT, PAGE_READWRITE)) return EXCEPTION_CONTINUE_SEARCH; - hos_mem_note(HOS_MEM_TOUCHED, guest & ~0xFFFFULL, 0x10000); /* ((Triforce) the census: atomics only, no lock) */ + /* (only the reserved pages of its 64 KB: a commit over committed ones resets their protection, heap_commit.h) */ + const uint64_t run = (uint64_t)((const uint8_t *)mbi.BaseAddress - A64_MEM); + uint64_t lo, hi; + heap_touch_range(guest, run, run + mbi.RegionSize, &lo, &hi); + if (!VirtualAlloc(GP(lo), (SIZE_T)(hi - lo), MEM_COMMIT, PAGE_READWRITE) && + !(GetLastError() == ERROR_COMMITMENT_LIMIT && commit_wait(GP(lo), (SIZE_T)(hi - lo)))) /* (at the limit: waited for) */ + return EXCEPTION_CONTINUE_SEARCH; + hos_mem_note(HOS_MEM_TOUCHED, lo, hi - lo); /* ((Triforce) the census: atomics only, no lock) */ + if (heap) { + const LONG n = InterlockedIncrement(&heap_touched); + if (heap_touch_said(n)) + hos_log("mem: the heap is committed as it is touched: %ld first touches so far, 64 KB each at most (%ld MB of its " + "%llu; HOS_HEAP_COMMIT=1: all of it at the start, as before)", (long)n, (long)(n >> 4), + (unsigned long long)(hos_heap_size >> 20)); + } return EXCEPTION_CONTINUE_EXECUTION; } @@ -785,23 +907,81 @@ void hos_host_thread_register_self(const char *name) { /* (hos_prof.c: names f wchar_t w[64]; if (MultiByteToWideChar(CP_UTF8, 0, name, -1, w, 64)) SetThreadDescription(GetCurrentThread(), w); } -void hos_process_memory(uint64_t *private_bytes, uint64_t *working_set) { - typedef struct { - DWORD cb, PageFaultCount; - SIZE_T PeakWorkingSetSize, WorkingSetSize, QuotaPeakPagedPoolUsage, QuotaPagedPoolUsage, - QuotaPeakNonPagedPoolUsage, QuotaNonPagedPoolUsage, PagefileUsage, PeakPagefileUsage, PrivateUsage; - } Counters; - typedef BOOL(WINAPI * Fn)(HANDLE, Counters *, DWORD); +typedef struct { + DWORD cb, PageFaultCount; + SIZE_T PeakWorkingSetSize, WorkingSetSize, QuotaPeakPagedPoolUsage, QuotaPagedPoolUsage, + QuotaPeakNonPagedPoolUsage, QuotaNonPagedPoolUsage, PagefileUsage, PeakPagefileUsage, PrivateUsage; +} ProcessCounters; +/* (the process's memory counters as Windows keeps them; all zero when they cannot be read) */ +static void process_counters(ProcessCounters *m) { + typedef BOOL(WINAPI * Fn)(HANDLE, ProcessCounters *, DWORD); static Fn fn; if (!fn) fn = (Fn)(void *)GetProcAddress(GetModuleHandleA("kernel32.dll"), "K32GetProcessMemoryInfo"); - Counters m; - memset(&m, 0, sizeof m); - m.cb = sizeof m; - if (!fn || !fn(GetCurrentProcess(), &m, sizeof m)) memset(&m, 0, sizeof m); + memset(m, 0, sizeof *m); + m->cb = sizeof *m; + if (!fn || !fn(GetCurrentProcess(), m, sizeof *m)) memset(m, 0, sizeof *m); +} +void hos_process_memory(uint64_t *private_bytes, uint64_t *working_set) { + ProcessCounters m; + process_counters(&m); *private_bytes = m.PrivateUsage; *working_set = m.WorkingSetSize; } +/* ---- (Triforce, 11-22, the performance audit's OS-51, after Source B's) the host's part of a long frame -------------------- + host/stall_host.h has what is counted and the line. The counters are here, and the look the present makes + (nvn_native.c nvnQueuePresentTexture, on the presenting thread): every present takes what was counted since the last + one, with the process's page faults and CPU time (one K32GetProcessMemoryInfo and one GetProcessTimes), and a present + that comes the stall line's threshold or more after the last one logs it. HOS_STALL_HOST=0, or RX_VK_STALL_LOG=0: none + of it, and neither system call */ +StallHostCause hos_stall_host[STALL_HOST_CAUSES]; +int64_t hos_stall_host_us = -1; +/* (the process's page faults so far, soft and hard together, and its CPU time in the kernel and in its own code, in ns; + zero where they cannot be read) */ +static void process_work(uint64_t *page_faults, int64_t *kernel_ns, int64_t *user_ns) { + ProcessCounters m; + process_counters(&m); + *page_faults = m.PageFaultCount; + *kernel_ns = *user_ns = 0; +#if defined(_WIN32) + FILETIME made, ended, kernel, user; + if (GetProcessTimes(GetCurrentProcess(), &made, &ended, &kernel, &user)) { + *kernel_ns = (int64_t)(((uint64_t)kernel.dwHighDateTime << 32) | kernel.dwLowDateTime) * 100; + *user_ns = (int64_t)(((uint64_t)user.dwHighDateTime << 32) | user.dwLowDateTime) * 100; + } +#endif +} +void hos_stall_host_presented(void) { + void hos_vcore_waits_take(int64_t *ns, int64_t *count, int64_t *longest); + extern volatile LONG64 hos_present_count; /* display.c */ + static int64_t last_ns, last_kernel, last_user; + static uint64_t last_faults; + static StallHostBound bound = STALL_HOST_BOUND; + const int64_t threshold_us = stall_host_us(); + if (!threshold_us) return; + const int64_t now = hos_now_ns(); + StallHostFrame f; + uint64_t faults; + int64_t kernel, user; + process_work(&faults, &kernel, &user); + for (int i = 0; i < STALL_HOST_CAUSES; i++) f.cause[i] = stall_host_take(&hos_stall_host[i]); + { /* (the waits for a virtual core are counted where they end, with the virtual cores' lock held: hos_thread.c) */ + StallHostCause *w = &f.cause[STALL_HOST_CORE_WAIT]; + hos_vcore_waits_take(&w->ns, &w->count, &w->longest); + w->bytes = 0; + } + if (last_ns && now - last_ns >= threshold_us * 1000 && stall_host_bound_take(&bound, now)) { + char line[1024]; + f.frame = (int64_t)hos_present_count, f.gap_ns = now - last_ns; + f.page_faults = (uint32_t)(faults - last_faults); /* (Windows' count is 32 bits and wraps) */ + f.kernel_ns = kernel - last_kernel, f.user_ns = user - last_user; + f.held = bound.held, bound.held = 0; + stall_host_line(&f, line, sizeof line); + hos_log("%s", line); + } + last_ns = now, last_faults = faults, last_kernel = kernel, last_user = user; +} + /* ---- (Triforce) the guest memory census's thread (host/guest_census.h has why) --------------------------------------------- HOS_MEM_CENSUS= (5 by default; 0: none): every that many seconds a "mem: census" line: what the runtime handed out of the guest window by kind (the program and the system region, the heap, the address regions and the game's pages in them, diff --git a/host/mods_music.c b/host/mods_music.c new file mode 100644 index 0000000..07423fd --- /dev/null +++ b/host/mods_music.c @@ -0,0 +1,111 @@ +/* (Triforce, 11-20) The Mods screen's music: the game's shop theme under the Mods screen opened from the title, the + title's own music faded out meanwhile (mods_music.h has the rule and the owner's words). Here are the file and the + sound thread's side. + + Which tune: the Happy Mask Shop's, where the Salesman of the Mods screen keeps his masks, else the Market's shop + (the game's shops share the tune): Sound/Resource/Stream/BGM_Shop_Market_Mask.bwav of the player's game files, read + through nn_fs.c (a mod's where a mod replaces it), decoded by menu_sound.c's decoder (two channels of DSP-ADPCM at + 48 kHz, a minute long, looping from 5.6 s to its end as its file says). It is read once, on a thread of its own, + when the title's list first shows with its Mods entry, so that it is there when the screen opens; about 23 MB of + samples are kept from then on. Nothing is shipped: without the file the title's music plays on as it is. + + When: the options say (gpu/vk/vk_menu.cpp menu_mods_music, HOS_OPT_MODS_MUSIC), asked at each block the sound thread + mixes, 5 ms. How loud: MODS_MUSIC_GAIN under the Audio tab's Menu sounds row, and the Master volume over it as over + everything; with that row at 0 there is no tune and the title's music stays. */ +#include +#include +#include +#include + +#include "hos.h" +#include "menu_sound.h" +#include "mods_music.h" +#include "../gpu/vk/vk_api.h" /* (HOS_OPT_MODS_MUSIC, HOS_OPT_MENU_SOUND_VOLUME) */ +#define WIN32_LEAN_AND_MEAN +#include + +int rx_option(int id, int otherwise); /* rx_backend.c */ +uint8_t *hos_fs_rom_file(const char *rel, uint64_t most, uint64_t *size); /* nn_fs.c */ + +#define FILE_MOST (32u << 20) /* (the shop theme is 3.3 MB) */ +#define SECONDS_MOST 600 + +static const char *const files[] = { + "Sound/Resource/Stream/BGM_Shop_Market_Mask.bwav", + "Sound/Resource/Stream/BGM_Shop_Market.bwav", +}; + +static ModsMusic music; /* (the sound thread's) */ +/* (what the reading thread made: written before its release store of 2 into loaded, taken by the sound thread after) */ +static struct { float *pcm; uint32_t frames, loop_start, loop_end; } read_tune; +static atomic_int loaded; /* 0 not yet, 1 being read, 2 ready, 3 none in the game's files */ + +static uint32_t u32(const uint8_t *p) { return (uint32_t)p[0] | (uint32_t)p[1] << 8 | (uint32_t)p[2] << 16 | (uint32_t)p[3] << 24; } + +static DWORD WINAPI load_main(void *arg) { + (void)arg; + for (size_t i = 0; i < sizeof files / sizeof *files; i++) { + uint64_t size = 0; + uint8_t *b = hos_fs_rom_file(files[i], FILE_MOST, &size); + if (!b) continue; + uint32_t frames = 0; + float *pcm = hos_menu_sound_decode_long(b, size, &frames, SECONDS_MOST); + /* (the first channel's: its rate, and the loop's end and start in the file's own samples, 0xffffffff for none) */ + uint32_t rate = 0, loop_end = 0, loop_start = 0; + if (pcm && size >= 0x10 + 0x4c) rate = u32(b + 0x14), loop_end = u32(b + 0x10 + 0x3c), loop_start = u32(b + 0x10 + 0x40); + free(b); + if (!pcm) { + hos_log("mods music: %s is not a BWAV file the Mods screen plays (PCM16 or DSP-ADPCM, one or two channels, at most " + "%d s): left out", files[i], SECONDS_MOST); + continue; + } + if (loop_end == 0xffffffffu || !rate) loop_end = loop_start = 0; + else { /* (at the mixer's rate, as the samples are) */ + loop_end = (uint32_t)((uint64_t)loop_end * MODS_MUSIC_RATE / rate); + loop_start = (uint32_t)((uint64_t)loop_start * MODS_MUSIC_RATE / rate); + } + read_tune.pcm = pcm, read_tune.frames = frames, read_tune.loop_start = loop_start, read_tune.loop_end = loop_end; + hos_log("mods music: the Mods screen's tune from the game's files: %s, %.1f s%s", files[i], + (double)frames / MODS_MUSIC_RATE, loop_end > loop_start ? ", looping" : ", played once"); + atomic_store_explicit(&loaded, 2, memory_order_release); + return 0; + } + hos_log("mods music: the game's files have no shop theme (%s): the Mods screen has no music, the title's plays on", files[0]); + atomic_store_explicit(&loaded, 3, memory_order_release); + return 0; +} + +void hos_mods_music_frame(float *const *channels, int nch, uint32_t frames) { + if (nch < 1) return; + const int mode = rx_option(HOS_OPT_MODS_MUSIC, 0); + int state = atomic_load_explicit(&loaded, memory_order_acquire); + if (!mode && state != 2) return; /* (nothing asked and nothing playing: no other option is read) */ + const int volume = rx_option(HOS_OPT_MENU_SOUND_VOLUME, -1); + if (state == 0 && mode >= 1 && volume > 0) { + int none = 0; + if (atomic_compare_exchange_strong(&loaded, &none, 1)) { /* (read off the sound thread: a disk may take its time) */ + const HANDLE t = CreateThread(NULL, 0, load_main, NULL, 0, NULL); + if (t) CloseHandle(t); + else { + hos_log("mods music: no thread to read the Mods screen's tune: no music there"); + atomic_store_explicit(&loaded, 3, memory_order_release); + } + } + return; + } + if (state != 2) return; + if (!music.pcm) { /* (the tune was read: the sound thread's from here) */ + music.pcm = read_tune.pcm, music.frames = read_tune.frames; + mods_music_loop(&music, read_tune.loop_start, read_tune.loop_end); + } + const int want = mode == 2 && volume > 0; + static int wanted; + if (want != wanted) { + wanted = want; + hos_log(want ? "mods music: the Mods screen opened from the title: the shop theme comes, the title's music goes (0.6 s)" + : "mods music: the Mods screen shut: the shop theme goes, the title's music comes back (0.6 s)"); + } + /* (the tune into the front pair; a single channel takes both of its sides) */ + mods_music_block(&music, want, channels, nch, channels[0], channels[nch > 1 ? 1 : 0], frames, + MODS_MUSIC_GAIN * hos_menu_sound_gain(volume) * (nch > 1 ? 1.0f : 0.5f)); +} diff --git a/host/mods_music.h b/host/mods_music.h new file mode 100644 index 0000000..b792dfb --- /dev/null +++ b/host/mods_music.h @@ -0,0 +1,66 @@ +/* (Triforce, 11-20) The Mods screen's music. The owner: "have the in game mods menu in the title screen play the shop + theme quietly in the background ... it mutes the title screen music's audio while in the mod menu and fades in/out". + While the Mods screen is open from the title's list, the game's own shop theme (the Happy Mask Shop's, the Salesman's: + Sound/Resource/Stream/BGM_Shop_Market_Mask.bwav of the player's game files, a mod's where a mod replaces it) plays + under the screen, quietly, and the game's own sound (the title's music) is turned down to nothing as it comes and + back up as it goes. Nothing is shipped: game files without the tune leave the title's music as it is. + + This is the rule alone, no files and no threads (tests/test_mods_music.py compiles it): one level, 0 to 1, moves a + sample at a time toward 1 while the screen wants the music and toward 0 while it does not, over MODS_MUSIC_FADE + samples each way. The game's channels are multiplied by 1 - level and the tune is added times level, so the two + cross over with no step and no click. The tune loops between the points its file names (the shop theme: from 5.6 s + to its end); once the level is back at 0 it is at its top again for the next time the screen opens. */ +#pragma once +#include +#include + +#define MODS_MUSIC_RATE 48000 /* (nn_audio.c RATE, menu_sound.c MIX_RATE: the output's) */ +#define MODS_MUSIC_FADE (MODS_MUSIC_RATE * 6 / 10) /* the cross-over's length each way: 0.6 s */ +#define MODS_MUSIC_GAIN 0.6f /* "quietly": the file's own level is low (peak 0.25); 4 dB under it */ + +typedef struct ModsMusic { + const float *pcm; /* interleaved stereo at 48 kHz; NULL: no tune (not read, or not in the files) */ + uint32_t frames, loop_start, loop_end; /* loop_end > loop_start: it goes round between them; else once, then silence */ + uint32_t pos; /* the next frame of the tune */ + float level; /* how much of the tune is heard, 0 to 1; the game's own has 1 - level */ +} ModsMusic; + +/* The loop a file names, as the tune keeps it: [start, end) inside its frames, or none (0, 0) */ +static inline void mods_music_loop(ModsMusic *m, uint32_t start, uint32_t end) { + if (end > m->frames) end = m->frames; + if (end > start + MODS_MUSIC_RATE / 10) m->loop_start = start, m->loop_end = end; /* (a loop under 0.1 s is no tune's) */ + else m->loop_start = m->loop_end = 0; +} + +/* One block of the output. want: the Mods screen is open from the title and its sounds are not off. game: the nch + channels the game mixed, frames samples each, turned down by the level sample by sample; the tune is added to left + and right (the game's front pair) times gain. With no tune nothing changes, whatever want is. Returns the level. */ +static inline float mods_music_block(ModsMusic *m, int want, float *const *game, int nch, float *left, float *right, + uint32_t frames, float gain) { + if (!m->pcm || !m->frames) { + m->level = 0.0f; + return 0.0f; + } + const float step = 1.0f / (float)MODS_MUSIC_FADE; + if (!want && m->level <= 0.0f) return 0.0f; /* (shut and silent: the block is the game's as it is) */ + for (uint32_t i = 0; i < frames; i++) { + float level = m->level + (want ? step : -step); + level = level < 0.0f ? 0.0f : level > 1.0f ? 1.0f : level; + m->level = level; + const float game_gain = 1.0f - level; + for (int ch = 0; ch < nch; ch++) game[ch][i] *= game_gain; + if (level <= 0.0f) { /* (gone: at its top for the next time; the rest of the block is the game's) */ + m->pos = 0; + if (!want) break; + continue; + } + if (m->pos < m->frames) { + const float g = level * gain; + left[i] += m->pcm[2 * (size_t)m->pos] * g; + right[i] += m->pcm[2 * (size_t)m->pos + 1] * g; + m->pos++; + } + if (m->loop_end > m->loop_start && m->pos >= m->loop_end) m->pos = m->loop_start; + } + return m->level; +} diff --git a/host/movie_pace.h b/host/movie_pace.h new file mode 100644 index 0000000..6a0f9c7 --- /dev/null +++ b/host/movie_pace.h @@ -0,0 +1,54 @@ +/* (Triforce, 11-22, the performance audit's PRESENT-30, after Source A's) The frames paced at 240 while a movie plays at + the 60 cap (nn_vi.c movie_pace_decide, pace_fps, realtime_step). + + The game's movie player takes one picture at each of its updates at most, and its updates follow the frame loop. At + a cap of 60 the loop runs 60 times a second at best, a movie has 59.94 pictures a second, and every frame that comes + late is a picture the player never takes: the decoder skips it (nn_video.c's catch-up). A PC that misses 60 in a + movie skips a picture for every frame it misses. While a movie plays, at a cap of exactly 60, the frames are now + paced by 240, with the game's own limiter and its time step following as they do at a cap of 240: after a late + frame the next comes at once, and the player takes the picture that is due. Game time stays real time. When no + movie player has updated for a quarter second the cap is 60 again. + + Not at any other cap (above 60 the loop is fast enough already; under 60 the player asked for it), and not with the + Steady frame rate row on, which paces by a period of its own. HOS_MOVIE_PACE=0: the cap kept through movies, as + before. + + The rules alone, no game and no clock: tests/test_movie_pace.py compiles them. */ +#pragma once + +#define MOVIE_PACE_FPS 240 /* what the frames are paced by while a movie plays at the 60 cap */ + +/* 1 when this present is paced at MOVIE_PACE_FPS. on: HOS_MOVIE_PACE is not 0; cap: the frame rate cap; steady: the + Steady frame rate row is on; movie: a movie player updated in the last quarter second */ +static inline int movie_pace_raise(int on, int cap, int steady, int movie) { return on && cap == 60 && !steady && movie; } + +/* the rate the pacing, the game's limiter and its time step go by */ +static inline int movie_pace_fps(int raise, int cap) { return raise ? MOVIE_PACE_FPS : cap; } + +/* The game's own frame limiter (its rate, 0 or less: none) under the rate the frames are paced by. Above 60 it is + raised to that rate where it is lower, as it always was for a cap above 60; what it was before is kept, and put back + once the rate is 60 or less again, unless the game has written another value over the raised one meanwhile */ +typedef struct { int before, raised; } MovieLimit; +static inline int movie_pace_limit(MovieLimit *m, int limit, int pace) { + if (pace > 60) { + if (limit > 0 && limit < pace) { + if (!m->before) m->before = limit; + m->raised = pace; + return pace; + } + return limit; + } + if (m->before) { + const int back = limit == m->raised ? m->before : limit; + m->before = m->raised = 0; + return back; + } + return limit; +} + +/* The game's frame rate slot when the rate the frames are paced by comes down to 60 or less from `before`: a rate above + 60 that the time step left in it (only without the frame-time hook, where the slot follows the pace) goes back to 60, + which the step then moves as before. Without this the step would find a rate it does not own and leave it alone */ +static inline int movie_pace_slot(int slot, int before, int pace) { + return before > 60 && pace <= 60 && slot > 60 && slot <= before ? 60 : slot; +} diff --git a/host/nn_applet.c b/host/nn_applet.c index 1008d99..83b2ca6 100644 --- a/host/nn_applet.c +++ b/host/nn_applet.c @@ -152,6 +152,11 @@ uint32_t hos_output_size(int height) { much as the pool (nx2/oot_patches.py): a null pointer (a player's log, v0.3.7). With the 900 MB, 11-15 on an RTX 4070 SUPER: that output with RenoDX HDR boots and played in the world, 17,160 presents at about 50 fps, RenoDX's four shaders in place, no error from the pool (one short session, not every area; its scene target and each copy are 118 MB there). */ +/* ((Triforce, 11-21, the performance audit's TEX-85) Whether the render targets' heap is held to a size, with no room to + give: HOS_RT_POOL_MB set, or the 5120-wide outputs' 900 MB. -1 until hos_rt_pool_size has run. host/shadow_slot.h reserves + no stepped storage there unless asked to) */ +static int rt_pool_pinned = -1; +int hos_rt_pool_pinned(void) { return __atomic_load_n(&rt_pool_pinned, __ATOMIC_RELAXED); } uint64_t hos_rt_pool_size(uint64_t game) { const char *e = getenv("HOS_RT_POOL_MB"); extern volatile uint64_t hos_wide_out; @@ -169,6 +174,7 @@ uint64_t hos_rt_pool_size(uint64_t game) { const uint64_t asked = (e ? strtoull(e, NULL, 0) << 20 : scaled < minimum ? minimum : is_five_k && scaled > five_k ? five_k : scaled) + (e ? 0 : rgba16f); /* (the 5120 x 2880 output: 900 MB for the whole pool, see above; an explicit HOS_RT_POOL_MB still wins) */ const uint64_t v = !e && is_five_k && asked > five_k ? five_k : asked; + __atomic_store_n(&rt_pool_pinned, e != NULL || v != asked || (is_five_k && scaled > five_k), __ATOMIC_RELAXED); /* ((Triforce, 11-21, TEX-85)) */ static int logged; if (!logged++) { hos_log("applet: render target pool %llu MB (the game's %llu MB)", (unsigned long long)(v >> 20), (unsigned long long)(game >> 20)); diff --git a/host/nn_audio.c b/host/nn_audio.c index a1f03ec..8cddfab 100644 --- a/host/nn_audio.c +++ b/host/nn_audio.c @@ -8,8 +8,10 @@ -> sub mixes in dependency order (sub mix volume, then to their destination) -> the final mix (volume) -> the device sink (its channel map: 5.1 as it is to a 5.1 device, else down to stereo) -> waveOut - and signals the renderer event the game's audio thread waits on. Effects are accepted and left out (dry signal); - aux effects pass through. Played sample counts and released wave buffers go back to the game as on hardware. + and signals the renderer event the game's audio thread waits on. A mix's effects (I3DL2 and plain reverbs, delays, + biquad filters) run on its buffers in the order the game added them, and splitter destinations have their biquad + filters (see "mix effects"); aux effects pass through. Played sample counts and released wave buffers go back to + the game as on hardware. The game's parameter changes take effect together at RequestUpdateAudioRenderer, and volume changes ramp over a frame, as the console's renderer does them (see "parameter updates"). HOS_AUDIO_DUMP=: also write what is played (checking without speakers); HOS_AUDIO=0: silence; @@ -51,7 +53,7 @@ void hos_system_event_signal(uint64_t key); #define MAX_WB 32 /* queued wave buffers per voice */ #define MAX_REL 128 -enum { K_VOICE = 1, K_MIX, K_SPLITTER, K_SINK, K_POOL, K_EFFECT, K_CONFIG }; +enum { K_VOICE = 1, K_MIX, K_SPLITTER, K_SINK, K_POOL, K_EFFECT, K_CONFIG, K_BIQUAD, K_DELAY }; enum { FMT_INVALID, FMT_PCM8, FMT_PCM16, FMT_PCM24, FMT_PCM32, FMT_FLOAT, FMT_ADPCM }; enum { PLAY_STARTED, PLAY_STOPPED, PLAY_PAUSED }; enum { DEST_NONE, DEST_MIX, DEST_SPLITTER }; @@ -71,6 +73,8 @@ typedef struct { int dest_mix[MAX_SPLIT_DESTS]; float vol[MAX_SPLIT_DESTS][MAX_CH][MAX_BUF]; float ramp[MAX_SPLIT_DESTS][MAX_CH][MAX_BUF]; /* the volumes the last frame ended with */ + int bq_on[MAX_SPLIT_DESTS][2]; /* each destination's two biquad filters (SetSplitterBiquadFilterParameter) */ + float bq_b[MAX_SPLIT_DESTS][2][3], bq_a[MAX_SPLIT_DESTS][2][2], bq_z[MAX_SPLIT_DESTS][2][MAX_CH][2]; } Splitter; typedef struct { @@ -299,12 +303,61 @@ typedef struct { float line[4][REV_LINE]; int pos[4], len[4]; float lp[4]; /* damping filter states */ + int seq; /* (the order effects were added in: effects_frame) */ + /* a plain reverb's (AddReverb) parameters: gains 0 to 1, times as nn::TimeSpan; plain_reverb_params turns them + into the fields above each frame */ + int plain, early_mode, late_mode; + float p_early, p_late, p_reverb, p_out, p_dry, p_hf, p_color; + int64_t p_decay_ns, p_pre_ns; + uint64_t buffer; /* (the game's work buffer, for RemoveReverb) */ } Reverb; static Reverb reverbs[MAX_EFFECTS]; +static int fx_seq; +/* HOS_AUDIO_EFFECTS=0: the plain reverb, delay and biquad filter effects and the splitter biquad filters accepted and + left out, as before they were emulated (the I3DL2 reverb still runs) */ +static int effects_on(void) { + static int on = -1; + if (on < 0) on = !(getenv("HOS_AUDIO_EFFECTS") && atoi(getenv("HOS_AUDIO_EFFECTS")) == 0); + return on; +} +/* an effect left disabled still copies its inputs to its outputs where they differ, as the renderer's do */ +static void fx_bypass(const int8_t *in, const int8_t *out, int n, Mix *m) { + for (int c = 0; c < n; c++) + if (in[c] != out[c] && in[c] >= 0 && in[c] < m->buffers && out[c] >= 0 && out[c] < m->buffers) + memcpy(m->buf[out[c]], m->buf[in[c]], sizeof m->buf[0]); +} +static float clamp01(float f) { return f > 0.0f ? (f < 1.0f ? f : 1.0f) : 0.0f; } +static float gain_mb(float g) { return g > 0.00001f ? 2000.0f * log10f(g) : -10000.0f; } +/* The plain reverb on the same delay network: EarlyMode SmallRoom 0, LargeRoom 1, Hall 2, Cavern 3, + NoEarlyReflection 4; LateMode Room 0, Hall 1, MetallicCorridor 2, Cavern 3, MaximumDelay 4. Reverb gain feeds + both parts, early and late gain each part, out gain the wet sum; coloration is less diffusion. The DSP's own early + and late tables are not ours: the modes set the line lengths and the late part's delay. */ +static void plain_reverb_params(Reverb *rv) { + static const float density[5] = {50, 100, 25, 100, 100}, late_ms[5] = {5, 10, 4, 20, 30}; + const int em = rv->early_mode >= 0 && rv->early_mode < 5 ? rv->early_mode : 2; + const int lm = rv->late_mode >= 0 && rv->late_mode < 5 ? rv->late_mode : 1; + const float wet = clamp01(rv->p_out) * clamp01(rv->p_reverb); + rv->room = 0; + rv->refl = em == 4 ? -10000.0f : gain_mb(wet * clamp01(rv->p_early)); + rv->reverb = gain_mb(wet * clamp01(rv->p_late)); + rv->dry = clamp01(rv->p_dry); + rv->hf_ratio = rv->p_hf < 0.1f ? 0.1f : rv->p_hf > 1.0f ? 1.0f : rv->p_hf; + rv->hf_ref = 5000; + rv->diffusion = 100.0f * (1.0f - 0.7f * clamp01(rv->p_color)); + rv->density = density[lm]; + rv->decay_ns = rv->p_decay_ns > 0 ? rv->p_decay_ns : 1000000000ll; + rv->refl_delay_ns = rv->p_pre_ns > 0 ? rv->p_pre_ns : 0; + rv->late_delay_ns = (int64_t)(late_ms[lm] * 1e6f); +} static float mb_gain(float mb) { return powf(10.0f, mb / 2000.0f); } static void reverb_frame(Reverb *rv, Mix *m) { static int off = -1; if (off < 0) off = getenv("HOS_AUDIO_NO_REVERB") != NULL; + if (rv->plain) { + if (!effects_on()) return; + if (!rv->enabled) { fx_bypass(rv->in, rv->out, rv->n, m); return; } + plain_reverb_params(rv); + } if (off || !rv->enabled || rv->n <= 0) return; static const int base_len[4] = {1559, 1617, 1491, 1422}; const float density = rv->density > 0 ? rv->density / 100.0f : 1.0f; @@ -370,9 +423,148 @@ static void reverb_frame(Reverb *rv, Mix *m) { } } } + +/* ---- mix effects: biquad filter and delay ---------------------------------------------------------------------- + As the console's renderer runs them on a mix's buffers, each channel its input buffer to its output buffer + (SetXInputOutput; the same buffer in place). Biquad filter: the game's coefficients (TBiquadFilterParameter, + the denominator added, as the voices' filters) in transposed direct form II. Delay: a line per channel of the + delay time, fed in gain x input plus the lines' outputs through the feedback gain (x 0.98) and the channel spread + (to the neighbouring channels), through a one-pole low pass (0.95 x the low pass amount); out = dry gain x input + + the line's output. Both allocate nothing here: the delay's lines are taken by AddDelay. */ +#define MAX_FX 32 +#define DELAY_CH 6 /* (the renderer's most) */ +#define DELAY_MAX (RATE * 5) /* a delay line: 5 s at most */ +typedef struct { + int used, mix, enabled, on, n, seq; /* (on: the parameter's own enable) */ + int8_t in[MAX_CH], out[MAX_CH]; + float b[3], a[2], z[MAX_CH][2]; +} Biquad; +typedef struct { + int used, mix, enabled, n, seq, cap, len, pos; + int8_t in[DELAY_CH], out[DELAY_CH]; + int64_t time_ns, max_ns; + float in_gain, feedback, dry, spread, lowpass; + float lp[DELAY_CH]; + float *line; /* cap frames a channel, one channel after another */ + uint64_t buffer; /* (the game's work buffer, for RemoveDelay) */ +} Delay; +static Biquad biquads[MAX_FX]; +static Delay delays[MAX_FX]; +static void biquad_run(const float *b, const float *a, float *z, const float *in, float *out) { /* (in may be out) */ + float z0 = z[0], z1 = z[1]; + for (int i = 0; i < FRAME; i++) { + const float x = in[i], y = b[0] * x + z0; + z0 = b[1] * x + a[0] * y + z1; + z1 = b[2] * x + a[1] * y; + out[i] = y; + } + if (!isfinite(z0) || !isfinite(z1)) { /* (blown up: silence and start over rather than a full-scale burst) */ + z0 = z1 = 0; + memset(out, 0, FRAME * sizeof *out); + } + z[0] = z0; + z[1] = z1; +} +static void biquad_frame(Biquad *f, Mix *m) { + if (!f->enabled || !f->on) { fx_bypass(f->in, f->out, f->n, m); return; } + for (int c = 0; c < f->n; c++) { + const int bi = f->in[c], bo = f->out[c]; + if (bi >= 0 && bi < m->buffers && bo >= 0 && bo < m->buffers) biquad_run(f->b, f->a, f->z[c], m->buf[bi], m->buf[bo]); + } +} +static void delay_frame(Delay *dl, Mix *m) { + if (!dl->enabled || !dl->line || dl->n <= 0) { fx_bypass(dl->in, dl->out, dl->n, m); return; } + const int n = dl->n, cap = dl->cap; + const double want = (double)dl->time_ns * RATE / 1e9; + const int len = want < 1 ? 1 : want < cap ? (int)want : cap; + if (len != dl->len) { dl->len = len; if (dl->pos >= len) dl->pos = 0; } + /* the feedback matrix: stereo each line to the other, 4 channels to the two beside it, 5.1 front to centre and + the rear on its side, centre to both fronts, LFE to itself alone */ + const float fb = clamp01(dl->feedback) * 0.98f, spread = clamp01(dl->spread), base = (1.0f - spread) * fb; + const float cross = (n == 4 || n == 6 ? 0.5f : 1.0f) * spread * fb; + float mx[DELAY_CH][DELAY_CH] = {{0}}; + for (int c = 0; c < n; c++) mx[c][c] = n == 2 || n == 4 || n == 6 ? base : fb; + if (n == 2) mx[0][1] = mx[1][0] = cross; + if (n == 4) + for (int c = 0; c < 4; c++) mx[c][(c + 1) % 4] = mx[c][(c + 3) % 4] = cross; + if (n == 6) { + mx[0][2] = mx[0][4] = mx[1][2] = mx[1][5] = mx[2][0] = mx[2][1] = cross; + mx[4][0] = mx[4][5] = mx[5][1] = mx[5][4] = cross; + mx[3][3] = fb; + } + const float lp_fb = 0.95f * clamp01(dl->lowpass), lp_in = 1.0f - lp_fb, in_gain = dl->in_gain, dry = dl->dry; + const float *src[DELAY_CH]; + float *dst[DELAY_CH]; + for (int c = 0; c < n; c++) { + src[c] = dl->in[c] >= 0 && dl->in[c] < m->buffers ? m->buf[dl->in[c]] : NULL; + dst[c] = dl->out[c] >= 0 && dl->out[c] < m->buffers ? m->buf[dl->out[c]] : NULL; + } + int pos = dl->pos; + for (int i = 0; i < FRAME; i++) { + float x[DELAY_CH], d[DELAY_CH]; + for (int c = 0; c < n; c++) { + x[c] = src[c] ? src[c][i] : 0.0f; + d[c] = dl->line[c * cap + pos]; + } + for (int c = 0; c < n; c++) { + float t = in_gain * x[c]; + for (int k = 0; k < n; k++) t += mx[c][k] * d[k]; + dl->lp[c] = lp_fb * dl->lp[c] + lp_in * t; + dl->line[c * cap + pos] = dl->lp[c]; + } + for (int c = 0; c < n; c++) + if (dst[c]) dst[c][i] = dry * x[c] + d[c]; + if (++pos >= len) pos = 0; + } + dl->pos = pos; + for (int c = 0; c < n; c++) + if (!isfinite(dl->lp[c])) { /* (a NaN would stay in the line for good) */ + memset(dl->line, 0, (size_t)cap * n * sizeof *dl->line); + memset(dl->lp, 0, sizeof dl->lp); + break; + } +} +/* A mix's effects in the order the game added them; HOS_AUDIO_EFFECTS=0: the I3DL2 reverb alone, as before */ static void effects_frame(int mix) { + struct { int seq, kind, i; } run[MAX_EFFECTS + 2 * MAX_FX]; + int n = 0; + const int all = effects_on(); for (int e = 0; e < MAX_EFFECTS; e++) - if (reverbs[e].used && reverbs[e].mix == mix) reverb_frame(&reverbs[e], &mixes[mix]); + if (reverbs[e].used && reverbs[e].mix == mix && (all || !reverbs[e].plain)) run[n].seq = reverbs[e].seq, run[n].kind = 0, run[n++].i = e; + for (int e = 0; all && e < MAX_FX; e++) { + if (biquads[e].used && biquads[e].mix == mix) run[n].seq = biquads[e].seq, run[n].kind = 1, run[n++].i = e; + if (delays[e].used && delays[e].mix == mix) run[n].seq = delays[e].seq, run[n].kind = 2, run[n++].i = e; + } + for (int a = 1; a < n; a++) + for (int b = a; b > 0 && run[b].seq < run[b - 1].seq; b--) { + const int s = run[b].seq, k = run[b].kind, i = run[b].i; + run[b] = run[b - 1]; + run[b - 1].seq = s, run[b - 1].kind = k, run[b - 1].i = i; + } + for (int k = 0; k < n; k++) { + if (run[k].kind == 0) reverb_frame(&reverbs[run[k].i], &mixes[mix]); + else if (run[k].kind == 1) biquad_frame(&biquads[run[k].i], &mixes[mix]); + else delay_frame(&delays[run[k].i], &mixes[mix]); + } +} +static void effects_drop_mix(int mix) { /* (a mix slot taken again: the effects of the one released there go; the + I3DL2 reverb's as before, kept) */ + for (int e = 0; e < MAX_EFFECTS; e++) + if (reverbs[e].plain && reverbs[e].mix == mix) reverbs[e].used = 0; + for (int e = 0; e < MAX_FX; e++) { + if (biquads[e].mix == mix) biquads[e].used = 0; + if (delays[e].mix == mix) delays[e].used = 0; /* (its lines stay for the next AddDelay to free) */ + } +} +/* A splitter destination's two biquad filters on one channel of a voice: the filtered samples (in tmp), or in */ +static const float *splitter_filtered(Splitter *s, int d, int ch, const float *in, float *tmp) { + const float *src = in; + for (int q = 0; q < 2; q++) { + if (!s->bq_on[d][q]) continue; + biquad_run(s->bq_b[d][q], s->bq_a[d][q], s->bq_z[d][q][ch], src, tmp); + src = tmp; + } + return src; } /* ---- the mixer ------------------------------------------------------------------------------------------------- */ @@ -435,17 +627,22 @@ static void render_voice(Voice *v) { } } else if (v->dest_kind == DEST_SPLITTER && splitters[v->dest].used) { Splitter *s = &splitters[v->dest]; + const int fx = effects_on(); /* (the destinations' biquad filters: HOS_AUDIO_EFFECTS=0 leaves them out) */ for (int d = 0; d < s->dests; d++) { const int mi = s->dest_mix[d]; if (mi < 0 || !mixes[mi].used) continue; Mix *m = &mixes[mi]; - for (int ch = 0; ch < v->channels && ch < MAX_CH; ch++) + const int filtered = fx && (s->bq_on[d][0] || s->bq_on[d][1]); + for (int ch = 0; ch < v->channels && ch < MAX_CH; ch++) { + float tmp[FRAME]; + const float *src = filtered ? splitter_filtered(s, d, ch, out[ch], tmp) : out[ch]; for (int b = 0; b < m->buffers; b++) { const float g0 = s->ramp[d][ch][b], g1 = s->vol[d][ch][b]; if (g0 == 0.0f && g1 == 0.0f) continue; const float dg = (g1 - g0) / FRAME; - for (int i = 0; i < FRAME; i++) m->buf[b][i] += out[ch][i] * (g0 + dg * (float)(i + 1)); + for (int i = 0; i < FRAME; i++) m->buf[b][i] += src[i] * (g0 + dg * (float)(i + 1)); } + } } } memcpy(v->ramp_vol, v->vol, sizeof v->ramp_vol); /* (splitters catch up in render_frame) */ @@ -590,6 +787,13 @@ static void render_frame(int16_t *pcm, int channels) { /* 240 frames of the de lim_env = want < lim_env ? want : lim_env + (want - lim_env) * 0.0004f; for (int ch = 0; ch < nch; ch++) out[ch][i] *= lim_in * lim_env * lim_out; } + { /* ((Triforce, 11-20) the Mods screen's music (mods_music.c): while that screen is open from the title, the game's + own mix (the title's music) fades to nothing and the game's shop theme plays quietly in its place; after the + limiter, the game's mix's own, and before the options' sounds, which stay as loud as they are) */ + float *channels[6]; + for (int ch = 0; ch < nch && ch < 6; ch++) channels[ch] = out[ch]; + hos_mods_music_frame(channels, nch < 6 ? nch : 6, FRAME); + } /* (Triforce) the options' own sounds, the game's interface sounds (menu_sound.c), into the front left and right: after the limiter, which is the game's mix's own, and before the Master volume, which is over them too */ hos_menu_sound_frame(out[0], out[1], FRAME); @@ -1011,7 +1215,7 @@ static void output_frame(const int16_t *pcm) { HOS_AUDIO_IMMEDIATE=1: every change applied as it comes (as before). */ enum { OP_VOICE_RELEASE, OP_VOICE_DEST, OP_VOICE_MIX_VOLUME, OP_VOICE_VOLUME, OP_VOICE_PITCH, OP_VOICE_BIQUAD, OP_VOICE_STATE, OP_VOICE_APPEND, OP_MIX_RELEASE, OP_MIX_VOLUME, OP_MIX_DEST, OP_MIX_MIX_VOLUME, OP_SPLIT_RELEASE, - OP_SPLIT_DEST, OP_SPLIT_VOLUME }; + OP_SPLIT_DEST, OP_SPLIT_VOLUME, OP_SPLIT_BIQUAD }; typedef struct { int op, idx, tgt; /* our voice / mix / splitter; a destination mix or splitter (-1: none) */ int i0, i1, i2; /* (op-specific: a kind, an index, channels, a state) */ @@ -1097,6 +1301,12 @@ static void apply(const Cmd *k) { /* (with the lock held) */ s->vol[k->i0][k->i1][k->i2] = k->f; if (s->dest_mix[k->i0] < 0) s->dest_mix[k->i0] = tgt_mix; break; + case OP_SPLIT_BIQUAD: /* (destination i0, filter i1, on i2) */ + if (k->i2 && !s->bq_on[k->i0][k->i1]) memset(s->bq_z[k->i0][k->i1], 0, sizeof s->bq_z[k->i0][k->i1]); + s->bq_on[k->i0][k->i1] = k->i2; + memcpy(s->bq_b[k->i0][k->i1], k->b, sizeof k->b); + memcpy(s->bq_a[k->i0][k->i1], k->a, sizeof k->a); + break; } } static void submit(const Cmd *k) { /* (with the lock held) */ @@ -1278,6 +1488,7 @@ static int new_mix(uint64_t obj, int final_mix, int buffers) { while (i < MAX_MIXES && mixes[i].used) i++; if (i == MAX_MIXES) { au_unlock(); return 0; } memset(&mixes[i], 0, sizeof mixes[i]); + effects_drop_mix(i); mixes[i].used = 1; mixes[i].final_mix = final_mix; mixes[i].buffers = buffers < 1 ? 1 : buffers > MAX_BUF ? MAX_BUF : buffers; @@ -1397,6 +1608,28 @@ static void set_split_volume(Ctx *c) { } HLE(_ZN2nn5audio20SetSplitterMixVolumeEPNS0_12SplitterTypeEiPNS0_10SubMixTypeEfii) { set_split_volume(c); } HLE(_ZN2nn5audio20SetSplitterMixVolumeEPNS0_12SplitterTypeEiPNS0_12FinalMixTypeEfii) { set_split_volume(c); } +/* SetSplitterBiquadFilterParameter(SplitterType*, int destinationIndex, mix, int filterIndex, + const TBiquadFilterParameter&): bool @0, numerator[3] @4, additive-feedback denominator[2] @16, as the + voices'. The game sets both of a destination's filters (off) as it sets the destination, then its own. */ +static void set_split_biquad(Ctx *c) { + au_lock(); + const int d = (int)ARG(1), f = (int)ARG(3); + const uint64_t p = ARG(4); + if (splitter_of(ARG(0)) && d >= 0 && d < MAX_SPLIT_DESTS && f >= 0 && f < 2 && p) { + Cmd k = {.op = OP_SPLIT_BIQUAD, .idx = splitter_index(ARG(0)), .tgt = mix_index(ARG(2)), .i0 = d, .i1 = f, + .i2 = *(uint8_t *)GP(p) != 0}; + memcpy(k.b, GP(p + 4), sizeof k.b); + memcpy(k.a, GP(p + 16), sizeof k.a); + submit(&k); + static int logged; + if (k.i2 && logged++ < 8) + AUT("audio: splitter %d destination %d biquad %d: b %.3f %.3f %.3f a %.3f %.3f", k.idx, d, f, k.b[0], k.b[1], + k.b[2], k.a[0], k.a[1]); + } + au_unlock(); +} +HLE(_ZN2nn5audio32SetSplitterBiquadFilterParameterEPNS0_12SplitterTypeEiPNS0_10SubMixTypeEiRKNS0_22TBiquadFilterParameterIfEE) { set_split_biquad(c); } +HLE(_ZN2nn5audio32SetSplitterBiquadFilterParameterEPNS0_12SplitterTypeEiPNS0_12FinalMixTypeEiRKNS0_22TBiquadFilterParameterIfEE) { set_split_biquad(c); } /* ---- the device sink: AddDeviceSink(config, DeviceSinkType*, FinalMixType*, const int8* input, int count, name) - */ HLE(_ZN2nn5audio13AddDeviceSinkEPNS0_19AudioRendererConfigEPNS0_14DeviceSinkTypeEPNS0_12FinalMixTypeEPKaiPKc) { @@ -1570,6 +1803,7 @@ HLE(_ZN2nn5audio16AppendWaveBufferEPNS0_9VoiceTypeEPKNS0_10WaveBufferE) { FILE *f = fopen(path, "wb"); if (f) { fwrite(v->coefs, 1, 32, f); + hos_guest_touch(w.buffer, (uint64_t)w.size); /* ((Triforce, 11-21, MEM-09) fwrite hands it to the kernel) */ const int io = hos_guest_io_begin(GP(w.buffer), (uint64_t)w.size); /* (Triforce: rx_backend.c) */ fwrite(GP(w.buffer), 1, (size_t)w.size, f); hos_guest_io_end(io); @@ -1619,9 +1853,13 @@ HLE(_ZN2nn5audio25GetVoicePlayedSampleCountEPKNS0_9VoiceTypeE) { } HLE(_ZN2nn5audio20IsVoiceDroppedFlagOnEPKNS0_9VoiceTypeE) { RET(0); } -/* ---- effects: accepted, not processed (the dry signal goes through) -------------------------------------------- */ +/* ---- aux effects: accepted, passed through ---------------------------------------------------------------------- + The game reads the send buffer and writes the return buffer on its own threads (its CPU effects); with nothing to + read, it writes nothing back, and the mix goes through as it came. */ HLE(_ZN2nn5audio6AddAuxEPNS0_19AudioRendererConfigEPNS0_7AuxTypeEPNS0_10SubMixTypeEPvS7_m) { RET(0); } HLE(_ZN2nn5audio6AddAuxEPNS0_19AudioRendererConfigEPNS0_7AuxTypeEPNS0_12FinalMixTypeEPvS7_m) { RET(0); } +HLE(_ZN2nn5audio13SetAuxEnabledEPNS0_7AuxTypeEb) {} +HLE(_ZN2nn5audio17SetAuxInputOutputEPNS0_7AuxTypeEPKaS4_i) {} HLE(_ZN2nn5audio12IsAuxEnabledEPKNS0_7AuxTypeE) { RET(1); } HLE(_ZN2nn5audio14IsAuxRemovableEPKNS0_7AuxTypeE) { RET(1); } HLE(_ZN2nn5audio17ReadAuxSendBufferEPNS0_7AuxTypeEPii) { RET(0); } @@ -1645,14 +1883,14 @@ static Reverb *reverb_of(uint64_t obj) { const int i = get_handle(obj, K_EFFECT); return i >= 0 && i < MAX_EFFECTS && reverbs[i].used ? &reverbs[i] : NULL; } -static void add_reverb(Ctx *c) { +static void add_reverb(Ctx *c, int plain) { au_lock(); int e = 0; while (e < MAX_EFFECTS && reverbs[e].used) e++; const int m = mix_index(ARG(4)); if (e == MAX_EFFECTS || m < 0) { au_unlock(); - AUT("audio: i3dl2 reverb refused (mix %d)", m); + AUT("audio: %s reverb refused (mix %d)", plain ? "plain" : "i3dl2", m); RET(0x1A9 << 9 | 153); /* (out of effects: an audio result) */ return; } @@ -1666,13 +1904,22 @@ static void add_reverb(Ctx *c) { rv->room = -1000; rv->room_hf = -100; rv->refl = -2602; rv->reverb = 200; rv->hf_ratio = 0.83f; rv->diffusion = 100; rv->density = 100; rv->hf_ref = 5000; rv->dry = 1.0f; rv->decay_ns = 1490000000ll; rv->refl_delay_ns = 7000000; rv->late_delay_ns = 11000000; + rv->seq = ++fx_seq; + rv->buffer = ARG(2); + if (plain) { /* (until the game sets its own: a hall, half early and late) */ + rv->plain = 1; + rv->early_mode = 2; + rv->late_mode = 1; + rv->p_early = rv->p_late = 0.5f; rv->p_reverb = rv->p_out = rv->p_dry = 1.0f; rv->p_hf = 0.5f; rv->p_color = 0.5f; + rv->p_decay_ns = 1500000000ll; rv->p_pre_ns = 10000000; + } put_handle(ARG(1), K_EFFECT, e); au_unlock(); - AUT("audio: i3dl2 reverb %d on mix %d, %d channels", e, m, rv->n); + AUT("audio: %s reverb %d on mix %d, %d channels", plain ? "plain" : "i3dl2", e, m, rv->n); RET(0); } -HLE(_ZN2nn5audio14AddI3dl2ReverbEPNS0_19AudioRendererConfigEPNS0_15I3dl2ReverbTypeEPvmPNS0_10SubMixTypeEi) { add_reverb(c); } -HLE(_ZN2nn5audio14AddI3dl2ReverbEPNS0_19AudioRendererConfigEPNS0_15I3dl2ReverbTypeEPvmPNS0_12FinalMixTypeEi) { add_reverb(c); } +HLE(_ZN2nn5audio14AddI3dl2ReverbEPNS0_19AudioRendererConfigEPNS0_15I3dl2ReverbTypeEPvmPNS0_10SubMixTypeEi) { add_reverb(c, 0); } +HLE(_ZN2nn5audio14AddI3dl2ReverbEPNS0_19AudioRendererConfigEPNS0_15I3dl2ReverbTypeEPvmPNS0_12FinalMixTypeEi) { add_reverb(c, 0); } static void ret_float(Ctx *c, float f) { uint32_t bits; memcpy(&bits, &f, 4); @@ -1737,3 +1984,241 @@ HLE(_ZN2nn5audio25GetI3dl2ReverbInputOutputEPKNS0_15I3dl2ReverbTypeEPaS4_Pii) { } if (ARG(3)) G32(ARG(3)) = (uint32_t)n; } + +/* ---- plain reverb: AddReverb(config, ReverbType*, buffer, size, SubMixType* / FinalMixType*, channels) ---------- */ +static Reverb *plain_of(uint64_t obj) { + Reverb *rv = reverb_of(obj); + return rv && rv->plain ? rv : NULL; +} +HLE(_ZN2nn5audio9AddReverbEPNS0_19AudioRendererConfigEPNS0_10ReverbTypeEPvmPNS0_10SubMixTypeEi) { add_reverb(c, 1); } +HLE(_ZN2nn5audio9AddReverbEPNS0_19AudioRendererConfigEPNS0_10ReverbTypeEPvmPNS0_12FinalMixTypeEi) { add_reverb(c, 1); } +static void remove_reverb(Ctx *c) { /* (RemoveReverb(config, ReverbType*, mix): the work buffer back) */ + au_lock(); + Reverb *rv = plain_of(ARG(1)); + uint64_t buffer = 0; + if (rv) rv->used = 0, buffer = rv->buffer; + au_unlock(); + if (rv) G64(ARG(1)) = 0; + RET(buffer); +} +HLE(_ZN2nn5audio12RemoveReverbEPNS0_19AudioRendererConfigEPNS0_10ReverbTypeEPNS0_10SubMixTypeE) { remove_reverb(c); } +HLE(_ZN2nn5audio12RemoveReverbEPNS0_19AudioRendererConfigEPNS0_10ReverbTypeEPNS0_12FinalMixTypeE) { remove_reverb(c); } +HLE(_ZN2nn5audio18SetReverbEarlyModeEPNS0_10ReverbTypeENS1_9EarlyModeE) { Reverb *rv = plain_of(ARG(0)); if (rv) rv->early_mode = (int)ARG(1); } +HLE(_ZN2nn5audio17SetReverbLateModeEPNS0_10ReverbTypeENS1_8LateModeE) { Reverb *rv = plain_of(ARG(0)); if (rv) rv->late_mode = (int)ARG(1); } +HLE(_ZN2nn5audio18SetReverbEarlyGainEPNS0_10ReverbTypeEf) { Reverb *rv = plain_of(ARG(0)); if (rv) rv->p_early = farg(c, 0); } +HLE(_ZN2nn5audio17SetReverbLateGainEPNS0_10ReverbTypeEf) { Reverb *rv = plain_of(ARG(0)); if (rv) rv->p_late = farg(c, 0); } +HLE(_ZN2nn5audio19SetReverbReverbGainEPNS0_10ReverbTypeEf) { Reverb *rv = plain_of(ARG(0)); if (rv) rv->p_reverb = farg(c, 0); } +HLE(_ZN2nn5audio16SetReverbOutGainEPNS0_10ReverbTypeEf) { Reverb *rv = plain_of(ARG(0)); if (rv) rv->p_out = farg(c, 0); } +HLE(_ZN2nn5audio16SetReverbDryGainEPNS0_10ReverbTypeEf) { Reverb *rv = plain_of(ARG(0)); if (rv) rv->p_dry = farg(c, 0); } +HLE(_ZN2nn5audio32SetReverbHighFrequencyDecayRatioEPNS0_10ReverbTypeEf) { Reverb *rv = plain_of(ARG(0)); if (rv) rv->p_hf = farg(c, 0); } +HLE(_ZN2nn5audio19SetReverbColorationEPNS0_10ReverbTypeEf) { Reverb *rv = plain_of(ARG(0)); if (rv) rv->p_color = farg(c, 0); } +HLE(_ZN2nn5audio18SetReverbDecayTimeEPNS0_10ReverbTypeENS_8TimeSpanE) { Reverb *rv = plain_of(ARG(0)); if (rv) rv->p_decay_ns = (int64_t)ARG(1); } +HLE(_ZN2nn5audio21SetReverbPredelayTimeEPNS0_10ReverbTypeENS_8TimeSpanE) { Reverb *rv = plain_of(ARG(0)); if (rv) rv->p_pre_ns = (int64_t)ARG(1); } +HLE(_ZN2nn5audio16SetReverbEnabledEPNS0_10ReverbTypeEb) { + Reverb *rv = plain_of(ARG(0)); + if (rv) rv->enabled = (int)(ARG(1) & 1); + AUT("audio: plain reverb %s", ARG(1) & 1 ? "on" : "off"); +} +HLE(_ZN2nn5audio15IsReverbEnabledEPKNS0_10ReverbTypeE) { Reverb *rv = plain_of(ARG(0)); RET(rv ? rv->enabled : 0); } +HLE(_ZN2nn5audio20SetReverbInputOutputEPNS0_10ReverbTypeEPKaS4_i) { + Reverb *rv = plain_of(ARG(0)); + if (!rv) return; + const int n = (int)ARG(3) > MAX_CH ? MAX_CH : (int)ARG(3) < 0 ? 0 : (int)ARG(3); + for (int k = 0; k < n; k++) { + rv->in[k] = *(int8_t *)GP(ARG(1) + k); + rv->out[k] = *(int8_t *)GP(ARG(2) + k); + } + rv->n = n; +} +HLE(_ZN2nn5audio20GetReverbInputOutputEPKNS0_10ReverbTypeEPaS4_Pii) { + Reverb *rv = plain_of(ARG(0)); + const int n = rv ? rv->n : 0; + for (int k = 0; k < n && k < (int)ARG(4); k++) { + *(int8_t *)GP(ARG(1) + k) = rv->in[k]; + *(int8_t *)GP(ARG(2) + k) = rv->out[k]; + } + if (ARG(3)) G32(ARG(3)) = (uint32_t)n; +} + +/* ---- biquad filter: AddBiquadFilter(config, BiquadFilterType*, SubMixType* / FinalMixType*) -------------------- */ +static Biquad *biquad_of(uint64_t obj) { + const int i = get_handle(obj, K_BIQUAD); + return i >= 0 && i < MAX_FX && biquads[i].used ? &biquads[i] : NULL; +} +static void add_biquad(Ctx *c) { + au_lock(); + int e = 0; + while (e < MAX_FX && biquads[e].used) e++; + const int m = mix_index(ARG(2)); + if (e == MAX_FX || m < 0) { + au_unlock(); + AUT("audio: biquad filter refused (mix %d)", m); + RET(0x1A9 << 9 | 153); + return; + } + Biquad *f = &biquads[e]; + memset(f, 0, sizeof *f); + f->used = 1; + f->mix = m; + f->seq = ++fx_seq; + f->n = mixes[m].buffers < MAX_CH ? mixes[m].buffers : MAX_CH; /* (until SetBiquadFilterInputOutput) */ + for (int k = 0; k < MAX_CH; k++) f->in[k] = f->out[k] = (int8_t)k; + f->b[0] = 1.0f; + put_handle(ARG(1), K_BIQUAD, e); + au_unlock(); + AUT("audio: biquad filter %d on mix %d", e, m); + RET(0); +} +HLE(_ZN2nn5audio15AddBiquadFilterEPNS0_19AudioRendererConfigEPNS0_16BiquadFilterTypeEPNS0_10SubMixTypeE) { add_biquad(c); } +HLE(_ZN2nn5audio15AddBiquadFilterEPNS0_19AudioRendererConfigEPNS0_16BiquadFilterTypeEPNS0_12FinalMixTypeE) { add_biquad(c); } +static void remove_biquad(Ctx *c) { + au_lock(); + Biquad *f = biquad_of(ARG(1)); + if (f) f->used = 0; + au_unlock(); + if (f) G64(ARG(1)) = 0; +} +HLE(_ZN2nn5audio18RemoveBiquadFilterEPNS0_19AudioRendererConfigEPNS0_16BiquadFilterTypeEPNS0_10SubMixTypeE) { remove_biquad(c); } +HLE(_ZN2nn5audio18RemoveBiquadFilterEPNS0_19AudioRendererConfigEPNS0_16BiquadFilterTypeEPNS0_12FinalMixTypeE) { remove_biquad(c); } +HLE(_ZN2nn5audio26SetBiquadFilterInputOutputEPNS0_16BiquadFilterTypeEPKaS4_i) { + au_lock(); + Biquad *f = biquad_of(ARG(0)); + if (f) { + const int n = (int)ARG(3) > MAX_CH ? MAX_CH : (int)ARG(3) < 0 ? 0 : (int)ARG(3); + for (int k = 0; k < n; k++) { + f->in[k] = *(int8_t *)GP(ARG(1) + k); + f->out[k] = *(int8_t *)GP(ARG(2) + k); + } + f->n = n; + AUT("audio: biquad filter %d channels: %d", (int)(f - biquads), n); + } + au_unlock(); +} +/* TBiquadFilterParameter: bool @0, numerator[3] @4, additive-feedback denominator[2] @16 (as the voices'). + The filter runs while both it (SetBiquadFilterEnabled) and its parameter are on. */ +HLE(_ZN2nn5audio24SetBiquadFilterParameterEPNS0_16BiquadFilterTypeERKNS0_22TBiquadFilterParameterIfEE) { + au_lock(); + Biquad *f = biquad_of(ARG(0)); + const uint64_t p = ARG(1); + if (f && p) { + const int on = *(uint8_t *)GP(p) != 0; + if (on && !f->on) memset(f->z, 0, sizeof f->z); + f->on = on; + memcpy(f->b, GP(p + 4), sizeof f->b); + memcpy(f->a, GP(p + 16), sizeof f->a); + static int logged; + if (on && logged++ < 8) + AUT("audio: biquad filter %d: b %.3f %.3f %.3f a %.3f %.3f", (int)(f - biquads), f->b[0], f->b[1], f->b[2], + f->a[0], f->a[1]); + } + au_unlock(); +} +HLE(_ZN2nn5audio22SetBiquadFilterEnabledEPNS0_16BiquadFilterTypeEb) { + au_lock(); + Biquad *f = biquad_of(ARG(0)); + if (f) { + if ((ARG(1) & 1) && !f->enabled) memset(f->z, 0, sizeof f->z); + f->enabled = (int)(ARG(1) & 1); + } + au_unlock(); +} +HLE(_ZN2nn5audio21IsBiquadFilterEnabledEPKNS0_16BiquadFilterTypeE) { Biquad *f = biquad_of(ARG(0)); RET(f ? f->enabled : 0); } + +/* ---- delay: AddDelay(config, DelayType*, buffer, size, SubMixType* / FinalMixType*, TimeSpan delayTimeMax, + channels). The lines are ours, taken here (the game's buffer is not used); until the game sets its gains the + delay passes the dry signal alone. There is no out gain to set: the line's output is added at 1. */ +static Delay *delay_of(uint64_t obj) { + const int i = get_handle(obj, K_DELAY); + return i >= 0 && i < MAX_FX && delays[i].used ? &delays[i] : NULL; +} +static void add_delay(Ctx *c) { + const int64_t max_ns = (int64_t)ARG(5); + const int n = (int)ARG(6) < 1 ? 1 : (int)ARG(6) > DELAY_CH ? DELAY_CH : (int)ARG(6); + const double frames = (double)max_ns * RATE / 1e9; + const int cap = frames < 1 ? 1 : frames < DELAY_MAX ? (int)frames + 1 : DELAY_MAX; + float *line = (float *)calloc((size_t)cap * n, sizeof *line); /* (on the game's thread, before the lock) */ + au_lock(); + int e = 0; + while (e < MAX_FX && delays[e].used) e++; + const int m = mix_index(ARG(4)); + if (e == MAX_FX || m < 0) { + au_unlock(); + free(line); + AUT("audio: delay refused (mix %d)", m); + RET(0x1A9 << 9 | 153); + return; + } + Delay *dl = &delays[e]; + float *old = dl->line; + memset(dl, 0, sizeof *dl); + dl->used = 1; + dl->mix = m; + dl->seq = ++fx_seq; + dl->n = n; + for (int k = 0; k < DELAY_CH; k++) dl->in[k] = dl->out[k] = (int8_t)k; + dl->line = line; + dl->cap = line ? cap : 0; + dl->max_ns = dl->time_ns = max_ns; + dl->dry = 1.0f; + dl->buffer = ARG(2); + put_handle(ARG(1), K_DELAY, e); + au_unlock(); + free(old); + AUT("audio: delay %d on mix %d, %d channels, %.0f ms at most", e, m, n, (double)max_ns / 1e6); + RET(0); +} +HLE(_ZN2nn5audio8AddDelayEPNS0_19AudioRendererConfigEPNS0_9DelayTypeEPvmPNS0_10SubMixTypeENS_8TimeSpanEi) { add_delay(c); } +HLE(_ZN2nn5audio8AddDelayEPNS0_19AudioRendererConfigEPNS0_9DelayTypeEPvmPNS0_12FinalMixTypeENS_8TimeSpanEi) { add_delay(c); } +static void remove_delay(Ctx *c) { /* (RemoveDelay(config, DelayType*, mix): the work buffer back) */ + au_lock(); + Delay *dl = delay_of(ARG(1)); + uint64_t buffer = 0; + float *line = NULL; + if (dl) dl->used = 0, buffer = dl->buffer, line = dl->line, dl->line = NULL, dl->cap = 0; + au_unlock(); + free(line); + if (dl) G64(ARG(1)) = 0; + RET(buffer); +} +HLE(_ZN2nn5audio11RemoveDelayEPNS0_19AudioRendererConfigEPNS0_9DelayTypeEPNS0_10SubMixTypeE) { remove_delay(c); } +HLE(_ZN2nn5audio11RemoveDelayEPNS0_19AudioRendererConfigEPNS0_9DelayTypeEPNS0_12FinalMixTypeE) { remove_delay(c); } +HLE(_ZN2nn5audio16IsDelayRemovableEPKNS0_9DelayTypeE) { RET(1); } +HLE(_ZN2nn5audio15SetDelayEnabledEPNS0_9DelayTypeEb) { + au_lock(); + Delay *dl = delay_of(ARG(0)); + if (dl) { + if ((ARG(1) & 1) && !dl->enabled && dl->line) { /* (switched on: the lines start empty) */ + memset(dl->line, 0, (size_t)dl->cap * dl->n * sizeof *dl->line); + memset(dl->lp, 0, sizeof dl->lp); + dl->pos = 0; + } + dl->enabled = (int)(ARG(1) & 1); + } + au_unlock(); + AUT("audio: delay %s", ARG(1) & 1 ? "on" : "off"); +} +HLE(_ZN2nn5audio14IsDelayEnabledEPKNS0_9DelayTypeE) { Delay *dl = delay_of(ARG(0)); RET(dl ? dl->enabled : 0); } +HLE(_ZN2nn5audio12SetDelayTimeEPNS0_9DelayTypeENS_8TimeSpanE) { + Delay *dl = delay_of(ARG(0)); + if (dl) dl->time_ns = (int64_t)ARG(1) < 0 ? 0 : (int64_t)ARG(1) > dl->max_ns ? dl->max_ns : (int64_t)ARG(1); +} +HLE(_ZN2nn5audio12GetDelayTimeEPKNS0_9DelayTypeE) { Delay *dl = delay_of(ARG(0)); RET(dl ? (uint64_t)dl->time_ns : 0); } +HLE(_ZN2nn5audio14SetDelayInGainEPNS0_9DelayTypeEf) { Delay *dl = delay_of(ARG(0)); if (dl) dl->in_gain = farg(c, 0); } +HLE(_ZN2nn5audio20SetDelayFeedbackGainEPNS0_9DelayTypeEf) { Delay *dl = delay_of(ARG(0)); if (dl) dl->feedback = farg(c, 0); } +HLE(_ZN2nn5audio15SetDelayDryGainEPNS0_9DelayTypeEf) { Delay *dl = delay_of(ARG(0)); if (dl) dl->dry = farg(c, 0); } +HLE(_ZN2nn5audio21SetDelayChannelSpreadEPNS0_9DelayTypeEf) { Delay *dl = delay_of(ARG(0)); if (dl) dl->spread = farg(c, 0); } +HLE(_ZN2nn5audio21SetDelayLowPassAmountEPNS0_9DelayTypeEf) { Delay *dl = delay_of(ARG(0)); if (dl) dl->lowpass = farg(c, 0); } +HLE(_ZN2nn5audio19SetDelayInputOutputEPNS0_9DelayTypeEPKaS4_i) { + au_lock(); + Delay *dl = delay_of(ARG(0)); + if (dl) { /* (no more channels than AddDelay took lines for) */ + const int n = (int)ARG(3) > dl->n ? dl->n : (int)ARG(3) < 0 ? 0 : (int)ARG(3); + for (int k = 0; k < n; k++) { + dl->in[k] = *(int8_t *)GP(ARG(1) + k); + dl->out[k] = *(int8_t *)GP(ARG(2) + k); + } + dl->n = n; + AUT("audio: delay %d channels: %d", (int)(dl - delays), n); + } + au_unlock(); +} diff --git a/host/nn_fs.c b/host/nn_fs.c index 0a86623..05e19bb 100644 --- a/host/nn_fs.c +++ b/host/nn_fs.c @@ -7,6 +7,7 @@ hos_romfs_dir and hos_user_dir are set by hos_main.c. Reads are positional (ReadFile with an offset), so any number of guest threads can read one file at once. Results: switchbrew "Error codes" (module 2 = fs). */ #include "hos.h" +#include "stall_host.h" /* ((Triforce, 11-22, OS-51) the host's part of a long frame, counted where it is spent) */ #include #include #include @@ -532,8 +533,10 @@ HLE(_ZN2nn2fs6CommitEPKc) { if (i < ntx) { const int away = hos_vcore_away(); const ULONGLONG t0 = GetTickCount64(); + const int64_t stall0 = stall_host_begin(); /* ((Triforce, OS-51) the host's part of a long frame: stall_host.h) */ tx_wait_locked(); /* (the Commit before this one: its staging folder is this one's) */ const int staged = save_tx_stage(&tx_mounts[i].p); + stall_host_end(STALL_HOST_COMMIT, stall0, 0); if (staged == 1) { tx_finish_paths = tx_mounts[i].p; tx_thread = CreateThread(NULL, 0, tx_finish_main, NULL, 0, NULL); @@ -562,9 +565,27 @@ HLE(_ZN2nn2fs8OpenFileEPNS0_10FileHandleEPKci) { char path[HOS_FS_PATH]; int writable = 0; const char *g = gstr(ARG(1)); - if (!host_path(g, path, sizeof path, &writable)) { hos_log("fs: OpenFile(%s): unknown mount", g); RET(R_PATH_NOT_FOUND); return; } + /* ((Triforce, 11-21, the performance audit's IO-24, after Source A's) The virtual core is given up from here, not only + around CreateFileA below: host_path reaches the button prompts' builder (host/prompts.c hos_prompts_romfs), which + builds a file at its first open, a quarter of a second and more for the font or a texture, while the other openers + wait for it in Sleep(1); none of them holds a core now. Only host code runs from here to the open: the guest's path + is read above, and every return takes the core back before it answers the game. HOS_OPEN_KEEP_CORE=1: given up + around CreateFileA alone, as before */ + static int open_keep_core = -1; + if (open_keep_core < 0) open_keep_core = getenv("HOS_OPEN_KEEP_CORE") != NULL; + int away = open_keep_core ? 0 : hos_vcore_away(); + if (!host_path(g, path, sizeof path, &writable)) { + hos_log("fs: OpenFile(%s): unknown mount", g); + if (away) hos_vcore_back(); + RET(R_PATH_NOT_FOUND); + return; + } int want_write = (ARG(2) & 6) != 0; - if (want_write && !writable) { RET(R_PATH_NOT_FOUND); return; } + if (want_write && !writable) { + if (away) hos_vcore_back(); + RET(R_PATH_NOT_FOUND); + return; + } { void hos_bench_file_opened(const char *guest); hos_bench_file_opened(g); } /* ((Triforce) benchmark mode: which save file loads */ /* (mkw-recomp) HOS_NO_MOVIES=1: MKW's WebM movies (cup openings, billboards, emblems) are not found, and the game goes on without them (decoded by nn_video.c otherwise) */ @@ -573,6 +594,7 @@ HLE(_ZN2nn2fs8OpenFileEPNS0_10FileHandleEPKci) { if (n > 5 && !_stricmp(g + n - 5, ".webm")) { static int logged; if (!logged++) hos_log("fs: OpenFile(%s): movies are left out (HOS_NO_MOVIES)", g); + if (away) hos_vcore_back(); RET(R_PATH_NOT_FOUND); return; } @@ -580,7 +602,11 @@ HLE(_ZN2nn2fs8OpenFileEPNS0_10FileHandleEPKci) { { int hos_ultrawide_redirect(const char *guest, char *out, size_t cap); /* (totk_ultrawide_ui.c: the HUD) */ char alt[HOS_FS_PATH]; if (!want_write && hos_ultrawide_redirect(g, alt, sizeof alt)) snprintf(path, sizeof path, "%s", alt); } - if (!want_write && hos_blank_cached_icon(g, path)) { RET(R_PATH_NOT_FOUND); return; } + if (!want_write && hos_blank_cached_icon(g, path)) { + if (away) hos_vcore_back(); + RET(R_PATH_NOT_FOUND); + return; + } { /* HOS_FS_OVERLAY= (a test): a file of the game's data found there is read in place of the game's own */ const char *ov = getenv("HOS_FS_OVERLAY"); const char *rel = !want_write && ov ? strstr(g, ":/") : NULL; @@ -595,7 +621,7 @@ HLE(_ZN2nn2fs8OpenFileEPNS0_10FileHandleEPKci) { } /* (mkw-recomp 16bc3bc: the host's file calls block, so the virtual core goes to another thread meanwhile, as Horizon would run another while this one waits for the file system. OoT's ARes loaders held it up to 250 ms) */ - const int away = hos_vcore_away(); + if (open_keep_core) away = hos_vcore_away(); /* ((Triforce, 11-21, IO-24) otherwise given up already, above) */ HANDLE h = CreateFileA(path, GENERIC_READ | (want_write ? GENERIC_WRITE : 0), FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); if (away) hos_vcore_back(); @@ -629,6 +655,10 @@ static uint64_t read_at(Ctx *c, uint64_t id, int64_t off, uint64_t buf, uint64_t hos_sleep_ns(delay_ns); } const int away = hos_vcore_away(); /* (as OpenFile: no core held through the read) */ + const int64_t stall0 = stall_host_begin(); /* ((Triforce, OS-51) the read itself, not the wait for the core after it) */ + /* ((Triforce, 11-21, MEM-09) the kernel's ReadFile does not fault pages in: heap pages nothing touched yet (committed + as they are touched now, heap_commit.h) are touched for it first) */ + hos_guest_touch(buf, size); const int io = hos_guest_io_begin(GP(buf), size); /* (Triforce: the kernel writes there: rx_backend.c) */ while (done < size) { OVERLAPPED ov = {0}; @@ -640,6 +670,7 @@ static uint64_t read_at(Ctx *c, uint64_t id, int64_t off, uint64_t buf, uint64_t done += got; } hos_guest_io_end(io); + stall_host_end(STALL_HOST_READ, stall0, done); if (away) hos_vcore_back(); HOS_WRITTEN(GP(buf), (size_t)done); /* (into guest memory: the GPU may read it) */ { /* HOS_FS_WATCH=: (debugging): reads into that guest range, with the file */ @@ -683,9 +714,12 @@ HLE(_ZN2nn2fs9WriteFileENS0_10FileHandleElPKvmRKNS0_11WriteOptionE) { DWORD put = 0; /* (hookshot) from guest memory only, and the whole size: a DWORD cut wrote less than asked and said so */ if (ARG(3) <= 0xFFFFFFFFu && hos_guest_range(ARG(2), ARG(3))) { + const int64_t stall0 = stall_host_begin(); /* ((Triforce, OS-51) the host's part of a long frame: stall_host.h) */ + hos_guest_touch(ARG(2), ARG(3)); /* ((Triforce, 11-21, MEM-09) as the read: touched before the kernel reads them) */ const int io = hos_guest_io_begin(GP(ARG(2)), ARG(3)); /* (Triforce: the kernel reads there: rx_backend.c) */ WriteFile(fh->h, GP(ARG(2)), (DWORD)ARG(3), &put, &ov); hos_guest_io_end(io); + stall_host_end(STALL_HOST_WRITE, stall0, put); } RET(put == ARG(3) ? 0 : 0x2EE202); } @@ -697,7 +731,12 @@ HLE(_ZN2nn2fs11SetFileSizeENS0_10FileHandleEl) { SetEndOfFile(fh->h); RET(0); } -HLE(_ZN2nn2fs9FlushFileENS0_10FileHandleE) { FlushFileBuffers(get_handle(c, ARG(0))->h); RET(0); } +HLE(_ZN2nn2fs9FlushFileENS0_10FileHandleE) { + const int64_t stall0 = stall_host_begin(); /* ((Triforce, OS-51) the host's part of a long frame: stall_host.h) */ + FlushFileBuffers(get_handle(c, ARG(0))->h); + stall_host_end(STALL_HOST_FLUSH, stall0, 0); + RET(0); +} HLE(_ZN2nn2fs9CloseFileENS0_10FileHandleE) { Handle *fh = get_handle(c, ARG(0)); CloseHandle(fh->h); diff --git a/host/nn_hid.c b/host/nn_hid.c index 6074c08..8545371 100644 --- a/host/nn_hid.c +++ b/host/nn_hid.c @@ -963,7 +963,6 @@ static int flight_jump_shortcut(uint64_t *buttons, int mode, int menu_open, int static void read_input(uint64_t *buttons, int32_t stick[4]) { *buttons = 0; memset(stick, 0, 4 * sizeof *stick); - void hos_mouse_native_stick(int held); scripted_input(buttons, stick); /* ((Triforce, 11-19) benchmark mode (host/hos_bench.c) drives the game itself: its presses and stick alone, with the window in front or behind other windows; the player's devices do not count while it runs (a mouse or a pad would move the camera @@ -1090,9 +1089,6 @@ static void read_input(uint64_t *buttons, int32_t stick[4]) { controller's stick and the scripted presses are as they are) */ if (walk_vk && key(walk_vk)) walk_keys(keys, rx_option(HOS_OPT_WALK_SCALE, 51)); stick_max(stick, keys); - /* (hookshot, mouse_native.c) the keys and the pads are the right stick too: while they turn the camera, its angles are - the native mouse's new start, so the two do not undo each other */ - hos_mouse_native_stick(stick[2] != 0 || stick[3] != 0); } static void vibration_send(uint64_t handle, uint64_t value); @@ -1212,6 +1208,13 @@ static void npad_states(Ctx *c) { } if (connected) { read_input(&buttons, stick); + /* (mouse_native.c) the keys, the pads and the scripted presses are the right stick too: while they turn the camera, + its angles are the native mouse's new start, so the two do not undo each other. ((Triforce, 11-22) Told after + every read, not at the end of the window-in-front one alone: with the window behind another and Controller in + background on, the pad's turns were put back each frame and the camera moved a frame's worth (#bug-tracker, + 11-21); the benchmark's camera turn too) */ + void hos_mouse_native_stick(int held); + hos_mouse_native_stick(stick[2] != 0 || stick[3] != 0); mods_input(&buttons, stick); /* (Triforce) the Mods entry's: below */ { /* (Triforce) the sprint button's press is its own while Link plays (host/oot_sprint.c) */ void hos_oot_sprint_pad(uint64_t *buttons); diff --git a/host/nn_os_sync.c b/host/nn_os_sync.c index d6d6351..0a2f660 100644 --- a/host/nn_os_sync.c +++ b/host/nn_os_sync.c @@ -56,11 +56,17 @@ static SRWLOCK table_lock = SRWLOCK_INIT; static Obj **table; static uint64_t *table_keys; /* each slot's object's key, beside it: a probe compares keys without visiting objects */ static uint64_t cap, used; -typedef struct Table { /* (the slots, then the keys: cap of each) */ - uint64_t cap; +/* TICK-062 A4/B7, HOS_OBJ_PACK=1: the slot pointer and its key side by side (stride 2: {Obj *, key} pairs, 16 bytes) instead of + the pointer array and then the key array (stride 1): a probe touches one cache line, not two, before the object itself. + The same values at the same slot numbers, so every lookup answers as before. */ +static uint64_t tstride = 1; +typedef struct Table { /* (stride 1: the slots, then the keys, cap of each; stride 2: cap pairs) */ + uint64_t cap, stride; Obj *slot[]; } Table; -static uint64_t *table_keys_of(const Table *t) { return (uint64_t *)(void *)(t->slot + t->cap); } +static uint64_t *table_keys_of(const Table *t) { + return t->stride == 2 ? (uint64_t *)(void *)t->slot + 1 : (uint64_t *)(void *)(t->slot + t->cap); +} static Table *live; static uint64_t slot_of(uint64_t key, uint64_t c) { return (key * 0x9E3779B97F4A7C15ULL >> 20) & (c - 1); } @@ -68,8 +74,8 @@ static uint64_t slot_of(uint64_t key, uint64_t c) { return (key * 0x9E3779B97F4A static Obj *find_locked(uint64_t key) { if (!table) return NULL; for (uint64_t i = slot_of(key, cap);; i = (i + 1) & (cap - 1)) { - if (!table[i]) return NULL; - if (table_keys[i] == key && table[i]->kind) return table[i]; + if (!table[i * tstride]) return NULL; + if (table_keys[i * tstride] == key && table[i * tstride]->kind) return table[i * tstride]; } } @@ -135,40 +141,43 @@ static void bury_locked(Obj *o) { static Obj *find_dead_locked(uint64_t key) { if (!table) return NULL; for (uint64_t i = slot_of(key, cap);; i = (i + 1) & (cap - 1)) { - if (!table[i]) return NULL; - if (table_keys[i] == key && !table[i]->kind) return table[i]; + if (!table[i * tstride]) return NULL; + if (table_keys[i * tstride] == key && !table[i * tstride]->kind) return table[i * tstride]; } } static void insert_locked(Obj *o) { if (!table || used * 2 >= cap) { uint64_t ncap = cap ? cap * 2 : 1 << 16; + if (!cap) tstride = getenv("HOS_OBJ_PACK") ? 2 : 1; + const uint64_t ns = tstride; Table *t = (Table *)calloc(1, sizeof(Table) + ncap * (sizeof(Obj *) + sizeof(uint64_t))); Obj **nt = t->slot; t->cap = ncap; + t->stride = ns; uint64_t *nk = table_keys_of(t); for (uint64_t i = 0; i < cap; i++) - if (table[i] && table[i]->kind) { - uint64_t j = slot_of(table[i]->key, ncap); - while (nt[j]) j = (j + 1) & (ncap - 1); - nt[j] = table[i]; - nk[j] = table[i]->key; - } else if (table[i]) { - bury_locked(table[i]); + if (table[i * tstride] && table[i * tstride]->kind) { + uint64_t j = slot_of(table[i * tstride]->key, ncap); + while (nt[j * ns]) j = (j + 1) & (ncap - 1); + nt[j * ns] = table[i * tstride]; + nk[j * ns] = table[i * tstride]->key; + } else if (table[i * tstride]) { + bury_locked(table[i * tstride]); } table = nt; table_keys = nk; cap = ncap; used = 0; - for (uint64_t i = 0; i < cap; i++) used += table[i] != NULL; + for (uint64_t i = 0; i < cap; i++) used += table[i * tstride] != NULL; __atomic_store_n(&live, t, __ATOMIC_RELEASE); } uint64_t i = slot_of(o->key, cap); - while (table[i] && table[i]->kind) i = (i + 1) & (cap - 1); - if (!table[i]) used++; - else bury_locked(table[i]); /* (a tombstone of another address) */ - __atomic_store_n(&table_keys[i], o->key, __ATOMIC_RELAXED); /* (before the object: see find_unlocked) */ - __atomic_store_n(&table[i], o, __ATOMIC_RELEASE); + while (table[i * tstride] && table[i * tstride]->kind) i = (i + 1) & (cap - 1); + if (!table[i * tstride]) used++; + else bury_locked(table[i * tstride]); /* (a tombstone of another address) */ + __atomic_store_n(&table_keys[i * tstride], o->key, __ATOMIC_RELAXED); /* (before the object: see find_unlocked) */ + __atomic_store_n(&table[i * tstride], o, __ATOMIC_RELEASE); } static Obj *find_unlocked(uint64_t key, int kind) { @@ -176,10 +185,10 @@ static Obj *find_unlocked(uint64_t key, int kind) { if (!t) return NULL; const uint64_t *keys = table_keys_of(t); for (uint64_t i = slot_of(key, t->cap), n = 0; n < t->cap; i = (i + 1) & (t->cap - 1), n++) { - Obj *o = __atomic_load_n(&t->slot[i], __ATOMIC_ACQUIRE); + Obj *o = __atomic_load_n(&t->slot[i * t->stride], __ATOMIC_ACQUIRE); if (!o) return NULL; /* (the key beside the slot first; a slot changing meanwhile gives a miss, and the locked lookup answers) */ - if (__atomic_load_n(&keys[i], __ATOMIC_RELAXED) == key && o->key == key && + if (__atomic_load_n(&keys[i * t->stride], __ATOMIC_RELAXED) == key && o->key == key && __atomic_load_n(&o->kind, __ATOMIC_ACQUIRE) == kind) return o; } @@ -190,10 +199,109 @@ static Obj *obj_lookup(uint64_t key, int kind); /* Every nn::os call looks its object up by the guest address (~1.6% of the game threads' time at the castle: a probe of the shared table and a cache miss on a scattered object). Game threads use a few objects over and over: a small direct-mapped cache per thread answers those. An entry holds while its object is still that address's and of the - kind asked for (objects are reused, never freed: spares). HOS_NO_OBJ_CACHE=1: off. */ + kind asked for (objects are reused, never freed: spares). HOS_NO_OBJ_CACHE=1: off. + + (Triforce) The direct-mapped cache still missed at every step of a walk over many objects at a fixed stride: the game's + record pass under the graphics front end (f_80364080) locks and unlocks some 4,000 per-slot mutexes 0x80 apart, and the + multiply-shift index folds such keys onto few entries (3,541 of 4,000 missed a pass with 4,096 entries). So by default + the cache is 16,384 entries, four ways to a 64-byte set, the set picked by murmur3's finalizer, the oldest way replaced + (a stale way of the same key in place); a hit is checked by key and kind as before. The table is allocated at a + thread's first lookup (256 KB, kept for the thread's life); without memory, the shared table answers. Each thread counts + its lookups and misses; after 65,536 lookups a miss asks obj (never obj_nockpt, which runs under host locks) to add them + up, and every 10 s the log says "sync: object cache". HOS_OBJ_CACHE=: n entries (256 to 65,536, a power of two); + 0: the direct-mapped 4,096 entries, as before. */ +typedef struct { uint64_t key; Obj *o; } ObjWay; +enum { OBJ_CACHE_WAYS = 4, OBJ_CACHE_DEFAULT = 16384, OBJ_CACHE_DUE = 65536 }; +static _Thread_local uint32_t obj_cache_lookups, obj_cache_missed; +static _Thread_local int obj_cache_due; +/* HOS_OBJ_CACHE: unset, empty or not a number 16,384; 0 the direct-mapped cache; else 256 to 65,536, down to a power of two */ +static int obj_cache_entries(void) { + static int n = -1; + if (n < 0) { + const char *e = getenv("HOS_OBJ_CACHE"); + long v = -1; + if (e && *e) { + v = 0; + for (const char *p = e; *p; p++) { + if (*p < '0' || *p > '9') { v = -1; break; } + if (v < 1000000) v = v * 10 + (*p - '0'); + } + } + int r = OBJ_CACHE_DEFAULT; + if (v == 0) r = 0; + else if (v > 0) { + if (v > 65536) v = 65536; + r = 256; + while ((long)r * 2 <= v) r *= 2; + } + n = r; + } + return n; +} +/* The four-way cache (above); entries: obj_cache_entries's, not 0 */ +static Obj *obj_lookup_cached(uint64_t key, int kind, int entries) { + static _Thread_local ObjWay *ways; + static _Thread_local int no_memory; + if (!ways) { + if (no_memory) return obj_lookup(key, kind); + char *raw = (char *)calloc((size_t)entries + OBJ_CACHE_WAYS, sizeof(ObjWay)); /* (one set more: aligned to 64 bytes) */ + if (!raw) { + no_memory = 1; + return obj_lookup(key, kind); + } + ways = (ObjWay *)(((uintptr_t)raw + 63) & ~(uintptr_t)63); + } + uint64_t h = key; /* (murmur3's 64-bit finalizer) */ + h ^= h >> 33; + h *= 0xff51afd7ed558ccdULL; + h ^= h >> 33; + h *= 0xc4ceb9fe1a85ec53ULL; + h ^= h >> 33; + ObjWay *set = ways + (h & (uint64_t)(entries / OBJ_CACHE_WAYS - 1)) * OBJ_CACHE_WAYS; + obj_cache_lookups++; + for (int w = 0; w < OBJ_CACHE_WAYS; w++) { + Obj *o = set[w].o; + if (set[w].key == key && o && __atomic_load_n(&o->key, __ATOMIC_RELAXED) == key && + __atomic_load_n(&o->kind, __ATOMIC_ACQUIRE) == kind) + return o; + } + obj_cache_missed++; + if (obj_cache_lookups >= OBJ_CACHE_DUE) obj_cache_due = 1; + Obj *o = obj_lookup(key, kind); + int w = 0; + while (w < OBJ_CACHE_WAYS && set[w].key != key) w++; + if (w == OBJ_CACHE_WAYS) { /* (the oldest way, the last, goes; the newest is the first) */ + memmove(set + 1, set, (OBJ_CACHE_WAYS - 1) * sizeof *set); + w = 0; + } + set[w].key = key, set[w].o = o; + return o; +} +/* (obj, outside every host lock) this thread's counts into the process's; every 10 s the log line */ +static void obj_cache_flush(void) { + static uint64_t lookups, missed; + static int64_t said; + obj_cache_due = 0; + __atomic_fetch_add(&lookups, obj_cache_lookups, __ATOMIC_RELAXED); + __atomic_fetch_add(&missed, obj_cache_missed, __ATOMIC_RELAXED); + obj_cache_lookups = obj_cache_missed = 0; + const int64_t now = hos_now_ns(); + int64_t was = __atomic_load_n(&said, __ATOMIC_RELAXED); + if (!was) { + __atomic_compare_exchange_n(&said, &was, now, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED); + return; + } + if (now - was < 10000000000LL || !__atomic_compare_exchange_n(&said, &was, now, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) + return; + const uint64_t l = __atomic_exchange_n(&lookups, 0, __ATOMIC_RELAXED), m = __atomic_exchange_n(&missed, 0, __ATOMIC_RELAXED); + hos_log("sync: object cache (%d entries, %d ways a set; HOS_OBJ_CACHE): %llu lookups, %llu missed (%.2f %%) in %.0f s", + obj_cache_entries(), OBJ_CACHE_WAYS, (unsigned long long)l, (unsigned long long)m, l ? 100.0 * m / l : 0.0, + (now - was) / 1e9); +} static Obj *obj_nockpt(uint64_t key, int kind); static Obj *obj(uint64_t key, int kind) { hos_vcore_checkpoint(); /* (virtual cores: every nn::os call, where a better waiter takes the core) */ + if (obj_cache_due) obj_cache_flush(); return obj_nockpt(key, kind); } /* obj without the checkpoint, for lookups under a host lock: a checkpoint there can give the core up and wait for one @@ -202,11 +310,13 @@ static Obj *obj(uint64_t key, int kind) { static Obj *obj_nockpt(uint64_t key, int kind) { /* (hookshot: 128 still missed: obj_lookup was the busiest leaf in dense Kokiri, 3.2% of all samples, 4.3% of the instants when one thread alone ran; each actor has its own mutexes) */ - static _Thread_local struct { uint64_t key; Obj *o; } cache[1024]; + static _Thread_local struct { uint64_t key; Obj *o; } cache[4096]; /* ((Triforce, 11-21) 4096, was 1024: obj_lookup still 1.9% of the game threads */ static int off = -1; if (off < 0) off = getenv("HOS_NO_OBJ_CACHE") != NULL; if (!off) { - const unsigned i = (unsigned)((key * 0x9E3779B97F4A7C15ULL) >> 54); + const int entries = obj_cache_entries(); + if (entries) return obj_lookup_cached(key, kind, entries); + const unsigned i = (unsigned)((key * 0x9E3779B97F4A7C15ULL) >> 52); /* (12 bits: 4096 entries) */ Obj *o = cache[i].o; if (cache[i].key == key && o && __atomic_load_n(&o->key, __ATOMIC_RELAXED) == key && __atomic_load_n(&o->kind, __ATOMIC_ACQUIRE) == kind) @@ -336,7 +446,8 @@ static int64_t spin_ns(void) { static int64_t ns = -1; if (ns < 0) { const char *e = getenv("HOS_SPIN_US"); - ns = (e ? atoll(e) : 20) * 1000; + ns = (e ? atoll(e) : 40) * 1000; /* ((Triforce, 11-21) 40 us, was 20: a spin now gives its core to a waiter that asks + (hos_vcore_asked), so a longer one blocks nobody. Lab A/B 79.7 -> 80.4, every round) */ } return ns; } @@ -367,6 +478,7 @@ static void spin_until_set(Obj *o, volatile int *flag) { spin_end(o, misses, 0); return; } + if (hos_vcore_asked()) return; /* ((Triforce, 11-21) a waiter wants this core: to the sleep, which gives it up */ } } static void spin_until_free(Obj *o) { @@ -385,6 +497,7 @@ static void spin_until_free(Obj *o) { spin_end(o, misses, 0); return; } + if (hos_vcore_asked()) return; /* ((Triforce, 11-21) a waiter wants this core: to the sleep, which gives it up */ } } #else @@ -448,12 +561,12 @@ static void cv_enter(Obj *cv, Obj *m) { if (cv->cv_waiting && cv->cv_mutex != m) cv->cv_mixed = 1; if (!cv->cv_waiting) cv->cv_mixed = 0; cv->cv_mutex = m; - cv->cv_waiting++; + __atomic_add_fetch(&cv->cv_waiting, 1, __ATOMIC_SEQ_CST); } /* a waiter out (cv->lk held): whether a signal reserved its mutex m for it. A reservation made on another mutex (a waiter with a different one came in after the signal) is given back: never left owed to nobody */ static int cv_leave(Obj *cv, Obj *m) { - cv->cv_waiting--; + __atomic_sub_fetch(&cv->cv_waiting, 1, __ATOMIC_SEQ_CST); if (cv->cv_owed <= 0) return 0; Obj *r = cv->cv_owed_mutex[--cv->cv_owed]; if (r == m) return 1; @@ -578,6 +691,18 @@ HLE(_ZN2nn2os25FinalizeConditionVariableEPNS0_21ConditionVariableTypeE) { drop(A HLE(_ZN2nn2os23SignalConditionVariableEPNS0_21ConditionVariableTypeE) { Obj *cv = obj(ARG(0), K_COND); if (watched_cv(ARG(0))) watch_log("signal", watch_mutex(), ARG(0), c, NULL); + /* TICK-062 B1, HOS_CV_SIGNAL_FAST=1: a signal with no waiter changes nothing a later wait can see, so it returns before + taking cv->lk. No wake is lost: (1) a waiter registers (cv_waiting++, seq_cst) under cv->lk BEFORE it releases the guest + mutex (mutex_release_all) and before it sleeps, and reads cv->gen before registering. (2) A signal the guest orders + after a wait's registration (it took the mutex the waiter released, or any release/acquire chain from it) reads + cv_waiting with a seq_cst load after that, so it sees >= 1 and takes the slow path below, exactly as before. (3) A + signal not ordered with the registration (no mutex, truly concurrent) may read 0: that is the signal ordered before the + wait, which the slow path also loses (it bumps gen before the waiter reads gen, so the waiter sleeps on). (4) With no + waiter the slow path only bumps gen (compared only as a delta by waiters that registered later) and cv_reserve (a no-op + with cv_waiting <= cv_owed, owed being 0 with no waiter) and wakes nobody. The wake order of waiters is untouched. */ + static int fast = -1; + if (fast < 0) fast = !(getenv("HOS_CV_SIGNAL_FAST") && atoi(getenv("HOS_CV_SIGNAL_FAST")) == 0); /* ((Triforce) on by default since 11-21, lab +5.8%; =0: off */ + if (fast && !__atomic_load_n(&cv->cv_waiting, __ATOMIC_SEQ_CST)) return; AcquireSRWLockExclusive(&cv->lk); cv_reserve(cv); cv->gen++; @@ -862,6 +987,12 @@ HLE(_ZN2nn2os19TimedWaitLightEventEPNS0_14LightEventTypeENS_8TimeSpanE) { /* System events are created by other HLE APIs (vsync, audio device changes ...) with hos_system_event_init. */ void hos_system_event_init(uint64_t key, int autoclear) { event_init(key, 0, autoclear); } void hos_system_event_signal(uint64_t key) { event_signal(obj(key, K_EVENT)); } +/* ((Triforce, 11-22, the performance audit's IO-34, after Source A's) A system event cleared, and set, by an HLE that + holds a host lock of its own while it does (nn_video.c: the movie decoder's lock, so the event's state and the + decoder's count change together). Looked up without the virtual-core checkpoint: one there can give the core up and + wait for another with that lock held, while a thread that holds a core waits for the lock (see obj_nockpt) */ +void hos_system_event_clear(uint64_t key) { event_clear(obj_nockpt(key, K_EVENT)); } +void hos_system_event_signal_held(uint64_t key) { event_signal(obj_nockpt(key, K_EVENT)); } HLE(_ZN2nn2os18DestroySystemEventEPNS0_15SystemEventTypeE) { drop(ARG(0)); } HLE(_ZN2nn2os16ClearSystemEventEPNS0_15SystemEventTypeE) { event_clear(obj(ARG(0), K_EVENT)); } HLE(_ZN2nn2os15WaitSystemEventEPNS0_15SystemEventTypeE) { logged_wait(c, obj(ARG(0), K_EVENT), 0, 0, "system"); } diff --git a/host/nn_vi.c b/host/nn_vi.c index 31ad98b..4b48c11 100644 --- a/host/nn_vi.c +++ b/host/nn_vi.c @@ -5,6 +5,7 @@ #include "pace_wait.h" /* ((Triforce) the acquire's wait to the frame's time: pace_sleep) */ #include "step_steady.h" /* ((Triforce) HOS_STEP_STEADY=1: realtime_step) */ #include "pace_steady.h" /* ((Triforce) the Steady frame rate row: steady_period) */ +#include "movie_pace.h" /* ((Triforce, 11-22, PRESENT-30) paced at 240 while a movie plays at the 60 cap) */ #include "../gpu/vk/vk_api.h" /* (HOS_OPT_HDR_*: the Insert menu's HDR, answered by the renderer) */ #define WIN32_LEAN_AND_MEAN #include @@ -286,6 +287,32 @@ void hos_vi_native_acquire(int active) { const int64_t now = hos_now_ns(); if (until > now) pace_sleep(until, now); } +/* ((Triforce, 11-22, the performance audit's PRESENT-30, after Source A's) While a movie plays at a cap of exactly 60 the + frames are paced by 240, and the game's limiter and its time step follow as at a cap of 240 (movie_pace.h): the movie + player takes one picture an update at most, so at 60 every late frame was a picture the decoder then skipped. Decided + once a present (movie_pace_decide, from hos_vi_native_presented); pace_fps is what the pacing and realtime_step go by, + fps_max stays the cap itself, which the Steady row, the present interval and the other limiters' hand-off read. + Not with the Steady frame rate row on. HOS_MOVIE_PACE=0: the cap kept through movies, as before */ +static int movie_raise; +static int pace_fps(void) { + return movie_pace_fps(movie_raise, fps_max()); +} +static void movie_pace_decide(void) { + static int on = -1, steady = -1; + static long long raised; + if (on < 0) on = !(getenv("HOS_MOVIE_PACE") && *getenv("HOS_MOVIE_PACE") && atoi(getenv("HOS_MOVIE_PACE")) == 0); + if (steady < 0) { /* (the Steady frame rate row, read as steady_period reads it) */ + const char *e = getenv("HOS_FPS_STEADY"); + steady = e && *e ? atoi(e) : hos_ini_option("steady_frame_rate", PACE_STEADY_OFF); + if (steady < PACE_STEADY_OFF || steady > PACE_STEADY_REFRESH) steady = PACE_STEADY_OFF; + } + const int raise = movie_pace_raise(on, fps_max(), steady != PACE_STEADY_OFF, movie_plays()); + if (raise && !movie_raise) + hos_log("vi: a movie plays at the 60 cap: frames paced at %d while it does (HOS_MOVIE_PACE=0: the cap kept)", MOVIE_PACE_FPS); + if (!raise && movie_raise) hos_log("vi: the movie is over: frames paced at the 60 cap again, after %lld at %d", raised, MOVIE_PACE_FPS), raised = 0; + if (raise) raised++; + movie_raise = raise; +} /* (mkw-recomp) MKW's frame pacer (0x806bcbb0, x19 when it sets the interval: nvn_native.c): its measured frame time (+0x128, ticks), its running excess (+0x218) and interval (+0x214), logged with the frame rate */ volatile uint64_t hos_mkw_pacer; @@ -510,6 +537,16 @@ static void realtime_step(int64_t now) { } const uint64_t mod = hos_gfx_module; if (!mod) return; + /* ((Triforce, 11-22, the performance audit's PRESENT-30) what the frames are paced by: the cap, or 240 while a movie + plays at the 60 cap (pace_fps). The limiter, the step and the game's rate below follow it, as they followed the cap */ + const int pace = pace_fps(); + { /* (when that comes back down to 60: a rate above 60 left in the game's slot goes back with it, movie_pace.h) */ + static int before; + int *slot = (int *)GP(mod + 0x2c8); + const int was = *slot, back = movie_pace_slot(was, before, pace); + if (back != was) *slot = back; + before = pace; + } { /* (hookshot) The frame limiter (f_8042cd20 sleeps to 1 / +0xfc). The engine's adaptive controller writes +0xfc when variable refresh is on (+0x231, f_804fc0f0), any rate from 40 up. HOS_FPS_TARGET=: that rate kept, 0: left alone; HOS_FPS_LOG=1: what the game asks for, every 300 presents */ @@ -534,12 +571,16 @@ static void realtime_step(int64_t now) { clock_before = game_clock, at_before = now; } if (target && *limit > 0 && *limit < target) *limit = target; - if (fps_max() > 60 && *limit > 0 && *limit < fps_max()) *limit = fps_max(); + /* ((Triforce, 11-22, PRESENT-30) raised to the pace above 60 as before, and now put back where it was once the + pace is 60 again: it was raised once and never lowered, which a cap that never changed did not need) */ + static MovieLimit raised; + const int was = *limit, to = movie_pace_limit(&raised, was, pace); + if (to != was) *limit = to; } /* (the Insert menu's Game speed "per frame": a frame is a 60th of a second of game time. Not past 60 fps: the game would run faster than real time, so a cap above 60 keeps the real-time step whatever that row says) */ /* (nor under 60: a cap of 30 would be the game at half speed for good) */ - if (!rx_option(0, 1) && fps_max() == 60) { + if (!rx_option(0, 1) && pace == 60) { int *slot = (int *)GP(mod + 0x2c8); if (*slot >= 15 && *slot <= 60) *slot = 60; return; @@ -549,6 +590,12 @@ static void realtime_step(int64_t now) { static unsigned count, next; static long long steps; static StepSteady steady; /* ((Triforce) HOS_STEP_STEADY=1, below) */ + { /* ((Triforce, 11-22, PRESENT-30) a change of the pace, a movie's start or its end: the step's average starts + again, as after a load, since the intervals before it were paced by another period) */ + static int paced; + if (paced && paced != pace) { count = next = 0; carry = 0; step_steady_reset(&steady); } + paced = pace; + } const double dt = (now - last) / 1e9; last = now; if (dt <= 0 || dt > 0.25) { count = next = 0; carry = 0; step_steady_reset(&steady); return; } /* (a load: the average starts again) */ @@ -569,8 +616,8 @@ static void realtime_step(int64_t now) { HOS_FPS_STEP=0, or a lift without the hook (hos_step_probe unset): the rate up to the cap, as before */ static int timed_step = -1; if (timed_step < 0) timed_step = !(getenv("HOS_FPS_STEP") && !atoi(getenv("HOS_FPS_STEP"))); - const int timed = fps_max() > 60 && timed_step && hos_step_probe; - const int top = timed ? 60 : fps_max(); + const int timed = pace > 60 && timed_step && hos_step_probe; + const int top = timed ? 60 : pace; /* (Triforce, after NX2navi's steady step: host/step_steady.h) HOS_STEP_STEADY (on unless =0): the step is not the mean of the last five intervals but that of the last 12 without their longest and shortest, the paced period where it is within 3% of it, and what the display showed and the game was not given repaid at a 64th a frame (10% of the step at most), within the bounds @@ -598,7 +645,7 @@ static void realtime_step(int64_t now) { } const double ratio = secs_avg * 60.0; int *slot = (int *)GP(mod + 0x2c8); - if (*slot < 15 || *slot > (fps_max() > 60 ? fps_max() : 60)) { /* (under a cap below 60 it starts at the game's 60) */ + if (*slot < 15 || *slot > (pace > 60 ? pace : 60)) { /* (under a cap below 60 it starts at the game's 60) */ static int logged; if (!logged++) hos_log("vi: the graphics module's frame rate reads %d: no real-time step", *slot); return; @@ -778,7 +825,8 @@ void hos_vi_native_presented(void) { const int64_t now = hos_now_ns(); { void hos_bench_present(int64_t now); hos_bench_present(now); } /* ((Triforce) benchmark mode: every present's interval) */ /* ((Triforce) the Steady frame rate row: a period this PC holds in the scene, never shorter than the cap's) */ - const int64_t period = steady_period(1000000000LL / fps_max() * interval, now); + movie_pace_decide(); /* ((Triforce, 11-22, PRESENT-30) once a present: 240 while a movie plays at the 60 cap) */ + const int64_t period = steady_period(1000000000LL / pace_fps() * interval, now); pace_period_ns = period; /* (Triforce: what another limiter is given to hold in this file's place, pace_handed) */ realtime_step(now); /* (behind by more than a frame: the next one is paced from now, with no burst to catch up) */ diff --git a/host/nn_video.c b/host/nn_video.c index 9efc511..0c16e57 100644 --- a/host/nn_video.c +++ b/host/nn_video.c @@ -12,6 +12,12 @@ free registered buffer (host memory: the pool's guest memory is not used). The video service itself (nn::gsv) is not emulated: nn::codec::video::Initialize does nothing. */ #include "hos.h" +#include "host_alloc.h" /* ((Triforce, 11-22, OS-04) allocations that wait at Windows' commit limit before they answer NULL) */ +#include "video_av1.h" /* ((Triforce, 11-22, IO-32) which AV1 decoder goes first on Windows) */ +#include "video_movie.h" /* ((Triforce, 11-22, IO-32) what a movie cost to decode, and its line) */ +#include "video_nv12.h" /* ((Triforce, 11-22, IO-36) a decoded picture kept as NV12 alone) */ +#include "video_catchup.h" /* ((Triforce, 11-22, THREAD-82) how far dec_acquire decodes ahead to catch up) */ +#include "video_input.h" /* ((Triforce, 11-22, IO-34) how many submissions the decoder holds before it refuses one) */ #include #include #include @@ -38,6 +44,8 @@ static int video_log(void) { /* HOS_VIDEO_LOG=1: submissions and acquisitions void hos_system_event_init(uint64_t key, int autoclear); /* nn_os_sync.c */ void hos_system_event_signal(uint64_t key); +void hos_system_event_clear(uint64_t key); /* ((Triforce, 11-22, IO-34) these two under a host lock: no checkpoint) */ +void hos_system_event_signal_held(uint64_t key); /* an image pool: its pictures, w x h, as RGBA and as NV12 (both kept: the reader asks for either) */ typedef struct { @@ -71,7 +79,12 @@ static DWORD WINAPI av1_notice(LPVOID unused) { (Triforce) It needs Microsoft's AV1 Video Extension, which not every PC has: without it the movies are decoded here, by dav1d, when the FFmpeg the build links has it (the Tools' FFmpeg with AV1; nx2/build.py takes whichever is there), as NX2navi decodes them always. No notice and no black movies then. HOS_AV1=dav1d: dav1d first, for tests; - HOS_AV1_MISSING=1: as if the extension were missing. */ + HOS_AV1_MISSING=1: as if the extension were missing. + (Triforce, 11-22, the performance audit's IO-32, after Source A's) The order is the other way round now: dav1d first + where that FFmpeg has it, Media Foundation when it has not or when libdav1d does not open (video_av1.h). Media + Foundation's path here decodes in software on every GPU (mf_av1.c takes the synchronous decoder, with no D3D + manager), and its decode time was never seen: each movie now says in the log what it took (video_movie.h). + HOS_AV1=mf: Media Foundation first, as before. */ typedef void (*mf_av1_picture_fn)(void *user, const uint8_t *y, const uint8_t *uv, int w, int h, int stride, int64_t pts); void *mf_av1_open(int w, int h, int tell); void mf_av1_decode(void *m, const uint8_t *data, size_t n, int64_t pts, mf_av1_picture_fn fn, void *user); @@ -92,12 +105,21 @@ typedef struct { Picture queue[MAX_QUEUE]; int qhead, qn; int eos; /* the last submission is in and decoded: an empty queue is the end */ + int stopped; /* a Stop or Flush with nothing submitted since: no picture will come (dec_acquire) */ Pending *phead, *ptail; /* submissions not decoded yet, oldest first */ int64_t submits, acquires; int clock_set; /* (hookshot) the movie's clock: the picture at clock_pts was due at clock_ns */ int64_t clock_ns, clock_pts; int64_t last_pts; /* the pts of the last picture handed out */ int64_t log_submits; /* (HOS_VIDEO_LOG) submissions still to log, after a flush or the end */ + VideoMovie movie; /* ((Triforce, 11-22, IO-32) this movie's pictures and decode time, for its line) */ + VideoPace pace; /* ((Triforce, 11-22, THREAD-82) the gap between its pictures and what a decode takes + lately: the catch-up's bound (dec_acquire, video_catchup.h)) */ + int npending; /* ((Triforce, 11-22, IO-34) submissions in phead's list: with qn, what the input limit + counts (video_input.h)) */ + uint64_t ready; /* ((Triforce, 11-22, IO-34) its "ready for a submission" event, a guest address: set, + and clear from a refused submission until there is room again) */ + int ready_clear; /* (... whether it is clear now; changed only under the lock, with the event) */ SRWLOCK lock; } Dec; @@ -127,7 +149,7 @@ static void set_host(uint64_t obj, uint32_t magic, void *p) { if (i == nhost_objs) { if (nhost_objs == cap_host_objs) { cap_host_objs = cap_host_objs ? cap_host_objs * 2 : 16; - host_objs = (HostObj *)realloc(host_objs, cap_host_objs * sizeof *host_objs); + host_objs = (HostObj *)hos_xrealloc(host_objs, cap_host_objs * sizeof *host_objs); /* ((Triforce, OS-04) host_alloc.h) */ if (!host_objs) hos_fatal(NULL, "video: out of memory"); } nhost_objs++; @@ -164,7 +186,7 @@ HLE(_ZN2nn5codec31CalculateDecoderImagePoolBufferEPmS1_RKNS0_27VideoDecoderImage /* InitializeDecoderImagePool(pool, void* memory, size, Preference const&, u32 count) */ HLE(_ZN2nn5codec26InitializeDecoderImagePoolEPNS0_25VideoDecoderImagePoolTypeEPvmRKNS0_27VideoDecoderImagePreferenceEj) { const Pref *p = (const Pref *)GP(ARG(3)); - Pool *o = (Pool *)calloc(1, sizeof *o); + Pool *o = (Pool *)hos_xcalloc(1, sizeof *o); /* ((Triforce, OS-04) host_alloc.h) */ if (!o) hos_fatal(NULL, "video: out of memory for an image pool"); o->w = p->magic == PREF_MAGIC && p->w ? p->w : 1280, o->h = p->magic == PREF_MAGIC && p->h ? p->h : 720; o->format = p->magic == PREF_MAGIC ? (int)p->format : 2; @@ -280,6 +302,40 @@ HLE(_ZN2nn5codec26ReadDecoderOutputImageNv12EPKNS0_25VideoDecoderImagePoolTypeEj /* ---- decoders ---- */ static void dec_init_sized(Ctx *c, enum AVCodecID id, const char *what, int w, int h); static void dec_init(Ctx *c, enum AVCodecID id, const char *what) { dec_init_sized(c, id, what, 0, 0); } +#ifdef _WIN32 +static const char *av1_first(void) { /* ((Triforce, 11-22, IO-32) HOS_AV1, read once: "mf" puts Media Foundation first) */ + static const char *value; + static int read; + if (!read) value = getenv("HOS_AV1"), read = 1; + return value; +} +#endif +/* ((Triforce, 11-22, the performance audit's IO-34, after Source A's) The input limit (video_input.h): the decoder + refuses a submission while this many wait, not decoded yet or decoded and not taken. Every submission was taken + before, and the game submits a whole movie at once on the thread that also takes the pictures: the pictures that + came due while it submitted were skipped by the catch-up, a stutter at each movie's start. + HOS_VIDEO_INPUT_LIMIT=, read once: that many instead of 16; 0: none, as before) */ +static int input_limit(void) { + static int limit = -1; + if (limit < 0) limit = video_input_limit(getenv("HOS_VIDEO_INPUT_LIMIT")); + return limit; +} +static int input_full(const Dec *d) { return video_input_full(input_limit(), d->npending, d->qn); } +/* (under d->lock, wherever the count went down) room again: the event a refused submission cleared is set */ +static void input_room(Dec *d) { + if (d->ready_clear && !input_full(d)) { + d->ready_clear = 0; + hos_system_event_signal_held(d->ready); + } +} +/* a decoder's own ready event, set: one for all decoders could not be clear for the full one alone. Never freed, as + the one event before it (64 bytes of the system region a decoder) */ +static uint64_t ready_new(void) { + const uint64_t event = hos_sys_alloc(0x40, 16); + hos_system_event_init(event, 0); + hos_system_event_signal(event); + return event; +} static void dec_init_sized(Ctx *c, enum AVCodecID id, const char *what, int w, int h) { const AVCodec *codec = NULL; #ifndef _WIN32 /* (linux) FFmpeg's own AV1 decoder needs a hardware context, and there is no Media Foundation to fall @@ -289,11 +345,12 @@ static void dec_init_sized(Ctx *c, enum AVCodecID id, const char *what, int w, i #endif Dec *d = (Dec *)calloc(1, sizeof *d); InitializeSRWLock(&d->lock); -#ifdef _WIN32 /* (Triforce) AV1: Windows' decoder, else dav1d; never FFmpeg's own, which needs a hardware context */ +#ifdef _WIN32 /* (Triforce) AV1: dav1d, else Windows' decoder; never FFmpeg's own, which needs a hardware context. + ((Triforce, 11-22, IO-32) It was Windows' decoder first: video_av1.h has the rule and HOS_AV1=mf the old + order. Media Foundation is still opened below when libdav1d is there and does not open) */ if (id == AV_CODEC_ID_AV1) { - const char *first = getenv("HOS_AV1"); codec = avcodec_find_decoder_by_name("libdav1d"); - if (!(codec && first && !strcmp(first, "dav1d"))) d->mf = mf_av1_open(w, h, 0); + if (video_av1_mf_first(av1_first(), codec != NULL)) d->mf = mf_av1_open(w, h, 0); if (d->mf) codec = NULL; } else #endif @@ -316,6 +373,7 @@ static void dec_init_sized(Ctx *c, enum AVCodecID id, const char *what, int w, i #endif d->pkt = av_packet_alloc(); d->frame = av_frame_alloc(); + d->ready = ready_new(); /* ((Triforce, 11-22, IO-34) before the game can ask for it or submit) */ set_host(ARG(0), DEC_MAGIC, d); static int logged; if (logged++ < 4) @@ -335,10 +393,25 @@ static void pending_free(Dec *d) { free(n); } d->ptail = NULL; + d->npending = 0; +} +/* ((Triforce, 11-22, the performance audit's IO-32, after Source A's) The movie's line, once, and its counts from zero + for the next movie: at a Stop, a Flush and the decoder's end (video_movie.h) */ +static void movie_end(Dec *d) { + char line[280]; + if (video_movie_line(&d->movie, d->ctx ? d->ctx->codec->name : "Media Foundation", video_pace_spacing(&d->pace), line, + sizeof line)) + hos_log("video: %s", line); + memset(&d->movie, 0, sizeof d->movie); + memset(&d->pace, 0, sizeof d->pace); /* ((Triforce, 11-22, THREAD-82) the next movie's gap and decode time are its own) */ } static void dec_finalize(Ctx *c) { Dec *d = (Dec *)host_of(ARG(0), DEC_MAGIC); if (!d) return; + movie_end(d); + /* ((Triforce, 11-22, IO-34) its ready event is left set: a thread that still waits for it is not left waiting. No + lock: nothing else has this decoder any more) */ + if (d->ready_clear) hos_system_event_signal(d->ready); pending_free(d); for (int i = 0; i < d->qn; i++) picture_free(&d->queue[(d->qhead + i) % MAX_QUEUE]); if (d->sws) sws_freeContext(d->sws); @@ -349,11 +422,24 @@ static void dec_finalize(Ctx *c) { free(d); set_host(ARG(0), DEC_MAGIC, NULL); } -/* a decoded frame into the queue, as RGBA and NV12 (the oldest dropped when the reader falls behind) */ +static int rgba_eager(void) { /* ((Triforce, 11-22, IO-36) HOS_VIDEO_RGBA=1, read once: planar pictures made RGBA at once) */ + static int on = -1; + if (on < 0) on = getenv("HOS_VIDEO_RGBA") != NULL && atoi(getenv("HOS_VIDEO_RGBA")) != 0; + return on; +} +/* a decoded frame into the queue, as RGBA and NV12 (the oldest dropped when the reader falls behind). + ((Triforce, 11-22, the performance audit's IO-36, after Source A's) As NV12 alone for libdav1d's planar 4:2:0 too, as + Media Foundation's NV12 already was: no swscale context, no RGBA buffer and no conversion, the pool making the RGBA + only if it is read (pool_rgba; dec_acquire copies no RGBA rows for such a picture). Each 1080p picture was converted + on the movie thread for a game that reads only the NV12. The NV12 buffer is zeroed only where a copy leaves bytes + out, and U and V are interleaved 16 pairs a step (video_nv12.h). HOS_VIDEO_RGBA=1: the RGBA of every planar picture, + as before) */ static void queue_frame(Dec *d, const AVFrame *f) { const int w = f->width, h = f->height; if (w <= 0 || h <= 0) return; - if (f->format != AV_PIX_FMT_NV12 && (!d->sws || d->sws_w != w || d->sws_h != h || d->sws_fmt != f->format)) { + const int planar = f->format == AV_PIX_FMT_YUV420P || f->format == AV_PIX_FMT_YUVA420P; + const int lazy = video_rgba_lazy(rgba_eager(), f->format == AV_PIX_FMT_NV12, f->format == AV_PIX_FMT_YUV420P); + if (!lazy && (!d->sws || d->sws_w != w || d->sws_h != h || d->sws_fmt != f->format)) { if (d->sws) sws_freeContext(d->sws); d->sws = sws_getContext(w, h, (enum AVPixelFormat)f->format, w, h, AV_PIX_FMT_RGBA, SWS_BILINEAR, NULL, NULL, NULL); d->sws_w = w, d->sws_h = h, d->sws_fmt = f->format; @@ -361,14 +447,15 @@ static void queue_frame(Dec *d, const AVFrame *f) { Picture p = {0}; p.w = w, p.h = h, p.pts = f->pts; const int nv12 = f->format == AV_PIX_FMT_NV12; /* (hookshot: Media Foundation's pictures, kept as NV12 alone) */ - p.rgba = nv12 ? NULL : (uint8_t *)malloc((size_t)w * h * 4); - p.nv12 = nv12 ? (uint8_t *)malloc((size_t)w * h * 3 / 2) : (uint8_t *)calloc(1, (size_t)w * h * 3 / 2); - if (!p.nv12 || (!nv12 && !p.rgba)) { /* (out of memory: this frame is dropped, not written through NULL; issue 342) */ + p.rgba = lazy ? NULL : (uint8_t *)malloc((size_t)w * h * 4); + p.nv12 = (uint8_t *)malloc((size_t)w * h * 3 / 2); + if (!p.nv12 || (!lazy && !p.rgba)) { /* (out of memory: this frame is dropped, not written through NULL; issue 342) */ static volatile LONG told; if (InterlockedIncrement(&told) <= 4) hos_log("video: no memory for a %dx%d frame: dropped", w, h); free(p.rgba), free(p.nv12); return; } + if (video_nv12_zero(w, h, nv12 || planar)) memset(p.nv12, 0, (size_t)w * h * 3 / 2); uint8_t *dst[4] = {p.rgba, NULL, NULL, NULL}; int dst_stride[4] = {w * 4, 0, 0, 0}; if (d->sws && p.rgba) sws_scale(d->sws, (const uint8_t *const *)f->data, f->linesize, 0, h, dst, dst_stride); @@ -376,14 +463,12 @@ static void queue_frame(Dec *d, const AVFrame *f) { for (int y = 0; y < h; y++) memcpy(p.nv12 + (size_t)y * w, f->data[0] + (size_t)y * f->linesize[0], (size_t)w); for (int y = 0; y < h / 2; y++) memcpy(p.nv12 + (size_t)w * h + (size_t)y * w, f->data[1] + (size_t)y * f->linesize[1], (size_t)w); - } else if (f->format == AV_PIX_FMT_YUV420P || f->format == AV_PIX_FMT_YUVA420P) { /* (planar 4:2:0 -> NV12) */ + } else if (planar) { /* (planar 4:2:0 -> NV12) */ for (int y = 0; y < h; y++) memcpy(p.nv12 + (size_t)y * w, f->data[0] + (size_t)y * f->linesize[0], (size_t)w); uint8_t *uv = p.nv12 + (size_t)w * h; for (int y = 0; y < h / 2; y++) - for (int x = 0; x < w / 2; x++) { - uv[(size_t)y * w + 2 * x] = f->data[1][(size_t)y * f->linesize[1] + x]; - uv[(size_t)y * w + 2 * x + 1] = f->data[2][(size_t)y * f->linesize[2] + x]; - } + video_uv_interleave(uv + (size_t)y * w, f->data[1] + (size_t)y * f->linesize[1], + f->data[2] + (size_t)y * f->linesize[2], w / 2); } if (d->qn == MAX_QUEUE) { picture_free(&d->queue[d->qhead]); @@ -391,6 +476,8 @@ static void queue_frame(Dec *d, const AVFrame *f) { } d->queue[(d->qhead + d->qn) % MAX_QUEUE] = p; d->qn++; + d->movie.pictures++; + video_pace_picture(&d->pace, f->pts); /* ((Triforce, 11-22, THREAD-82) the gap between pictures, from their own times) */ } static void mf_picture(void *user, const uint8_t *y, const uint8_t *uv, int w, int h, int stride, int64_t pts) { AVFrame f; @@ -417,9 +504,12 @@ static void decode_pending(Dec *d) { /* one submission decoded (its pictures into the queue, if any) */ static void decode_next(Dec *d) { if (d->phead) { + const int64_t began = hos_now_ns(); /* ((Triforce, 11-22, IO-32) each submission timed, for the movie's line) */ + const int64_t pictures = d->movie.pictures; Pending *n = d->phead; d->phead = n->next; if (!d->phead) d->ptail = NULL; + d->npending--; if (d->mf) { /* (hookshot) */ if (n->pkt) { d->eos = 0; @@ -448,6 +538,10 @@ static void decode_next(Dec *d) { avcodec_flush_buffers(d->ctx); /* (a stream after it starts again) */ } free(n); + const int64_t took = hos_now_ns() - began; + video_movie_decoded(&d->movie, took); + /* ((Triforce, 11-22, THREAD-82) and what a decode that gives a picture takes lately, for the catch-up's bound) */ + if (d->movie.pictures != pictures) video_pace_decoded(&d->pace, took); } } static void pending_push(Dec *d, AVPacket *pkt) { @@ -456,6 +550,7 @@ static void pending_push(Dec *d, AVPacket *pkt) { if (d->ptail) d->ptail->next = n; else d->phead = n; d->ptail = n; + d->npending++; } /* SubmitXDecoderInputBuffer(u64* id, decoder, data, size, TimeSpan pts, bool last): kept, decoded when a picture is acquired (the library submits a whole movie at once: its pictures decoded then would not fit any queue) */ @@ -526,6 +621,24 @@ static uint64_t submitted(void) { /* a submission: every handle given out befo static void dec_submit(Ctx *c) { Dec *d = (Dec *)host_of(ARG(1), DEC_MAGIC); + /* ((Triforce, 11-22, the performance audit's IO-34, after Source A's) Refused at once, with the console's result for + it, while the limit's worth of submissions wait (video_input.h): the game submits it again later. Never the + stream's last, and only by a decoder that decodes: one without would never make room. Before submitted(): a + refusal is no submission, and must not move the count the idle buffers' sync handles are compared with. The + event is cleared under the lock, with the flag input_room reads, so no room made on another thread is missed) */ + if (d && (d->ctx || d->mf) && input_limit()) { + AcquireSRWLockExclusive(&d->lock); + const int refuse = video_input_refuse(input_limit(), d->npending, d->qn, (int)(ARG(5) & 1)); + if (refuse) { + if (!d->ready_clear) d->ready_clear = 1, hos_system_event_clear(d->ready); + if (video_log() && d->movie.refused < 4) + hos_log("video: a submission refused (pts %lld): %d wait to be decoded and %d to be taken", (long long)ARG(4), + d->npending, d->qn); + d->movie.refused++; + } + ReleaseSRWLockExclusive(&d->lock); + if (refuse) { RET(VIDEO_RESULT_INPUT_FULL); return; } + } const uint64_t id = submitted(); if (ARG(0)) G64(ARG(0)) = id; if (!d || (!d->ctx && !d->mf)) { RET(0); return; } @@ -546,12 +659,14 @@ static void dec_submit(Ctx *c) { } } d->submits++; + d->stopped = 0; if (ARG(5) & 1) pending_push(d, NULL); const int logged_one = d->log_submits > 0; if (logged_one) d->log_submits--; if (video_log() && (d->submits <= 8 || (ARG(5) & 1) || logged_one)) hos_log("video: submit %lld (%llu bytes, pts %lld%s)", (long long)d->submits, (unsigned long long)ARG(3), (long long)ARG(4), (ARG(5) & 1) ? ", last" : ""); + input_room(d); /* ((Triforce, 11-22, IO-34) as at every unlock that ends a call: the event never stays clear with room) */ ReleaseSRWLockExclusive(&d->lock); RET(0); } @@ -564,14 +679,17 @@ static void dec_flush(Ctx *c, const char *what) { (long long)d->acquires, d->qn, d->eos ? "at the end" : d->phead ? "submissions left" : "none left", (unsigned long long)c->x[30]); d->log_submits = 8; + movie_end(d); /* ((Triforce, 11-22, IO-32) what the movie cost to decode, once) */ if (d->ctx) avcodec_flush_buffers(d->ctx); else mf_av1_flush(d->mf); pending_free(d); for (int i = 0; i < d->qn; i++) picture_free(&d->queue[(d->qhead + i) % MAX_QUEUE]); d->qhead = d->qn = 0; d->eos = 0; + d->stopped = 1; d->clock_set = 0; for (int i = 0; i < MAX_SLOTS; i++) d->slot_last[i] = 0; + input_room(d); /* ((Triforce, 11-22, IO-34) nothing waits any more: ready for submissions) */ ReleaseSRWLockExclusive(&d->lock); } RET(0); @@ -592,11 +710,22 @@ static void dec_acquire(Ctx *c) { Dec *d = (Dec *)host_of(ARG(2), DEC_MAGIC); if (ARG(1)) G64(ARG(1)) = 0; if (!d) { RET(RESULT_NO_OUTPUT); return; } - /* (the library aborts on any failure here: the hardware call waits for a picture. Up to ~100 ms for one decoded - by another thread's submission; then a free buffer as it was, so the caller is never refused) */ + /* (the library aborts on any failure here: the hardware call waits for a picture. Up to 100 ms by the clock for one + decoded by another thread's submission, none after a Stop or Flush with nothing submitted since; then a free + buffer as it was, so the caller is never refused. The wait was 50 sleeps of 2 ms, each 4-5 ms long on Windows: + ~250 ms after a movie's Stop, while the game's main thread joined the movie thread, a frame that long at every + load from the title. HOS_VIDEO_ACQUIRE_TRIES=1: the 50 sleeps as before. + The decode runs without the thread's virtual core: other game threads run on it meanwhile. It is taken back only + after the lock is let go, never under it (dec_submit holds a core while it waits for the lock). + HOS_VIDEO_KEEP_CORE=1: the core held throughout, as before) */ + static int by_tries = -1, keep_core = -1; + if (by_tries < 0) by_tries = getenv("HOS_VIDEO_ACQUIRE_TRIES") && atoi(getenv("HOS_VIDEO_ACQUIRE_TRIES")) == 1; + if (keep_core < 0) keep_core = getenv("HOS_VIDEO_KEEP_CORE") && atoi(getenv("HOS_VIDEO_KEEP_CORE")) == 1; + const int away = keep_core ? 0 : hos_vcore_away(); AcquireSRWLockExclusive(&d->lock); decode_pending(d); - for (int tries = 0; !d->qn && !d->eos && tries < 50; tries++) { + const int64_t wait_until = hos_now_ns() + 100000000; + for (int tries = 0; !d->qn && !d->eos && (by_tries ? tries < 50 : !d->stopped && hos_now_ns() < wait_until); tries++) { ReleaseSRWLockExclusive(&d->lock); hos_sleep_ns(2000000); AcquireSRWLockExclusive(&d->lock); @@ -606,7 +735,9 @@ static void dec_acquire(Ctx *c) { for (int i = 0; i < MAX_SLOTS && slot < 0; i++) if (d->slots[i] && !d->busy[i] && host_of(d->slots[i], POOL_MAGIC)) slot = i; if (slot < 0) { + input_room(d); /* ((Triforce, 11-22, IO-34) here and at the two exits below: what was decoded or taken made room) */ ReleaseSRWLockExclusive(&d->lock); + if (away) hos_vcore_back(); RET(RESULT_NO_OUTPUT); return; } @@ -621,7 +752,9 @@ static void dec_acquire(Ctx *c) { d->eos ? " (the end)" : idle ? " (idle: its sync not reached)" : "", slot); else if (d->acquires == 1 && !d->eos) /* (the start: the frame shows the buffer as it was, black since the pool's fill) */ hos_log("video: the first buffer went out before the first picture was decoded (shown black)"); + input_room(d); ReleaseSRWLockExclusive(&d->lock); + if (away) hos_vcore_back(); if (ARG(0)) G32(ARG(0)) = (uint32_t)slot; RET(0); return; @@ -629,8 +762,16 @@ static void dec_acquire(Ctx *c) { { /* (hookshot) The game takes one picture a frame, and the sound plays on: under 60 fps the movie fell behind its sound (the intro "slow motion, audio ahead", issues 146 and 179). Pictures behind the movie's clock are skipped: the next ones decoded, as the whole movie is submitted at once. HOS_VIDEO_NO_CATCHUP=1: as before */ - static int off = -1; + /* ((Triforce, 11-22, the performance audit's THREAD-82, after Source A's) Decoding ahead to skip had no bound but + the guard below. With a decoder slower than the movie every picture decoded to catch up puts the clock + further ahead, so an acquire decoded hundreds of pictures with the lock held, threw them away, and the next + one started further behind. It now stops when a decode takes longer than the gap between two pictures, or + once this acquire has decoded for one gap, and shows the late picture it has (video_catchup.h): such a movie + falls behind its sound instead, and its line says so. HOS_VIDEO_UNBOUNDED_CATCHUP=1: no bound, as before) */ + static int off = -1, unbounded = -1; if (off < 0) off = getenv("HOS_VIDEO_NO_CATCHUP") != NULL; + if (unbounded < 0) + unbounded = getenv("HOS_VIDEO_UNBOUNDED_CATCHUP") != NULL && atoi(getenv("HOS_VIDEO_UNBOUNDED_CATCHUP")) != 0; const int64_t now = hos_now_ns(), late = 25000000; /* (a frame and a half at 60 fps) */ Picture *h = &d->queue[d->qhead]; if (!d->clock_set || h->pts < d->clock_pts) d->clock_set = 1, d->clock_ns = now, d->clock_pts = h->pts; /* (a start, a loop) */ @@ -640,6 +781,14 @@ static void dec_acquire(Ctx *c) { if (d->queue[d->qhead].pts + late >= due) break; if (d->qn < 2) { if (!d->phead) break; + if (video_catchup_stop(unbounded, video_pace_spacing(&d->pace), d->pace.recent_ns, hos_now_ns() - now)) { + if (video_log()) + hos_log("video: the catch-up stops %.2f s behind: a decode takes %.1f ms lately, a picture is due " + "every %.1f ms, and this acquire has decoded for %.1f ms", (double)(due - d->queue[d->qhead].pts) / 1e9, + (double)d->pace.recent_ns / 1e6, (double)video_pace_spacing(&d->pace) / 1e6, + (double)(hos_now_ns() - now) / 1e6); + break; + } decode_next(d); continue; } @@ -648,6 +797,7 @@ static void dec_acquire(Ctx *c) { skipped++; } if (skipped && video_log()) hos_log("video: %d pictures skipped to catch up with the movie's clock", skipped); + d->movie.skipped += skipped; /* ((Triforce, 11-22, IO-32) and in the movie's line) */ } Picture *p = &d->queue[d->qhead]; Pool *o = (Pool *)host_of(d->slots[slot], POOL_MAGIC); @@ -703,7 +853,9 @@ static void dec_acquire(Ctx *c) { } picture_free(p); d->qhead = (d->qhead + 1) % MAX_QUEUE, d->qn--; + input_room(d); ReleaseSRWLockExclusive(&d->lock); + if (away) hos_vcore_back(); if (ARG(0)) G32(ARG(0)) = (uint32_t)slot; RET(0); } @@ -734,13 +886,13 @@ static void dec_properties(Ctx *c) { } RET(0); } -static uint64_t ready_event(void) { +/* GetXDecoderSubmissionReadyEvent(decoder). ((Triforce, 11-22, IO-34) The decoder's own event, clear while its input + is full: it was one event for every decoder, always set. That one is still what an unknown decoder gets) */ +static uint64_t ready_event(Ctx *c) { static uint64_t event; - if (!event) { - event = hos_sys_alloc(0x40, 16); - hos_system_event_init(event, 0); - hos_system_event_signal(event); - } + const Dec *d = (const Dec *)host_of(ARG(0), DEC_MAGIC); + if (d && d->ready) return d->ready; + if (!event) event = ready_new(); return event; } @@ -753,7 +905,7 @@ HLE(_ZN2nn5codec26UnregisterVp9DecoderBufferEPNS0_14Vp9DecoderTypeEj) { dec_unre HLE(_ZN2nn5codec15StartVp9DecoderEPNS0_14Vp9DecoderTypeE) { RET(0); } HLE(_ZN2nn5codec14StopVp9DecoderEPNS0_14Vp9DecoderTypeE) { dec_flush(c, "Stop"); } HLE(_ZN2nn5codec15FlushVp9DecoderEPNS0_14Vp9DecoderTypeE) { dec_flush(c, "Flush"); } -HLE(_ZN2nn5codec33GetVp9DecoderSubmissionReadyEventEPNS0_14Vp9DecoderTypeE) { RET(ready_event()); } +HLE(_ZN2nn5codec33GetVp9DecoderSubmissionReadyEventEPNS0_14Vp9DecoderTypeE) { RET(ready_event(c)); } HLE(_ZN2nn5codec27SubmitVp9DecoderInputBufferEPmPNS0_14Vp9DecoderTypeEPKvmNS_8TimeSpanEb) { dec_submit(c); } HLE(_ZN2nn5codec36SubmitEncryptedVp9DecoderInputBufferEPmPNS0_14Vp9DecoderTypeEPKNS0_18EncryptionPropertyEPKvmNS_8TimeSpanEb) { const uint64_t id = submitted(); @@ -778,7 +930,7 @@ HLE(_ZN2nn5codec24RegisterAv1DecoderBufferEPNS0_14Av1DecoderTypeEjPNS0_25VideoDe HLE(_ZN2nn5codec15StartAv1DecoderEPNS0_14Av1DecoderTypeE) { RET(0); } HLE(_ZN2nn5codec14StopAv1DecoderEPNS0_14Av1DecoderTypeE) { dec_flush(c, "Stop"); } HLE(_ZN2nn5codec15FlushAv1DecoderEPNS0_14Av1DecoderTypeE) { dec_flush(c, "Flush"); } -HLE(_ZN2nn5codec33GetAv1DecoderSubmissionReadyEventEPNS0_14Av1DecoderTypeE) { RET(ready_event()); } +HLE(_ZN2nn5codec33GetAv1DecoderSubmissionReadyEventEPNS0_14Av1DecoderTypeE) { RET(ready_event(c)); } HLE(_ZN2nn5codec27SubmitAv1DecoderInputBufferEPmPNS0_14Av1DecoderTypeEPKvmNS_8TimeSpanEb) { dec_submit(c); } HLE(_ZN2nn5codec29AcquireAv1DecoderOutputBufferEPjPNS_2gs16NativeSyncHandleEPNS0_14Av1DecoderTypeE) { dec_acquire(c); } HLE(_ZN2nn5codec29ReleaseAv1DecoderOutputBufferEPNS0_14Av1DecoderTypeEjNS_2gs16NativeSyncHandleE) { dec_release(c); } @@ -793,7 +945,7 @@ HLE(_ZN2nn5codec26UnregisterVp8DecoderBufferEPNS0_14Vp8DecoderTypeEj) { dec_unre HLE(_ZN2nn5codec15StartVp8DecoderEPNS0_14Vp8DecoderTypeE) { RET(0); } HLE(_ZN2nn5codec14StopVp8DecoderEPNS0_14Vp8DecoderTypeE) { dec_flush(c, "Stop"); } HLE(_ZN2nn5codec15FlushVp8DecoderEPNS0_14Vp8DecoderTypeE) { dec_flush(c, "Flush"); } -HLE(_ZN2nn5codec33GetVp8DecoderSubmissionReadyEventEPNS0_14Vp8DecoderTypeE) { RET(ready_event()); } +HLE(_ZN2nn5codec33GetVp8DecoderSubmissionReadyEventEPNS0_14Vp8DecoderTypeE) { RET(ready_event(c)); } HLE(_ZN2nn5codec27SubmitVp8DecoderInputBufferEPmPNS0_14Vp8DecoderTypeEPKvmNS_8TimeSpanEb) { dec_submit(c); } HLE(_ZN2nn5codec29AcquireVp8DecoderOutputBufferEPjPNS_2gs16NativeSyncHandleEPNS0_14Vp8DecoderTypeE) { dec_acquire(c); } HLE(_ZN2nn5codec29ReleaseVp8DecoderOutputBufferEPNS0_14Vp8DecoderTypeEjNS_2gs16NativeSyncHandleE) { dec_release(c); } diff --git a/host/nvn_native.c b/host/nvn_native.c index 7536716..8db91e8 100644 --- a/host/nvn_native.c +++ b/host/nvn_native.c @@ -19,6 +19,8 @@ own procs (program builders, native images, sparse tile layouts, recording modes). The SDK's exports reach the same functions as the proc table (nvn2_dispatch, from gen/nvn2_exports.c). */ #include "hos.h" +#include "host_alloc.h" /* ((Triforce, 11-22, OS-04) allocations that wait at Windows' commit limit before they answer NULL) */ +#include "shadow_slot.h" /* ((Triforce, 11-21, TEX-85) a character's shadow map's storage in steps) */ #include "guest_bounds.h" /* ((Triforce) hos_span_ok: the game's first and count checked without an int overflow) */ #include "../gpu/render/rx_nvn.h" #include @@ -673,17 +675,53 @@ static int no_builder(Ctx *c, const char *what) { if (told++ < 4) hos_log("nvn: %s of no texture builder (lr %#llx): 0", what, (unsigned long long)c->x[30]); return 1; } +/* ((Triforce, 11-21, the performance audit's TEX-85, after Source A's) A character's shadow map reserves the storage of the + next step of sides up (shadow_slot.h), so that the render targets the game makes after it keep their addresses while its + side changes from frame to frame. HOS_SHADOW_BUCKET= (384); 0: every texture its own size, as before; not set, 0 + where the target heap is pinned (nn_applet.c hos_rt_pool_pinned). The layout of the side it reserves in *out, or 0 when it + reserves its own. The texture's own layout (nvn_tex_layout of its builder) is not changed anywhere */ +static int shadow_slot_layout(const nvn_texdesc *d, nvn_texlayout *out) { + extern int hos_rt_pool_pinned(void); + static int step = -1; + static volatile long stepped; + int now = step; + if (now < 0) { + const char *e = getenv("HOS_SHADOW_BUCKET"); + const int pinned = hos_rt_pool_pinned(); /* (-1: the heap not sized yet: asked again at the next map) */ + now = (int)hos_shadow_slot_step(e, pinned > 0); + if (e || pinned >= 0) { + step = now; + hos_log("nvn: a character's shadow map reserves the storage of the next multiple of %d up (HOS_SHADOW_BUCKET%s; 0: " + "its own size, as before)", now, e ? "" : pinned > 0 ? ": the target heap is pinned, so 0" : ", 384 when not set"); + } + } + const uint32_t side = hos_shadow_slot_side(d->target == NVN_TEXTURE_TARGET_2D, d->format, d->levels, d->depth, d->width, + d->height, (uint32_t)now); + if (!side || side == d->width) return 0; + nvn_texdesc slot = *d; + slot.width = slot.height = side; + nvn_tex_layout(&slot, out); + const long n = __atomic_add_fetch(&stepped, 1, __ATOMIC_RELAXED); /* (as it doubles: how often the game asks) */ + if (n >= 1024 && !(n & (n - 1))) + hos_log("nvn: shadow maps answered a stepped storage: %ld so far (the last %ux%u as %ux%u)", n, d->width, d->height, side, side); + return 1; +} NV(nvnTextureBuilderGetStorageSize) { if (no_builder(c, "storage size")) { RET(0); return; } - nvn_texlayout L; + nvn_texlayout L, slot; nvn_tex_layout(&CTB(P(0))->d, &L); if (!L.bpb) hos_log("nvn: texture format %#x unknown: storage size 0", CTB(P(0))->d.format); + /* ((Triforce, 11-21, TEX-85) the larger of its own and its slot's) */ + if (L.bpb && shadow_slot_layout(&CTB(P(0))->d, &slot) && slot.size > L.size) L.size = slot.size; RET(L.size); } NV(nvnTextureBuilderGetStorageAlignment) { if (no_builder(c, "storage alignment")) { RET(512); return; } - nvn_texlayout L; + nvn_texlayout L, slot; nvn_tex_layout(&CTB(P(0))->d, &L); + /* ((Triforce, 11-21, TEX-85) the slot's where it is the larger: one alignment for every side within a step, and the + texture's own still met) */ + if (L.bpb && shadow_slot_layout(&CTB(P(0))->d, &slot) && slot.align > L.align) L.align = slot.align; RET(L.align); } NV(nvnTextureBuilderGetZCullStorageSize) { RET(0); } /* no ZCull: nothing to store */ @@ -1368,7 +1406,7 @@ static unsigned pool_class(size_t n) { static void *pool_get(size_t n, unsigned *cls) { const unsigned c = pool_off() ? POOL_MAX + 1 : pool_class(n); *cls = c; - if (c > POOL_MAX) return malloc(n); + if (c > POOL_MAX) return hos_xmalloc(n); /* ((Triforce, OS-04) waits at the commit limit before it answers NULL: host_alloc.h) */ AcquireSRWLockExclusive(&pool_lock); void *b = pool_head[c]; if (b) { @@ -1376,7 +1414,7 @@ static void *pool_get(size_t n, unsigned *cls) { pool_bytes[c] -= (uint64_t)1 << c; } ReleaseSRWLockExclusive(&pool_lock); - return b ? b : malloc((size_t)1 << c); + return b ? b : hos_xmalloc((size_t)1 << c); } static void pool_put(void *b, unsigned cls) { if (!b) return; @@ -1417,7 +1455,7 @@ static cmd_rec *cmd_adopt(nvn_cmd *o) { const uint32_t s = ncmd_free ? cmd_free[--ncmd_free] : cmd_next++; if ((s >> CMD_CHUNK_BITS) >= CMD_CHUNKS) die("too many command buffers"); if (!cmd_chunks[s >> CMD_CHUNK_BITS]) { - cmd_rec *k = (cmd_rec *)calloc((size_t)1 << CMD_CHUNK_BITS, sizeof *k); + cmd_rec *k = (cmd_rec *)hos_xcalloc((size_t)1 << CMD_CHUNK_BITS, sizeof *k); /* ((Triforce, OS-04) host_alloc.h) */ if (!k) die("command buffer table"); cmd_chunks[s >> CMD_CHUNK_BITS] = k; } @@ -1452,7 +1490,7 @@ static void cmd_release(nvn_cmd *o) { /* the object's slot and recording buffe memset(r, 0, sizeof *r); if (ncmd_free == cap_cmd_free) { cap_cmd_free = cap_cmd_free ? cap_cmd_free * 2 : 256; - cmd_free = (uint32_t *)realloc(cmd_free, cap_cmd_free * sizeof *cmd_free); + cmd_free = (uint32_t *)hos_xrealloc(cmd_free, cap_cmd_free * sizeof *cmd_free); /* ((Triforce, OS-04) host_alloc.h) */ if (!cmd_free) die("command buffer table"); } cmd_free[ncmd_free++] = o->slot; @@ -1560,8 +1598,8 @@ static size_t live_at(uint64_t h, size_t cap) { return (size_t)((h * 0x9E3779B97 static void live_handle_add(uint64_t h) { /* (copy_lock held exclusively) */ if ((nlive + 1) * 2 > live_cap) { const size_t cap = live_cap ? live_cap * 2 : 65536; - uint64_t *t = (uint64_t *)calloc(cap, sizeof *t); - live_table *pub = (live_table *)malloc(sizeof *pub); + uint64_t *t = (uint64_t *)hos_xcalloc(cap, sizeof *t); /* ((Triforce, OS-04) host_alloc.h) */ + live_table *pub = (live_table *)hos_xmalloc(sizeof *pub); if (!t || !pub) die("recording handle set"); for (size_t i = 0; i < live_cap; i++) { if (!live_handles[i]) continue; @@ -1635,7 +1673,7 @@ static void free_dead_copies(void) { /* (copy_lock held) */ static void dead_copy_add(copy_prefix *p, uint64_t after) { /* (copy_lock held) */ if (ndead == cap_dead) { cap_dead = cap_dead ? cap_dead * 2 : 1024; - dead = (dead_copy *)realloc(dead, cap_dead * sizeof *dead); + dead = (dead_copy *)hos_xrealloc(dead, cap_dead * sizeof *dead); /* ((Triforce, OS-04) host_alloc.h) */ if (!dead) die("recording copies let go"); } dead[ndead].p = p; @@ -1682,10 +1720,13 @@ static void release_copies(uint64_t lo, uint64_t hi) { or calling a recording waited behind it */ #define SWEEP_EVERY 1024u #define SWEEP_SLICE 2688u /* (65536 / 2688 * 1024: every bucket once in ~25,000 recordings) */ -static void sweep_overwritten_copies(void) { /* (copy_lock held) */ +/* ((Triforce, 11-22, the performance audit's THREAD-56, after Source A's) it returns how many it let go and logs nothing: + its line was written with copy_lock held exclusively, every thread recording or calling a recording waiting behind the + write. nvnCommandBufferEndRecording says it once the lock is released) */ +static size_t sweep_overwritten_copies(void) { /* (copy_lock held) */ static int off = -1; if (off < 0) off = getenv("HOS_NVN_NO_SWEEP") != NULL; - if (off || !rec_buckets) return; + if (off || !rec_buckets) return 0; free_dead_copies(); const uint64_t after = rx_nvn_pushed() + held_count(); size_t swept = 0; @@ -1721,8 +1762,7 @@ static void sweep_overwritten_copies(void) { /* (copy_lock held) */ swept++; } } - static int logged; - if (swept && logged++ < 12) hos_log("nvn: %zu copies of recordings whose control memory was reused let go (%zu live)", swept, ncopies); + return swept; } NV(nvnCommandBufferAddControlMemory) { nvn_cmd *o = CB(P(0)); @@ -1815,12 +1855,27 @@ NV(nvnCommandBufferEndRecording) { ncopies++; live_handle_add((uint64_t)(uintptr_t)copy); static uint64_t made; - if (++made % SWEEP_EVERY == 0) sweep_overwritten_copies(); - if (made % 50000 == 0) - hos_log("nvn: %llu recordings made, %zu live copies (%.1f MB), %zu texture copies; %lld command buffers' segments " - "(%.1f MB)", (unsigned long long)made, ncopies, copy_bytes / 1048576.0, npins, (long long)seg_count, - seg_bytes / 1048576.0); + /* ((Triforce, 11-22, the performance audit's THREAD-56, after Source A's) what the sweep's line and the 50,000-recording + line say is taken under the lock; the lines are written after it is released, with the virtual core given up for + when this thread makes the write itself (HOS_LOG_SYNC=1, or the log's queue full): a write that blocks then holds + neither the recordings' lock nor a core another game thread waits for) */ + static int sweeps_said; + const size_t swept = ++made % SWEEP_EVERY == 0 ? sweep_overwritten_copies() : 0; + const int say_swept = swept && sweeps_said < 12 ? ++sweeps_said : 0; + const int say_made = made % 50000 == 0; + const uint64_t made_now = made; + const size_t live_now = ncopies, pins_now = npins; ReleaseSRWLockExclusive(©_lock); + if (say_swept || say_made) { + const int away = hos_vcore_away(); + if (say_swept) + hos_log("nvn: %zu copies of recordings whose control memory was reused let go (%zu live)", swept, live_now); + if (say_made) + hos_log("nvn: %llu recordings made, %zu live copies (%.1f MB), %zu texture copies; %lld command buffers' segments " + "(%.1f MB)", (unsigned long long)made_now, live_now, copy_bytes / 1048576.0, pins_now, (long long)seg_count, + seg_bytes / 1048576.0); + if (away) hos_vcore_back(); + } if (watch_draw_cmd && watch_draw_cmd == (void *)o && nwatch_handles < 16) { watch_handles[nwatch_handles++] = (uint64_t)(uintptr_t)copy; watch_draw_cmd = NULL; @@ -2315,7 +2370,7 @@ static int pin_same(const nvn_tex *a, const nvn_tex *b) { } static void pins_grow(void) { const size_t n = pin_slots ? pin_slots * 2 : (1u << 16); - nvn_tex **t = (nvn_tex **)calloc(n, sizeof *t); + nvn_tex **t = (nvn_tex **)hos_xcalloc(n, sizeof *t); /* ((Triforce, OS-04) host_alloc.h) */ if (!t) die("texture copies table (%zu slots)", n); for (size_t i = 0; i < pin_slots; i++) if (pins[i]) { @@ -2734,8 +2789,12 @@ NV(nvnQueuePresentTexture) { extern volatile LONG64 hos_present_count; /* display.c */ InterlockedIncrement64(&hos_present_count); hos_vi_native_presented(); + /* ((Triforce, 11-22, the performance audit's OS-51) the host's part of a frame as long as the stall line's: mkw_glue.c) */ + { void hos_stall_host_presented(void); hos_stall_host_presented(); } void hos_mod_cheats_frame(void); /* (Triforce) mod_cheats.c: the mods' cheat files, once a game frame; nothing without one */ hos_mod_cheats_frame(); + void hos_achievements_frame(void); + hos_achievements_frame(); /* ((Triforce) host/achievements.c: its lists read and named to the menu; nothing while it is off) */ } /* ---- NVN2 (mkw-recomp): what the Switch 2's NVN adds, as MKW uses it ---- */ diff --git a/host/oot_menu.c b/host/oot_menu.c index dae12d7..f5e4b4d 100644 --- a/host/oot_menu.c +++ b/host/oot_menu.c @@ -28,14 +28,13 @@ #define PAUSE_SCREEN_VTABLE 0x84c0baa8ULL #define OPTIONS_FROM_PAUSE_NS 500000000LL /* (the pause menu's last update, at most this long before the Options' first) */ void rx_option_set(int id, int value); /* rx_backend.c */ -#if defined(_WIN32) && !defined(HOS_OOT_MENU_NO_MOUSE) /* (the macro: tests/test_oot_menu_mouse.py builds the other half on Windows) */ -#include -#include -#include +/* ((Triforce, 11-22) Both halves': the guest reads, a screen's list of entries and its buttons' boxes, and benchmark mode's + presses through the start's screens (bench_menu). Without them the Linux half pressed nothing, and the benchmark stood + on the title screen (#bug-tracker, 11-21). The mouse, Quit and Mods stay the Windows half's */ #include #include -#include "../gpu/vk/vk_api.h" /* (HOS_MOUSE_*) */ #include "hos_bench.h" /* ((Triforce, 11-19) benchmark mode) */ +void hos_hid_tap(const char *press); /* (nn_hid.c) */ /* A guest read, of the pointers the game's objects hold: inside the guest's space (below 1 MB: a null pointer's), it is read as the lifted code reads it; a page there not committed yet is committed on the touch (mkw_glue.c @@ -50,6 +49,77 @@ static uint64_t peek64(uint64_t a) { uint64_t v = 0; return peek(a, &v, 8) ? v : 0; } + +/* the screen's list of entries: fn(entry, button) for each, at most max */ +static int each_entry(uint64_t screen, int max, void (*fn)(int i, uint64_t entry, uint64_t button, void *arg), void *arg) { + const uint64_t head = screen + 136; + int32_t off = 0; + peek(screen + 156, &off, 4); + uint64_t node = peek64(screen + 144); + int i = 0; + for (; node && node != head && i < max; i++) { + const uint64_t entry = node - (uint64_t)(int64_t)off; + if (fn) fn(i, entry, peek64(entry + 24), arg); + node = peek64(node + 8); + } + return i; +} + +typedef struct { float x0, y0, x1, y1; } Rect; +/* The button's boxes (button+0x58, 0xb0 apart). The first is its hit box: its global matrix at +0x28 (3 x 4, by column: + the translation at +0x4c, +0x50; the layout's units, moving with a page as it slides; all 0 while its page is not shown) + and its rectangle at +0x84 (x, y, w, h, about the box's origin). The second is centred on the slot's picture (its + translation at +0xfc, +0x100; its own rectangle a fixed 700 x 700). A few slots' hit boxes cover only a side of the + picture (Pa_ItemIcon_05: 100 wide on the right of a slot 260 wide, its centre 80 to the right of the picture's): such + a box is mirrored about the picture's centre and both taken */ +static int button_rect(uint64_t button, Rect *r) { + float m[12], b[4], c[2]; + const uint64_t box = peek64(button + 0x58); + if (!peek(box + 0x28, m, sizeof m) || !peek(box + 0x84, b, sizeof b)) return 0; + const float sx = m[0], sy = m[4], tx = m[9], ty = m[10]; + if (!(sx > 1e-3f) || !(sy > 1e-3f) || !(b[2] > 0) || !(b[3] > 0)) return 0; + r->x0 = tx + sx * b[0], r->x1 = tx + sx * (b[0] + b[2]); + r->y0 = ty + sy * b[1], r->y1 = ty + sy * (b[1] + b[3]); + float s2; + if (peek(box + 0xd8, &s2, 4) && s2 > 1e-3f && peek(box + 0xfc, c, sizeof c)) { + const float w = r->x1 - r->x0, h = r->y1 - r->y0, cx = (r->x0 + r->x1) / 2, cy = (r->y0 + r->y1) / 2; + if (c[0] - cx < w && cx - c[0] < w && c[1] - cy < h && cy - c[1] < h) { /* (the picture's centre near the box) */ + const float x0 = 2 * c[0] - r->x1, x1 = 2 * c[0] - r->x0, y0 = 2 * c[1] - r->y1, y1 = 2 * c[1] - r->y0; + if (x0 < r->x0) r->x0 = x0; + if (x1 > r->x1) r->x1 = x1; + if (y0 < r->y0) r->y0 = y0; + if (y1 > r->y1) r->y1 = y1; + } + } + return 1; +} + +/* ((Triforce, 11-19) benchmark mode, host/hos_bench.c: the start's screens pressed through as each gets a focused button. The + file select is left on the file it opens on, the newest (its entries are a carousel's places, not files: focusing one by + its place loaded another file in a lab run) */ +typedef struct { int index, state; } BenchLook; +static void bench_entry(int i, uint64_t entry, uint64_t button, void *arg) { + (void)entry; + BenchLook *l = (BenchLook *)arg; + uint8_t state = 0; + if (peek(button + 56, &state, 1) && state > l->state) l->index = i, l->state = state; +} +static void bench_menu(Ctx *c, uint64_t screen, uint64_t vt) { + (void)c; + BenchLook l = {-1, 0}; + const int n = each_entry(screen, 64, bench_entry, &l); + int entry = -1; + if (hos_bench_menu(screen, vt, 1, l.index, n, &entry) == HOS_BENCH_MENU_PRESS_A) hos_hid_tap("0:A/120"); +} + +#if defined(_WIN32) && !defined(HOS_OOT_MENU_NO_MOUSE) /* (the macro: tests/test_oot_menu_mouse.py builds the other half on Windows) */ +#include +#include +#include +#include +#include +#include "../gpu/vk/vk_api.h" /* (HOS_MOUSE_*) */ + /* a printable C string at a (else "-") */ static const char *text_at(uint64_t a, char *buf, size_t cap) { if (!peek(a, buf, cap - 1)) return "-"; @@ -72,21 +142,6 @@ static uint64_t focused_button; /* (the last pane f_802ed660 was handed) */ static uint64_t asked; /* (the button the mouse asked the focus for, until it has it) */ static int page_now = -1; /* (the shown page: 0 Collection, 1 Equipment, 2 Items, 3 System; L and R step through them) */ -/* the screen's list of entries: fn(entry, button) for each, at most max */ -static int each_entry(uint64_t screen, int max, void (*fn)(int i, uint64_t entry, uint64_t button, void *arg), void *arg) { - const uint64_t head = screen + 136; - int32_t off = 0; - peek(screen + 156, &off, 4); - uint64_t node = peek64(screen + 144); - int i = 0; - for (; node && node != head && i < max; i++) { - const uint64_t entry = node - (uint64_t)(int64_t)off; - if (fn) fn(i, entry, peek64(entry + 24), arg); - node = peek64(node + 8); - } - return i; -} - /* the game's focus setter on a button (on the game's thread, at a screen's update) */ static int set_focus(Ctx *c, uint64_t screen, uint64_t button) { A64Fn fn = a64_lookup(0x8088ffc0ULL); @@ -270,7 +325,6 @@ static int map_floors(int *cur, int *top, int *bottom) { return *cur >= *bottom && *cur <= *top; } -typedef struct { float x0, y0, x1, y1; } Rect; /* (Key Items' items, in their grid's order: item_found puts the page's own positions of them in) */ static const char *const item_keys[9] = {"Item_Medal_Shadow", "Item_Medal_Light", "Item_Medal_Forest", "Item_Stone_Forest", "Item_Stone_Fire", "Item_Stone_Water", "Item_Medal_Spirit", "Item_Medal_Water", @@ -285,9 +339,6 @@ static void item_found(const char *key, float x, float y, void *arg) { *a->found |= 1 << i, a->x[i] = x, a->y[i] = y; } } -/* a button's rectangle in the layout's units (0 when it has none or is hidden) */ -static int button_rect(uint64_t button, Rect *r); - typedef struct { float x, y; uint64_t hit; float area; int n; uint64_t button[128]; Rect rect[128]; } Hit; static void shown_entry(int i, uint64_t entry, uint64_t button, void *arg) { (void)i, (void)entry; @@ -810,34 +861,6 @@ static void mouse_tick_locked(Ctx *c, uint64_t screen) { if (pressed & HOS_MOUSE_RIGHT) hos_hid_tap("0:B/80"); } -/* The button's boxes (button+0x58, 0xb0 apart). The first is its hit box: its global matrix at +0x28 (3 x 4, by column: - the translation at +0x4c, +0x50; the layout's units, moving with a page as it slides; all 0 while its page is not shown) - and its rectangle at +0x84 (x, y, w, h, about the box's origin). The second is centred on the slot's picture (its - translation at +0xfc, +0x100; its own rectangle a fixed 700 x 700). A few slots' hit boxes cover only a side of the - picture (Pa_ItemIcon_05: 100 wide on the right of a slot 260 wide, its centre 80 to the right of the picture's): such - a box is mirrored about the picture's centre and both taken */ -static int button_rect(uint64_t button, Rect *r) { - float m[12], b[4], c[2]; - const uint64_t box = peek64(button + 0x58); - if (!peek(box + 0x28, m, sizeof m) || !peek(box + 0x84, b, sizeof b)) return 0; - const float sx = m[0], sy = m[4], tx = m[9], ty = m[10]; - if (!(sx > 1e-3f) || !(sy > 1e-3f) || !(b[2] > 0) || !(b[3] > 0)) return 0; - r->x0 = tx + sx * b[0], r->x1 = tx + sx * (b[0] + b[2]); - r->y0 = ty + sy * b[1], r->y1 = ty + sy * (b[1] + b[3]); - float s2; - if (peek(box + 0xd8, &s2, 4) && s2 > 1e-3f && peek(box + 0xfc, c, sizeof c)) { - const float w = r->x1 - r->x0, h = r->y1 - r->y0, cx = (r->x0 + r->x1) / 2, cy = (r->y0 + r->y1) / 2; - if (c[0] - cx < w && cx - c[0] < w && c[1] - cy < h && cy - c[1] < h) { /* (the picture's centre near the box) */ - const float x0 = 2 * c[0] - r->x1, x1 = 2 * c[0] - r->x0, y0 = 2 * c[1] - r->y1, y1 = 2 * c[1] - r->y0; - if (x0 < r->x0) r->x0 = x0; - if (x1 > r->x1) r->x1 = x1; - if (y0 < r->y0) r->y0 = y0; - if (y1 > r->y1) r->y1 = y1; - } - } - return 1; -} - /* ---- (Triforce) The Mods entry under a game list's last button: the pause menu's System page (Save, Load, Options, Special Actions, Title Screen: its buttons left of the Basic Controls panel, x under 0) and the title's list (by its screen's vtable). Each frame such a list runs the renderer is told its last button's box and whether the game's cursor @@ -934,6 +957,15 @@ static void mods_list_tick(uint64_t screen, int title, ModsCall *call) { const Rect r = l.r[last]; const float pitch = (l.r[above].y0 + l.r[above].y1) / 2 - (r.y0 + r.y1) / 2; if (!(pitch > 20 && pitch < 600)) return; + /* ((Triforce, 11-20) the owner: "it should be synced with the timing of the other button ui so it seamlessly moves and + disappears with them". How much of the button the game shows is in its box beside its matrix (+0x78, a float, 0 to + 1: the lab's probe of the title's list read 0 for the 1.5 s the list runs before it shows, a rise over a second + as the title opens, 1 to 0 in one frame as Options is chosen, and the 15-frame rise as the list comes back). The + renderer drew the entry by a clock of its own from the first frame the list ran: a second and a half before the + game's buttons, and 150 ms and a fade after they had gone. It is told the game's own amount now, each frame. + Anything but a number from 0 to 1 there: all of it, as before) */ + float alpha = 1.0f; + if (!peek(peek64(l.b[last] + 0x58) + 0x78, &alpha, 4) || !(alpha >= 0.0f && alpha <= 1.0f)) alpha = 1.0f; /* The button's state byte (+56) is its focus animation's: 0 none, 1 coming, 3 focused, 2 going. While Mods has the cursor the game gets no input (nn_hid.c) and its last button is let go (mods_held, above) */ uint8_t state = 0; @@ -964,7 +996,14 @@ static void mods_list_tick(uint64_t screen, int title, ModsCall *call) { on it) The entry only while the game's cursor is on this list (its focus controller's entry is this screen's), or Mods has the cursor */ int pend = 0; - if (!mods_cursor_button(screen, &pend) && !mods_held.button) return; + if (!mods_cursor_button(screen, &pend) && !mods_held.button) { + /* ((Triforce, 11-20) the game's buttons are hidden too (Options or the Load list opened over the title: their + alpha goes to 0 on that frame): said, not left to the renderer's 150 ms without a report, so the entry + goes on the frame they do. With the buttons still showing (the cursor between two of them for a frame) it + is not said, and the entry stays as before) */ + if (alpha <= 0.0f) rx_option_set(HOS_OPT_MODS_LIST, 0); + return; + } } /* (coming or there; 2 is its focus going to another button. Let go for Mods, it is still the game's cursor's place) */ const int focused = state == 1 || state == 3 || (mods_held.button && mods_held.button == l.b[last]); @@ -979,7 +1018,8 @@ static void mods_list_tick(uint64_t screen, int title, ModsCall *call) { mods_entry = (Rect){mx - 550, my - pitch / 2, mx + 550, my + pitch / 2}; } quit_entry = title ? (Rect){0, 0, 0, 0} : (Rect){r.x0, r.y0 - 2 * pitch, r.x1, r.y1 - 2 * pitch}; - InterlockedExchange64(&mods_entry_ns, (LONG64)now); + if (alpha >= 0.5f) InterlockedExchange64(&mods_entry_ns, (LONG64)now); /* (the mouse's: an entry not shown yet is not there to click) */ + rx_option_set(HOS_OPT_MODS_ALPHA, (int)(alpha * 1000.0f + 0.5f)); rx_option_set(HOS_OPT_MODS_X0, (int)r.x0), rx_option_set(HOS_OPT_MODS_X1, (int)r.x1); rx_option_set(HOS_OPT_MODS_Y0, (int)r.y0), rx_option_set(HOS_OPT_MODS_Y1, (int)r.y1); rx_option_set(HOS_OPT_MODS_PITCH, (int)pitch); @@ -1050,24 +1090,6 @@ static Seen *screen_seen(uint64_t screen, int64_t now) { return free_one; } -/* ((Triforce, 11-19) benchmark mode, host/hos_bench.c: the start's screens pressed through as each gets a focused button. The - file select is left on the file it opens on, the newest (its entries are a carousel's places, not files: focusing one by - its place loaded another file in a lab run) */ -typedef struct { int index, state; } BenchLook; -static void bench_entry(int i, uint64_t entry, uint64_t button, void *arg) { - (void)entry; - BenchLook *l = (BenchLook *)arg; - uint8_t state = 0; - if (peek(button + 56, &state, 1) && state > l->state) l->index = i, l->state = state; -} -static void bench_menu(Ctx *c, uint64_t screen, uint64_t vt) { - (void)c; - BenchLook l = {-1, 0}; - const int n = each_entry(screen, 64, bench_entry, &l); - int entry = -1; - if (hos_bench_menu(screen, vt, 1, l.index, n, &entry) == HOS_BENCH_MENU_PRESS_A) hos_hid_tap("0:A/120"); -} - void hos_oot_menu_tick(Ctx *c, uint64_t screen) { if (!screen) return; if (trace < 0) trace = getenv("HOS_MENU_TRACE") != NULL; @@ -1157,11 +1179,23 @@ void hos_oot_menu_tick(Ctx *c, uint64_t screen) { #include "../gpu/vk/vk_api.h" /* (HOS_OPT_GAME_OPTIONS) */ static int64_t options_seen_ns; /* (the screens' updates run on several of the game's threads: read and written whole) */ void hos_oot_menu_focus(Ctx *c, uint64_t screen, uint64_t pane, uint64_t lr) { (void)c, (void)screen, (void)pane, (void)lr; } +/* ((Triforce, 11-22) benchmark mode: a screen's focused button as the Windows half's focus_entry finds it (its state set and its + box shown, the screen's focus manager there), then bench_menu as there */ +static void bench_focus_entry(int i, uint64_t entry, uint64_t button, void *arg) { + (void)i, (void)entry; + uint8_t state = 0; + Rect r; + if (!*(int *)arg && peek(button + 56, &state, 1) && state && button_rect(button, &r)) *(int *)arg = 1; +} void hos_oot_menu_tick(Ctx *c, uint64_t screen) { - (void)c; uint64_t vt = 0; if (screen < 0x100000 || screen >= HOS_GUEST_SPACE - 8) return; memcpy(&vt, GP(screen), 8); + if (hos_bench_on()) { + int focus = 0; + if (peek64(peek64(screen + 32) + 96)) each_entry(screen, 400, bench_focus_entry, &focus); + if (focus) bench_menu(c, screen, vt); + } static int64_t pause_ran_ns; /* ((Triforce, 11-20) the pause menu's last update: where the Options were chosen from) */ static int options_from_pause; const int64_t now = hos_now_ns(); diff --git a/host/oot_shadow_step.c b/host/oot_shadow_step.c new file mode 100644 index 0000000..bd6a0de --- /dev/null +++ b/host/oot_shadow_step.c @@ -0,0 +1,25 @@ +/* (Triforce, 11-21, the performance audit's GAME-05, after Source B's) The side of a character's or an object's shadow + map rounded up to a step: the lifted code's read of the side goes through hos_oot_shadow_side (nx2/oot_patches.py + patch_shadow_step). host/oot_shadow_step.h has why and the rule. + HOS_OOT_SHADOW_STEP=: the step (64 when not set); 0 or 1: the game's own side, as before. */ +#include "hos.h" +#include "oot_shadow_step.h" +#include + +uint32_t hos_oot_shadow_side(uint32_t side) { + static int step = -1; /* (read once: the same value whichever thread is first) */ + static volatile long rounded; /* (the log's count) */ + int s = __atomic_load_n(&step, __ATOMIC_RELAXED); + if (s < 0) { + s = (int)oot_shadow_step(getenv("HOS_OOT_SHADOW_STEP")); + int unset = -1; + if (__atomic_compare_exchange_n(&step, &unset, s, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) + hos_log("oot: shadow maps' sides rounded up to a multiple of %d (HOS_OOT_SHADOW_STEP; 0: the game's own, as before)", s <= 1 ? 0 : s); + } + const uint32_t out = oot_shadow_side(side, (uint32_t)s); + if (out != side) { + const long n = __atomic_add_fetch(&rounded, 1, __ATOMIC_RELAXED); + if (n >= 4096 && !(n & (n - 1))) hos_log("oot: shadow maps' sides rounded: %ld so far (the last %u as %u)", n, side, out); + } + return out; +} diff --git a/host/oot_shadow_step.h b/host/oot_shadow_step.h new file mode 100644 index 0000000..6978782 --- /dev/null +++ b/host/oot_shadow_step.h @@ -0,0 +1,41 @@ +/* (Triforce, 11-21, the performance audit's GAME-05, after Source B's) The side of a character's or an object's shadow + map, in steps. nx2/oot_patches.py patch_shadow_step hooks the place; host/oot_shadow_step.c reads the switch. + + The game gives each nearby character or object a square depth shadow map of its own, sized by its size on screen + (384 to 1,920 a side), so while anything moves the side changes every frame, and each frame asks for depth textures + of sizes not seen before: an image, its memory and its views made, and the old ones destroyed. The render targets' + pool gives an image back only to a target of exactly its shape. Rounded up to a multiple of 64 the sides are 25 + sizes, which come round again, and the pool serves them. Source B measured, strafing along a wall in Castle Town's + pot room, 4,000 to 10,000 new depth images in 600 frames against 30 to 90. + + The one read of the side, before the texture is asked for, is rounded; the function's later uses read the same + register, so the viewport and the texture stay in step. The projection is unchanged: a map has at most 63 more + texels a side. 768 is 12 x 64, so no map crosses the renderer's rule for maps of 768 and less. + + HOS_OOT_SHADOW_STEP=: the step, a power of two from 2 to 256; not set, 64; 0 or 1: the game's own side, as before. + + The rule alone, no game: tests/test_oot_shadow_step.py compiles it. */ +#ifndef OOT_SHADOW_STEP_H +#define OOT_SHADOW_STEP_H +#include + +enum { OOT_SHADOW_STEP_DEFAULT = 64, OOT_SHADOW_STEP_MOST = 256 }; + +/* The switch's value as a step: not set, the default; 0 or 1, off (1); a power of two up to OOT_SHADOW_STEP_MOST, itself; + anything else (not a number, not a power of two, more), the default */ +static inline uint32_t oot_shadow_step(const char *asked) { + if (!asked || !*asked) return OOT_SHADOW_STEP_DEFAULT; + uint32_t v = 0; + int digits = 0; + const char *p = asked; + for (; *p >= '0' && *p <= '9' && v < 100000; p++) v = v * 10 + (uint32_t)(*p - '0'), digits++; + if (!digits || *p) return OOT_SHADOW_STEP_DEFAULT; + if (v <= 1) return 1; + return v <= OOT_SHADOW_STEP_MOST && !(v & (v - 1)) ? v : OOT_SHADOW_STEP_DEFAULT; +} + +/* The side the game asks its shadow map with: the next multiple of step from side (step a power of two; 1: side itself) */ +static inline uint32_t oot_shadow_side(uint32_t side, uint32_t step) { + return step <= 1 ? side : (side + step - 1) & ~(step - 1); +} +#endif diff --git a/host/oot_tex_compact.c b/host/oot_tex_compact.c index a009f73..4a4568e 100644 --- a/host/oot_tex_compact.c +++ b/host/oot_tex_compact.c @@ -19,10 +19,16 @@ are not touched. Nothing is held unless the switch is set: whether the game can do with fewer cycles (its heap is the one that fills, host/oot_heap_retry.c) is for the lab to say first. - HOS_OOT_TEX_COMPACT_MS=: at most one cycle started every n ms. -1 (a test): none started at all. Unset or 0: as shipped. */ + HOS_OOT_TEX_COMPACT_MS=: at most one cycle started every n ms. -1 (a test): none started at all. Unset or 0: as shipped. + + (Triforce) And before that, the ask: the manager's step (f_80797240) asks for a round at most once a second + (hos_oot_tex_compact_due, nx2/oot_patches.py patch_tex_ask; the rule and HOS_OOT_TEX_ASK_MS in oot_tex_compact.h). Asking + every frame against never, standing at the Market: 11.0% and 9.8% of frames of 16.7 ms or more against 3.0% and 1.2%. */ #include "hos.h" +#include "oot_tex_compact.h" #include #include +#include /* (the supported build's) the compaction's record at the manager's +7992, its state at the record's +8: 3 idle (the next step plans), 0 to 2 a cycle's steps */ @@ -78,3 +84,32 @@ void hos_oot_tex_compact(Ctx *c, A64Fn game) { said = now, started = held = steps = 0; } } + +/* (Triforce) Whether the manager's step may ask for a round now (patch_tex_ask: called where it would ask, with no round asked + for and the ask's bit set). The asks let through and held are logged every 10 s while there are any */ +int hos_oot_tex_compact_due(void) { + static SRWLOCK lock = SRWLOCK_INIT; + static int ms = -2; + static int64_t last = -1, said; + static long let, held; + AcquireSRWLockExclusive(&lock); + if (ms == -2) { + const char *e = getenv("HOS_OOT_TEX_ASK_MS"); + ms = hos_oot_tex_ask_ms(e); + if (ms > 0) hos_log("oot: texture compaction: at most one ask every %d ms (HOS_OOT_TEX_ASK_MS; 0: every ask, as shipped)", ms); + else hos_log("oot: texture compaction: %s (HOS_OOT_TEX_ASK_MS=%s)", ms ? "no ask let through" : "every ask, as shipped", e); + } + const int64_t now = hos_now_ns(); + const int due = hos_oot_tex_ask_due(ms, now, &last); + if (due) let++; + else held++; + if (!said) said = now; + if (now - said >= 10000000000LL) { + if (let || held) + hos_log("oot: texture compaction asks in %.0f s: %ld let through, %ld held (HOS_OOT_TEX_ASK_MS=%d)", + (now - said) / 1e9, let, held, ms); + said = now, let = held = 0; + } + ReleaseSRWLockExclusive(&lock); + return due; +} diff --git a/host/oot_tex_compact.h b/host/oot_tex_compact.h new file mode 100644 index 0000000..65a8901 --- /dev/null +++ b/host/oot_tex_compact.h @@ -0,0 +1,39 @@ +/* (Triforce) When the game's texture data manager may ask its compaction thread for a round (host/oot_tex_compact.c + hos_oot_tex_compact_due, nx2/oot_patches.py patch_tex_ask), apart from the game so that tests/test_oot_tex_compact.py can + compile it alone. + + The manager steps every frame (f_80797240): with no round asked for (its +776) and bit 1 of its +960 set, it asks for one + and sends the thread the message, so a round could start every frame. Each round moves texture data in the game's heap, + and each texture moved is a new address to the renderer: captured, compared and hashed again. With a gap, an ask is let + through only once that gap has passed since the last one let through; a round started always finishes, and the game asks + again at its next step. + + HOS_OOT_TEX_ASK_MS=: at most one ask every n ms. Unset, empty or not a number: 1000. 0: every ask, as shipped. off (or a + negative number): none. */ +#pragma once + +#include +#include + +enum { HOS_OOT_TEX_ASK_DEFAULT_MS = 1000 }; + +/* The switch's value: the gap in ms, 0 every ask, -1 none */ +static inline int hos_oot_tex_ask_ms(const char *e) { + if (!e || !*e) return HOS_OOT_TEX_ASK_DEFAULT_MS; + if (e[0] == 'o' && e[1] == 'f' && e[2] == 'f' && !e[3]) return -1; + char *end; + const long v = strtol(e, &end, 10); + if (end == e || *end) return HOS_OOT_TEX_ASK_DEFAULT_MS; + if (v < 0) return -1; + return v > 3600000 ? 3600000 : (int)v; +} + +/* Whether an ask goes through now. ms: the switch's value; now: the clock (ns); last: the time of the last ask let through, + negative before the first, set here when this one goes through */ +static inline int hos_oot_tex_ask_due(int ms, int64_t now, int64_t *last) { + if (ms == 0) return 1; + if (ms < 0) return 0; + if (*last >= 0 && now - *last < (int64_t)ms * 1000000) return 0; + *last = now; + return 1; +} diff --git a/host/pgo_rt.c b/host/pgo_rt.c index 19d5973..d322546 100644 --- a/host/pgo_rt.c +++ b/host/pgo_rt.c @@ -83,7 +83,12 @@ static DWORD WINAPI pgo_thread(void *arg) { return 0; } +/* ((Triforce, 11-21, the performance audit's LIFT-10) the renderer's copy (triforce_vk, nx2/build.py RX_VK_PGO=gen) is + compiled with PGO_ENV "RX_VK_PGO_FILE": its counters are the DLL's own sections, written to its own file) */ +#ifndef PGO_ENV +#define PGO_ENV "HOS_PGO" +#endif __attribute__((constructor)) static void pgo_start(void) { - const char *e = getenv("HOS_PGO"); + const char *e = getenv(PGO_ENV); if (e) CreateThread(NULL, 0, pgo_thread, _strdup(e), 0, NULL); } diff --git a/host/prompts.c b/host/prompts.c index 0987f65..32869e9 100644 --- a/host/prompts.c +++ b/host/prompts.c @@ -833,7 +833,13 @@ int hos_prompts_romfs(const char *key, char *out, size_t cap); finds that one first) */ int hos_prompts_font_file(const char *key, char *out, size_t cap) { if (strcmp(key, FONT_PATH)) return 0; - return hos_prompts_romfs(FONT_PATH, out, cap); + /* ((Triforce, 11-21, the performance audit's IO-24) from the lifted code's size hook (host/mods.c), a game thread: the + core given up around the builder, which builds the font at its first call and holds its lock meanwhile; never inside + hos_prompts_romfs, where that lock is held) */ + const int away = hos_vcore_away(); + const int found = hos_prompts_romfs(FONT_PATH, out, cap); + if (away) hos_vcore_back(); + return found; } /* (Triforce) The title screen's third entry, "Options / ZELDA NOTES Setting" in US English, reads "Options / Triforce": diff --git a/host/rx_backend.c b/host/rx_backend.c index 9e667c9..27d7e3e 100644 --- a/host/rx_backend.c +++ b/host/rx_backend.c @@ -400,6 +400,8 @@ void hos_guest_io_end(int held) { has nothing left, or there is none. Both back end calls are found by name, as the lazy write-back's; without them nothing is told and the wait returns at once */ static void (*volatile submit_waits_fn)(const uint64_t *, int); /* (the back end's mkwvk_submit_waits, once it runs) */ +/* ((Triforce, 11-22, the performance audit's THREAD-25) the back end's mkwvk_invalidate_quick, once it runs: rx_nvn_invalidate) */ +static int (*volatile invalidate_quick_fn)(uint64_t, uint64_t); static int (*volatile replay_idle_wait_fn)(uint64_t); int rx_async_replay(void) { static volatile int on = -1; @@ -455,6 +457,8 @@ int rx_start(void *hwnd) { /* (Triforce: the replay thread's two calls, rx_async_replay above; a back end without them has none) */ submit_waits_fn = (void (*)(const uint64_t *, int))(void *)GetProcAddress(vk_module, "mkwvk_submit_waits"); replay_idle_wait_fn = (int (*)(uint64_t))(void *)GetProcAddress(vk_module, "mkwvk_replay_idle_wait"); + /* ((Triforce, THREAD-25) an invalidate with nothing to wait for; a back end without it is asked nothing) */ + invalidate_quick_fn = (int (*)(uint64_t, uint64_t))(void *)GetProcAddress(vk_module, "mkwvk_invalidate_quick"); hos_log("rx: renderer: %s (%s)", eden ? "Eden's" : "Vulkan", dll); flush_early(); return 1; @@ -596,9 +600,24 @@ void rx_nvn_texture_created(const nvn_tex *t) { if (vk && vk->nvn_texture_created) vk->nvn_texture_created(t); } void rx_nvn_program_released(const nvn_prog *p) { vk ? vk->nvn_program_released(p) : wgr_nvn_program_released(p); } +/* ((Triforce, 11-21, the performance audit's THREAD-20, after Source A's) ReadTexels and WriteTexels reach the back end, which + waits for all the GPU work submitted (its finish) and then for the replay's lock: a GPU drain of 6 ms and more, at an + area's arrival and at boot, where the game calls them from its loading threads. The thread kept its virtual core through + that wait, and the game threads allowed only on that core waited with it, up to the 250 ms escape. No guest code runs in + the call, so the core is given up for it, as around rx_nvn_invalidate below. HOS_TEXELS_KEEP_CORE=1: kept, as before */ +static int texels_keep_core(void) { + static int keep = -1; + if (keep < 0) { + const char *e = getenv("HOS_TEXELS_KEEP_CORE"); + keep = e && atoi(e) != 0; + } + return keep; +} void rx_nvn_read_texels(const nvn_tex *t, unsigned level, unsigned layer0, const NVNcopyRegion *r, uint8_t *dst, size_t row, size_t img) { + const int away = texels_keep_core() ? 0 : hos_vcore_away(); vk ? vk->nvn_read_texels(t, level, layer0, r, dst, row, img) : wgr_nvn_read_texels(t, level, layer0, r, dst, row, img); + if (away) hos_vcore_back(); } void rx_nvn_write_texels(const nvn_tex *t, unsigned level, unsigned layer0, const NVNcopyRegion *r, const uint8_t *src, size_t row, size_t img) { @@ -620,10 +639,18 @@ void rx_nvn_write_texels(const nvn_tex *t, unsigned level, unsigned layer0, cons } } } + /* ((Triforce, 11-21, THREAD-20) as the read: no core held through the back end's wait. The early path above never waits) */ + const int away = texels_keep_core() ? 0 : hos_vcore_away(); vk ? vk->nvn_write_texels(t, level, layer0, r, src, row, img) : wgr_nvn_write_texels(t, level, layer0, r, src, row, img); + if (away) hos_vcore_back(); } void rx_nvn_invalidate(uint64_t gpu, uint64_t size) { + /* ((Triforce, 11-22, the performance audit's THREAD-25, after Source A's) The game invalidates each shape's 64-byte + occlusion slot, per view, and nearly none of those calls has anything to wait for; each gave the core up and queued + for it again. The back end is asked first, without a lock and without a wait (gpu/vk/page_marks.h has what it + looks at): 1, and the call is left out with the core kept. It answers 0 unless RX_VK_INVALIDATE_QUICK=1 */ + if (vk && invalidate_quick_fn && invalidate_quick_fn(gpu, size)) return; /* The callback can wait for replay or GPU write-back: let another guest thread use this core meanwhile. */ const int away = hos_vcore_away(); int hos_ft_on(void); diff --git a/host/shadow_slot.h b/host/shadow_slot.h new file mode 100644 index 0000000..b8fb222 --- /dev/null +++ b/host/shadow_slot.h @@ -0,0 +1,52 @@ +/* (Triforce, 11-21, the performance audit's TEX-85, after Source A's) The storage a character's shadow map reserves + (host/nvn_native.c nvnTextureBuilderGetStorageSize and nvnTextureBuilderGetStorageAlignment). + + The game gives each shadow-casting character near the camera a square depth map (NVN format 0x34, one level) whose + side it works out anew every frame from the camera's distance: 384 from afar, more from near, a few pixels different + each frame while either moves. It makes a frame's render targets one after another, first fit, in its target heap, + asking the size and alignment of each here. So a map a few pixels larger pushes every target made after it to another + address, and each of those is a new texture to the game and, keyed by its address, a new image to the renderer: + Source A counted 15 to 16 textures made a frame in the Market with the camera turning, and 6 with this. + + Such a map is answered the storage size and alignment of the next multiple of a step up (384: sides 768, 1152, 1536, + 1920), so the targets after it stay where they are until the map crosses a step. Only the bytes the game sets aside + change, by less than one step's worth a map (1.8 MB at the first step, 5.3 MB at the worst); the texture keeps its own + size and layout, and nothing is drawn or sampled differently. + + HOS_SHADOW_BUCKET=: the step, 384 when not set; 0: every map its own size, as before. Not set, the step is 0 + where the target heap is pinned (HOS_RT_POOL_MB, or the 5120-wide outputs' 900 MB), where there is no room to give. + + The rule alone, no NVN: tests/test_shadow_slot.py compiles it. */ +#ifndef HOS_SHADOW_SLOT_H +#define HOS_SHADOW_SLOT_H + +#include + +enum { HOS_SHADOW_SLOT_STEP = 384, HOS_SHADOW_SLOT_FORMAT = 0x34, HOS_SHADOW_SLOT_LEAST = 384, HOS_SHADOW_SLOT_MOST = 1920 }; + +/* The side whose storage a texture of this description reserves: the next multiple of step from its own, for a square + two-dimensional one-level depth map wider than 384 and up to 1920, and never more than 1920. 0: its own (not such a + map, or step 0) */ +static inline uint32_t hos_shadow_slot_side(int is_2d, int format, uint32_t levels, uint32_t depth, uint32_t width, uint32_t height, + uint32_t step) { + if (!step || !is_2d || format != HOS_SHADOW_SLOT_FORMAT || levels != 1 || depth > 1 || width != height || + width <= HOS_SHADOW_SLOT_LEAST || width > HOS_SHADOW_SLOT_MOST) + return 0; + const uint32_t side = (width + step - 1) / step * step; + return side > HOS_SHADOW_SLOT_MOST ? HOS_SHADOW_SLOT_MOST : side; +} + +/* The step: asked, HOS_SHADOW_BUCKET's value (NULL: not set); pinned: the target heap is held to a size (no room to give). + Set: its value, 0 for off or a negative number. Not set: 0 on a pinned heap, else HOS_SHADOW_SLOT_STEP */ +static inline uint32_t hos_shadow_slot_step(const char *asked, int pinned) { + if (asked) { + long v = 0; + int digits = 0; + const char *p = asked; + for (; *p >= '0' && *p <= '9' && v < 100000; p++) v = v * 10 + (*p - '0'), digits++; + return digits && !*p ? (uint32_t)v : *asked ? 0 : HOS_SHADOW_SLOT_STEP; /* (not a number: off; empty: as not set) */ + } + return pinned ? 0 : HOS_SHADOW_SLOT_STEP; +} + +#endif diff --git a/host/short_sleep.h b/host/short_sleep.h new file mode 100644 index 0000000..a2c0a19 --- /dev/null +++ b/host/short_sleep.h @@ -0,0 +1,115 @@ +/* (Triforce, 11-21, the performance audit's OS-31, after Source A's) How sleeps under a millisecond wait. + + The game's idle job workers, and its main thread at its barriers, sleep 20 us and look again, about 7,600 times a + second. A sleep under a millisecond is yielded away until its time (the timer wakes about 0.5 ms late, which made + those waits 25 times longer on the frame's critical path), so each sleeper keeps a processor busy. On a PC with few + processors that is time the frame's own threads need: Source A measured, with the game held to 4 processors, a tenth + more frames, no frame over 250 ms where the yielding runs had them, and on 6 the same frame rate for a tenth less CPU; + from 8 on, no gain. So with 6 or fewer logical processors in the process's mask the game's own short sleeps wait on + the high-resolution timer, and the host's own (the frame limiter, the audio renderer) keep yielding. Above 6 nothing + changes. + + HOS_SHORT_SLEEP_TIMER: not set, by the processors; 1 (any value that is not off): every short sleep on the timer; 0, + off, no, false: every short sleep yields. + + (Triforce, 11-22, the performance audit's OS-32, after Source A's) Off Windows the processors are not counted, so + every short sleep yielded there whatever the PC, the Steam Deck's 8 logical processors included. Source A, off + Windows and with the switch not set, puts the game's own short sleeps on the timer at any processor count (the + timer there is the POSIX shim's clock_nanosleep, which wakes tens of microseconds late, not half a millisecond), + so idle workers stop holding processors the frame needs. That mode was never measured in the game by itself (with + every short sleep on the timer a 24-thread Linux PC lost 2.9% of its frame rate for a fifth less CPU), and it + cannot be run on the PC this was written on: it is built, and is not the default. HOS_SHORT_SLEEP_TIMER=game asks + for it on any system; SHORT_SLEEP_ELSEWHERE below is what a build that is not Windows' does with the switch not + set: SHORT_SLEEP_YIELD, as before, until a Steam Deck A/B says SHORT_SLEEP_GUEST_TIMER (Source A's default). + + The rule alone, no threads and no clock: tests/test_short_sleep_timer.py compiles it. */ +#pragma once +#include +#include + +enum { SHORT_SLEEP_TIMER_THREADS = 6 }; /* (no more than the game's six cores) */ +enum { SHORT_SLEEP_YIELD = 0, SHORT_SLEEP_GUEST_TIMER = 1, SHORT_SLEEP_ALL_TIMER = 2 }; + +/* A switch's value that turns it off: 0, off, no, false (and nothing at all) */ +static inline int short_sleep_value_off(const char *v) { + return !v || !*v || !strcmp(v, "0") || !strcmp(v, "off") || !strcmp(v, "no") || !strcmp(v, "false"); +} + +/* How short sleeps wait. asked: HOS_SHORT_SLEEP_TIMER's value, NULL when it is not set. threads: the logical processors + the process may run on, 0 when not known (more than one processor group, or not Windows) */ +static inline int short_sleep_timer(const char *asked, int threads) { + if (asked) return short_sleep_value_off(asked) ? SHORT_SLEEP_YIELD : SHORT_SLEEP_ALL_TIMER; + return threads > 0 && threads <= SHORT_SLEEP_TIMER_THREADS ? SHORT_SLEEP_GUEST_TIMER : SHORT_SLEEP_YIELD; +} + +/* ((Triforce, 11-22, the performance audit's OS-32) what a build that is not Windows' does with the switch not set */ +#define SHORT_SLEEP_ELSEWHERE SHORT_SLEEP_YIELD + +/* The same with the system: windows, a Windows build (the processors count there); elsewhere, what any other build does + with the switch not set (SHORT_SLEEP_ELSEWHERE). "game": the game's own short sleeps on the timer, the host's + yield, at any processor count on any system */ +static inline int short_sleep_timer_os(const char *asked, int threads, int windows, int elsewhere) { + if (asked && !strcmp(asked, "game")) return SHORT_SLEEP_GUEST_TIMER; + if (!asked && !windows) return elsewhere; + return short_sleep_timer(asked, threads); +} + +/* Whether a sleep of ns is yielded away until its time. guest: the game's own sleep (SleepThread), not the host's */ +static inline int short_sleep_yields(int64_t ns, int how, int guest) { + return ns < 1000000 && how < (guest ? SHORT_SLEEP_GUEST_TIMER : SHORT_SLEEP_ALL_TIMER); +} + +/* (Triforce, 11-22, the performance audit's OS-33, after Source A's) A cap on the game threads that yield through a short + sleep at once. Where the game's short sleeps yield (more than 6 logical processors), every idle job worker keeps a + processor busy. Source A lets at most a quarter of the processors' worth of game threads do so at a time (2 with 8, 4 + with 16, 6 with 24); the next sleeper waits on the timer, about 0.5 ms late. Its numbers are provisional: with 16 + processors 2.7% less CPU a frame for the same frame rate; a 24-thread run held to 2 yielding threads lost 1.7% of its + frame rate. The game's MainThread and Presentation Thread, which the frame waits on, and the host's own sleeps are + never counted and always yield. + + Off unless asked for, as it changes how the game's threads wait and was not measured here: + HOS_SHORT_SLEEP_POLLERS not set, or 0, off, no, false: no limit, as before; a number: at most that many (1024 at + most); auto (any other word): a quarter of the process's logical processors when they are more than 6, else no limit. + Any HOS_SHORT_SLEEP_TIMER value turns the cap off (the switch then says how every short sleep waits), and so does a + choice other than "all yield". */ +enum { SHORT_SLEEP_POLLERS_MOST = 1024 }; + +/* The cap: 0 for no limit. asked: HOS_SHORT_SLEEP_POLLERS's value, NULL when not set; timer_asked: HOS_SHORT_SLEEP_TIMER's; + how: short_sleep_timer's choice; threads: the process's logical processors, 0 when not known */ +static inline int short_sleep_pollers(const char *asked, const char *timer_asked, int how, int threads) { + if (timer_asked || how != SHORT_SLEEP_YIELD || short_sleep_value_off(asked)) return 0; + if (asked[0] >= '0' && asked[0] <= '9') { + int n = 0; + for (const char *p = asked; *p >= '0' && *p <= '9' && n <= SHORT_SLEEP_POLLERS_MOST; p++) n = n * 10 + (*p - '0'); + return n > SHORT_SLEEP_POLLERS_MOST ? SHORT_SLEEP_POLLERS_MOST : n; + } + return threads > SHORT_SLEEP_TIMER_THREADS ? threads / 4 : 0; +} + +/* The choice kept in one word, made once: how, and the cap above it */ +static inline int short_sleep_word(int how, int cap) { return how | cap << 2; } +static inline int short_sleep_how(int word) { return word & 3; } +static inline int short_sleep_cap(int word) { return word >> 2; } + +/* A thread the frame waits on, never counted: the game's MainThread (by any longer name too) and its Presentation Thread */ +static inline int short_sleep_serial_name(const char *name) { + return name && (strstr(name, "MainThread") != NULL || !strcmp(name, "Presentation Thread")); +} + +/* The cap's gate, around the yielding. 1: this sleep yields; 0: it waits on the timer. pollers: the counted threads + yielding now. cap 0, a host sleep or a serial thread: not counted, yields. A counted one yields while it is one of at + most `cap`; one over the cap gives its count back here. *took: the count is held and must be given back with + short_sleep_poll_leave once the yielding is over: one decrement on every path */ +static inline int short_sleep_poll_enter(int *pollers, int cap, int guest, int serial, int *took) { + *took = 0; + if (cap <= 0 || !guest || serial) return 1; + if (__atomic_add_fetch(pollers, 1, __ATOMIC_RELAXED) <= cap) { + *took = 1; + return 1; + } + __atomic_sub_fetch(pollers, 1, __ATOMIC_RELAXED); + return 0; +} +static inline void short_sleep_poll_leave(int *pollers, int took) { + if (took) __atomic_sub_fetch(pollers, 1, __ATOMIC_RELAXED); +} diff --git a/host/stall_host.h b/host/stall_host.h new file mode 100644 index 0000000..e8da5fb --- /dev/null +++ b/host/stall_host.h @@ -0,0 +1,143 @@ +/* (Triforce, 11-22, the performance audit's OS-51, after Source B's) The host's part of a long frame. + + The renderer's stall line (gpu/vk/stall_watch.h) splits a frame of 50 ms or more on the replay's side and calls the + rest "outside the replay and the present". This is that rest, counted where the host spends it: the game's file + reads, writes, flushes and save commits, the discards, zero-fills and decommits of memory the game lets go, the + give-back's passes, and the waits for a virtual core; with the process's page faults and its CPU time in the kernel + and in its own code over the same frame. Each cause is added to with relaxed atomics where it happens (a clock read + before and after: nothing else), and at each present all are taken: a present that comes the stall line's threshold + or more after the last one logs them as one line beside the stall line, at most 3 a second and 1500 in a session: + + vi: stall, the host's part: frame 4812, 143.2 ms: file reads 3 (12.0 MB, longest 41.0 ms); discards 2 (180.0 MB, + longest 6.1 ms); core waits 64.0 ms (212, longest 31.0); 41000 page faults; kernel 38.0 ms, user 120.0 + + Source B's found a read pacing of its own that way, sleeps of 36 to 151 ms at boot and area loads. Triforce has no + such pacing: what is ported is the log. No frame gets shorter by it; it names, or rules out, the host's causes. + + On whenever the stall line is (RX_VK_STALL_LOG, its threshold; 0 turns both off). HOS_STALL_HOST=0: off, nothing + counted and no system call at a present. + + Plain C, no system calls: tests/test_stall_host.py compiles it. */ +#ifndef STALL_HOST_H +#define STALL_HOST_H +#include +#include +#include + +enum { + STALL_HOST_READ, /* nn::fs ReadFile: the kernel's read into the game's memory (the virtual core given up) */ + STALL_HOST_WRITE, /* nn::fs WriteFile (the core held) */ + STALL_HOST_FLUSH, /* nn::fs FlushFile (the core held) */ + STALL_HOST_COMMIT, /* nn::fs Commit: the save staged (the core given up) */ + STALL_HOST_DISCARD, /* a freed region's or freed pages' discard */ + STALL_HOST_ZERO, /* freed pages a GPU import holds, zeroed where they are */ + STALL_HOST_DECOMMIT, /* FreeMemoryPages' decommit above 4 GB */ + STALL_HOST_GIVE_BACK, /* a pass of the give-back of pages let go */ + STALL_HOST_CORE_WAIT, /* waits for a virtual core, summed over the game's threads */ + STALL_HOST_CAUSES +}; + +/* One cause since the last present: the time in it, how often, the bytes it was about, its longest single time */ +typedef struct { int64_t ns, count, bytes, longest; } StallHostCause; + +static inline void stall_host_add(StallHostCause *c, int64_t ns, int64_t bytes) { + if (ns < 0) ns = 0; + __atomic_add_fetch(&c->ns, ns, __ATOMIC_RELAXED); + __atomic_add_fetch(&c->count, 1, __ATOMIC_RELAXED); + if (bytes) __atomic_add_fetch(&c->bytes, bytes, __ATOMIC_RELAXED); + int64_t was = __atomic_load_n(&c->longest, __ATOMIC_RELAXED); + while (ns > was && !__atomic_compare_exchange_n(&c->longest, &was, ns, 1, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) {} +} +/* What was counted, the counters left at zero (an add in the middle of a take lands in this frame or the next) */ +static inline StallHostCause stall_host_take(StallHostCause *c) { + StallHostCause t; + t.ns = __atomic_exchange_n(&c->ns, 0, __ATOMIC_RELAXED); + t.count = __atomic_exchange_n(&c->count, 0, __ATOMIC_RELAXED); + t.bytes = __atomic_exchange_n(&c->bytes, 0, __ATOMIC_RELAXED); + t.longest = __atomic_exchange_n(&c->longest, 0, __ATOMIC_RELAXED); + return t; +} + +/* The threshold in microseconds, 0 for off. stall_log: RX_VK_STALL_LOG's value (NULL: not set, the stall line's 50 ms; a + number: that many ms; 0 or no number: no stall line, and none of this); host: HOS_STALL_HOST's (0: off) */ +static inline int64_t stall_host_threshold_us(const char *stall_log, const char *host) { + if (host && atoi(host) == 0) return 0; + const double ms = stall_log ? atof(stall_log) : 50.0; + return ms > 0 ? (int64_t)(ms * 1000.0 + 0.5) : 0; +} + +/* At most per_second lines a second (a burst of that many after a quiet stretch) and total in a session: the stall + line's own bound (stall_watch.h Bound) */ +typedef struct { double per_second, tokens; int64_t last_ns, total, told, held; } StallHostBound; +#define STALL_HOST_BOUND {3.0, 3.0, 0, 1500, 0, 0} +static inline int stall_host_bound_take(StallHostBound *b, int64_t now_ns) { + if (b->last_ns) { + b->tokens += (double)(now_ns - b->last_ns) / 1e9 * b->per_second; + if (b->tokens > b->per_second) b->tokens = b->per_second; + } + b->last_ns = now_ns; + if (b->told >= b->total || b->tokens < 1.0) { + b->held++; + return 0; + } + b->tokens -= 1.0; + b->told++; + return 1; +} + +/* One long frame's account */ +typedef struct { + int64_t frame, gap_ns; /* the present's number, the time since the one before */ + StallHostCause cause[STALL_HOST_CAUSES]; + uint64_t page_faults; /* soft and hard together, as Windows counts them */ + int64_t kernel_ns, user_ns; /* the process's CPU time over the frame, all threads */ + int64_t held; /* lines the bound left out since the last one */ +} StallHostFrame; + +/* The line. Causes that did not happen are left out; with none, it says so (the frame's time was not the host's) */ +static inline int stall_host_line(const StallHostFrame *f, char *out, size_t cap) { + static const char *const name[STALL_HOST_CAUSES] = {"file reads", "file writes", "flushes", "save commits", "discards", + "zero-fills", "decommits", "give-backs", "core waits"}; + int n = snprintf(out, cap, "vi: stall, the host's part: frame %lld, %.1f ms:", (long long)f->frame, f->gap_ns / 1e6); + int any = 0; + for (int i = 0; i < STALL_HOST_CAUSES && n > 0 && (size_t)n < cap; i++) { + const StallHostCause *c = &f->cause[i]; + if (!c->count) continue; + const char *sep = any ? (i == STALL_HOST_DISCARD || i == STALL_HOST_CORE_WAIT ? ";" : ",") : ""; + any = 1; + if (i == STALL_HOST_CORE_WAIT) /* (summed over threads: more than the frame's own time is usual) */ + n += snprintf(out + n, cap - n, "%s core waits %.1f ms (%lld, longest %.1f)", sep, c->ns / 1e6, (long long)c->count, + c->longest / 1e6); + else if (c->bytes) + n += snprintf(out + n, cap - n, "%s %s %lld (%.1f MB, %.1f ms, longest %.1f)", sep, name[i], (long long)c->count, + c->bytes / 1048576.0, c->ns / 1e6, c->longest / 1e6); + else + n += snprintf(out + n, cap - n, "%s %s %lld (%.1f ms, longest %.1f)", sep, name[i], (long long)c->count, c->ns / 1e6, + c->longest / 1e6); + } + if (n > 0 && (size_t)n < cap) + n += snprintf(out + n, cap - n, "%s %llu page faults; kernel %.1f ms, user %.1f", any ? ";" : " none of the host's causes;", + (unsigned long long)f->page_faults, f->kernel_ns / 1e6, f->user_ns / 1e6); + if (f->held && n > 0 && (size_t)n < cap) n += snprintf(out + n, cap - n, " (%lld such lines left out before this one)", (long long)f->held); + return n; +} + +/* ---- the counters, and the two calls a counted place makes (host/mkw_glue.c has the counters and the present's look) ---- */ +extern StallHostCause hos_stall_host[STALL_HOST_CAUSES]; +extern int64_t hos_stall_host_us; /* (the threshold; -1: the switches not read yet) */ +int64_t hos_now_ns(void); + +static inline int64_t stall_host_us(void) { + int64_t us = __atomic_load_n(&hos_stall_host_us, __ATOMIC_RELAXED); + if (us < 0) { + us = stall_host_threshold_us(getenv("RX_VK_STALL_LOG"), getenv("HOS_STALL_HOST")); + __atomic_store_n(&hos_stall_host_us, us, __ATOMIC_RELAXED); + } + return us; +} +/* The clock before a counted call, 0 when nothing is counted; and the call counted after it */ +static inline int64_t stall_host_begin(void) { return stall_host_us() ? hos_now_ns() : 0; } +static inline void stall_host_end(int cause, int64_t began, uint64_t bytes) { + if (began) stall_host_add(&hos_stall_host[cause], hos_now_ns() - began, (int64_t)bytes); +} +#endif diff --git a/host/video_av1.h b/host/video_av1.h new file mode 100644 index 0000000..d2c3813 --- /dev/null +++ b/host/video_av1.h @@ -0,0 +1,19 @@ +/* (Triforce, 11-22, the performance audit's IO-32, after Source A's) Which AV1 decoder a movie gets first on Windows + (nn_video.c dec_init_sized). + + The game's movies are 1080p AV1. Windows decoded them with Media Foundation first (the AV1 Video Extension, where the PC + has it), and that path takes the extension's synchronous decoder with no D3D manager: software decoding on every GPU. + dav1d, through the FFmpeg the build links, was the fallback for a PC without the extension, and what Linux always + runs. It goes first now where that FFmpeg has it; Media Foundation stays the decoder of an FFmpeg without libdav1d, + and the fallback when libdav1d does not open. HOS_AV1=mf: Media Foundation first, as before. + + The rule alone, no decoder: tests/test_video_av1_order.py compiles it. */ +#pragma once +#include + +/* 1 when Media Foundation is tried before libdav1d. hos_av1: HOS_AV1's value, NULL when unset; have_dav1d: the FFmpeg + this build links has libdav1d. (HOS_AV1=dav1d, the old way to ask for dav1d first, is the default now) */ +static inline int video_av1_mf_first(const char *hos_av1, int have_dav1d) { + if (!have_dav1d) return 1; + return hos_av1 != NULL && !strcmp(hos_av1, "mf"); +} diff --git a/host/video_catchup.h b/host/video_catchup.h new file mode 100644 index 0000000..a37ca85 --- /dev/null +++ b/host/video_catchup.h @@ -0,0 +1,46 @@ +/* (Triforce, 11-22, the performance audit's THREAD-82, after Source A's) How far the movie decoder's acquire decodes + ahead to catch up with the movie's clock (nn_video.c dec_acquire). + + The game takes one picture a frame and the sound plays on, so the acquire skips the pictures that are behind the + movie's clock, decoding the next ones to skip them too. Nothing bounded that but a guard of 600 steps. With a + decoder that takes longer than the gap between two pictures (19.5 ms a picture against 16.7), every picture decoded + to catch up puts the clock further ahead: the acquire decoded up to about 300 pictures, near 6 s with its lock and + the movie thread held, threw nearly all of them away, and the next acquire started further behind. + + Now each decode that gives a picture is timed, about the last 8 of them kept as one number, and the acquire stops + decoding to catch up when that number is more than the gap between pictures, or once it has decoded for one gap: + the late picture is shown. A decoder faster than the movie catches up as before, a few pictures an acquire. + HOS_VIDEO_UNBOUNDED_CATCHUP=1: no bound, as before. + + The rule alone, no decoder and no clock: tests/test_video_catchup.py compiles it. */ +#pragma once +#include + +typedef struct { + int64_t n; /* pictures decoded since first_pts */ + int64_t first_pts, end_pts; /* the first of them and the newest, by the movie's own time (ns) */ + int64_t recent_ns; /* what a decode that gives a picture takes: about the last 8 */ +} VideoPace; + +/* a picture came out of the decoder, with its time in the movie. One whose time is before the last one's is a movie + that started again (a loop): the gap is counted from it */ +static inline void video_pace_picture(VideoPace *p, int64_t pts) { + if (!p->n || pts < p->end_pts) p->n = 0, p->first_pts = pts; + p->n++; + p->end_pts = pts; +} +/* the submission that gave it took ns to decode: seven parts of the old number and one of this */ +static inline void video_pace_decoded(VideoPace *p, int64_t ns) { + if (ns < 0) ns = 0; + p->recent_ns = (p->recent_ns * 7 + ns) / 8; +} +/* the gap between two pictures as the movie plays, in ns; 0 while fewer than two came out */ +static inline int64_t video_pace_spacing(const VideoPace *p) { + return p->n > 1 && p->end_pts > p->first_pts ? (p->end_pts - p->first_pts) / (p->n - 1) : 0; +} +/* 1 when the acquire stops decoding to catch up and shows the late picture it has. spent_ns: how long this acquire's + catch-up has run. Never with the switch, and never before the gap is known */ +static inline int video_catchup_stop(int unbounded, int64_t spacing_ns, int64_t recent_ns, int64_t spent_ns) { + if (unbounded || spacing_ns <= 0) return 0; + return recent_ns > spacing_ns || spent_ns >= spacing_ns; +} diff --git a/host/video_input.h b/host/video_input.h new file mode 100644 index 0000000..5553170 --- /dev/null +++ b/host/video_input.h @@ -0,0 +1,33 @@ +/* (Triforce, 11-22, the performance audit's IO-34, after Source A's) How many submissions the movie decoder holds before + it refuses one (nn_video.c dec_submit). + + The game's movie library submits a whole movie at once, on the thread that also takes the pictures, and every + submission was taken. While it submits it takes a picture only now and then, so at a movie's start the pictures + that came due meanwhile were skipped by the catch-up: a stutter at each movie's start. The console's decoder holds + only so much input and refuses a submission with a result of its own while it is full; the library submits it again + later. So does this one now: while 16 submissions wait, not decoded yet or decoded and not taken, a submission is + refused at once with that result. The stream's last submission is never refused: nothing would mark the end. The + decoder's "ready for a submission" event is clear while it is full and set again when there is room. + HOS_VIDEO_INPUT_LIMIT=: that many instead of 16; 0: no limit, every submission taken, as before. + + The rule alone, no decoder: tests/test_video_input_limit.py compiles it. */ +#pragma once +#include + +#define VIDEO_INPUT_LIMIT 16 /* submissions waiting, pending or decoded, before one is refused */ +#define VIDEO_RESULT_INPUT_FULL 0x1c266fu /* the console's result for a submission its decoder has no room for */ + +/* The limit from HOS_VIDEO_INPUT_LIMIT's value (NULL or empty: unset): 0 is no limit, and so is anything not a + positive number */ +static inline int video_input_limit(const char *value) { + if (!value || !*value) return VIDEO_INPUT_LIMIT; + const int n = atoi(value); + return n > 0 ? n : 0; +} +/* 1 while the decoder holds as much as the limit. pending: submissions not decoded yet; queued: pictures decoded and + not taken */ +static inline int video_input_full(int limit, int pending, int queued) { return limit > 0 && pending + queued >= limit; } +/* 1 when this submission is refused. last: it is the stream's last */ +static inline int video_input_refuse(int limit, int pending, int queued, int last) { + return !last && video_input_full(limit, pending, queued); +} diff --git a/host/video_movie.h b/host/video_movie.h new file mode 100644 index 0000000..b73688f --- /dev/null +++ b/host/video_movie.h @@ -0,0 +1,45 @@ +/* (Triforce, 11-22, the performance audit's IO-32, after Source A's) What one movie cost to decode: a count kept by the + decoder (nn_video.c decode_next, queue_frame, the catch-up in dec_acquire) and one line in the log when the movie is + stopped, flushed or its decoder finalized. Nothing was timed before: whether a decoder keeps up with a movie (a + picture is due every 16.7 ms) could only be guessed from the pictures skipped. + + (Triforce, 11-22, THREAD-82) The line also has the gap between two pictures as the movie plays, and says when the + decoder is slower than that: such a movie falls behind its sound (video_catchup.h). + + The counts and the line alone, no decoder and no clock: tests/test_video_av1_order.py compiles them. */ +#pragma once +#include +#include + +typedef struct { + int64_t pictures; /* pictures decoded since the movie's start */ + int64_t decode_ns; /* the time its submissions took to decode, pictures made ready for the game included */ + int64_t worst_ns; /* the longest of them */ + int64_t skipped; /* pictures dropped to catch up with the movie's clock */ + int64_t refused; /* ((Triforce, 11-22, IO-34) submissions refused while the decoder's input was full: video_input.h) */ +} VideoMovie; + +/* one submission decoded, in ns by the clock */ +static inline void video_movie_decoded(VideoMovie *m, int64_t ns) { + if (ns < 0) ns = 0; + m->decode_ns += ns; + if (ns > m->worst_ns) m->worst_ns = ns; +} + +/* The movie's line into out (without the log's "video: "): 0 and nothing written when no picture was decoded, so a + Stop followed by a Flush, or a decoder finalized after its Stop, says it once. spacing_ns: the gap between two + pictures as the movie plays, 0 when it is not known (video_catchup.h) */ +static inline int video_movie_line(const VideoMovie *m, const char *decoder, int64_t spacing_ns, char *out, size_t size) { + if (m->pictures <= 0) return 0; + char shown[80] = ""; + if (spacing_ns > 0) + snprintf(shown, sizeof shown, ", one shown every %.2f ms%s", (double)spacing_ns / 1e6, + m->decode_ns / m->pictures > spacing_ns ? " (slower than it plays)" : ""); + char refused[80] = ""; + if (m->refused > 0) + snprintf(refused, sizeof refused, ", %lld submissions refused while its input was full", (long long)m->refused); + snprintf(out, size, "a movie of %lld pictures on %s: %.2f ms to decode each (worst %.2f)%s; %lld skipped to catch up%s", + (long long)m->pictures, decoder, (double)m->decode_ns / 1e6 / (double)m->pictures, (double)m->worst_ns / 1e6, + shown, (long long)m->skipped, refused); + return 1; +} diff --git a/host/video_nv12.h b/host/video_nv12.h new file mode 100644 index 0000000..258ee42 --- /dev/null +++ b/host/video_nv12.h @@ -0,0 +1,42 @@ +/* (Triforce, 11-22, the performance audit's IO-36, after Source A's) A decoded movie picture kept as NV12 alone + (nn_video.c queue_frame). + + Media Foundation's pictures come as NV12 and were kept as that: the pool makes RGBA from it only if the game reads + RGBA (pool_rgba), which this game's movie library does not. libdav1d's come as planar 4:2:0 (Y, U and V planes), and + each one was converted to RGBA on the movie thread all the same: a full-size swscale pass, an 8 MB allocation and + an 8 MB row copy for a 1080p picture, with its NV12 made beside it in a zeroed buffer, one byte at a time. Planar + 4:2:0 is now kept as NV12 alone too. HOS_VIDEO_RGBA=1: its RGBA made for every picture, as before. + + The rules and the U/V interleave alone, no decoder: tests/test_video_lazy_rgba.py compiles them. */ +#pragma once +#include +#include +#if (defined(__x86_64__) || defined(_M_X64)) && !defined(__EMSCRIPTEN__) +#include +#define VIDEO_NV12_SSE2 1 +#endif + +/* 1 when a picture is kept as NV12 alone, its RGBA left to the pool that is read. nv12: the decoder gave NV12 (always + lazy, as before); planar420: it gave planar 4:2:0 without alpha (with alpha the RGBA is made at once: one made later + from NV12 would have lost it); eager: HOS_VIDEO_RGBA=1 */ +static inline int video_rgba_lazy(int eager, int nv12, int planar420) { return nv12 || (planar420 && !eager); } + +/* 1 when the picture's NV12 buffer (w * h * 3 / 2 bytes) is zeroed before it is filled. written: the format is one + queue_frame copies into it (NV12, planar 4:2:0 with or without alpha); any other leaves it as it is, so it is zeroed, + as the buffer always was. An odd width or height leaves bytes no copy reaches: a chroma row's last byte, and the + rows past h / 2 */ +static inline int video_nv12_zero(int w, int h, int written) { return ((w | h) & 1) != 0 || !written; } + +/* One row of NV12's chroma from a row of U and a row of V: n pairs, uv[2x] = u[x] and uv[2x + 1] = v[x]. 16 pairs a + step with SSE2 on x86-64, then one at a time */ +static inline void video_uv_interleave(uint8_t *uv, const uint8_t *u, const uint8_t *v, int n) { + int x = 0; +#ifdef VIDEO_NV12_SSE2 + for (; x + 16 <= n; x += 16) { + const __m128i us = _mm_loadu_si128((const __m128i *)(u + x)), vs = _mm_loadu_si128((const __m128i *)(v + x)); + _mm_storeu_si128((__m128i *)(uv + 2 * (size_t)x), _mm_unpacklo_epi8(us, vs)); + _mm_storeu_si128((__m128i *)(uv + 2 * (size_t)x + 16), _mm_unpackhi_epi8(us, vs)); + } +#endif + for (; x < n; x++) uv[2 * (size_t)x] = u[x], uv[2 * (size_t)x + 1] = v[x]; +} diff --git a/launcher/App/Backend.Bench.cs b/launcher/App/Backend.Bench.cs index 3a65051..522faba 100644 --- a/launcher/App/Backend.Bench.cs +++ b/launcher/App/Backend.Bench.cs @@ -176,7 +176,7 @@ public sealed partial class Backend /// The menu's own words for the values the benchmark touches (Triforce.Logic.GameOptionsSchema). static string BenchLabel(string key, int v, IReadOnlyList all, bool after = false) { - if (GameOptionsSchema.ByKey(key) is not { } row) return v.ToString(); + if (GameOptionsSchema.ByKey(key) is not { } row) return GameOptionsSchema.Part(key)?.Label(v) ?? v.ToString(); // ((Triforce, options_version 8) a merged row's part: Steady frame rate, Frame generation) if (key == Benchmark.KResolution) { // (its label follows the Upscaler: "Quality (67%)" with DLSS / FSR, "67%" without) diff --git a/launcher/App/Backend.Core.cs b/launcher/App/Backend.Core.cs index eef463d..57ebe25 100644 --- a/launcher/App/Backend.Core.cs +++ b/launcher/App/Backend.Core.cs @@ -328,15 +328,21 @@ public sealed partial class Backend r.Line += l => { lock (logOut) logOut.AppendLine(l); }; var last = DateTime.MinValue; var eb = req.IsUpdate ? (release != null && workKind == "update" ? "Updating to " + release.Tag : "Building your game again") : "Building your game"; + // (the lines come on two threads, the tick on a third: one at a time) + var pushGate = new object(); + var ticking = true; void Push(bool force = false) { - if (!force && (DateTime.UtcNow - last).TotalMilliseconds < 250) return; - last = DateTime.UtcNow; - var e = r.Estimate ?? est; - var left = BuildClock.Remaining(e, r.Tracker); - WorkScreen(eb, BuildClock.LeftText(left), 0.08 + 0.92 * Fraction(e, r.Tracker), Steps(null, (e, r.Tracker)), - "Keep this window open; you can minimize it. You get a notice when the game is ready.", true, - req.IsUpdate ? null : new[] { ("Your game", "done"), ("Build", "now"), ("Play", "") }); + lock (pushGate) + { + if (!force && (DateTime.UtcNow - last).TotalMilliseconds < 250) return; + last = DateTime.UtcNow; + var e = r.Estimate ?? est; + var left = BuildClock.Remaining(e, r.Tracker); + WorkScreen(eb, BuildClock.LeftText(left), 0.08 + 0.92 * Fraction(e, r.Tracker), Steps(null, (e, r.Tracker)), + "Keep this window open; you can minimize it. You get a notice when the game is ready.", true, + req.IsUpdate ? null : new[] { ("Your game", "done"), ("Build", "now"), ("Play", "") }); + } } r.Event += _ => Push(); r.StageChanged += (_, _) => Push(true); @@ -344,7 +350,15 @@ public sealed partial class Backend Go("work"); using var reg = token.Register(() => r.Stop()); lastFailure = null; - var res = lastResult = await r.RunAsync(req, token); + // (a step can take minutes with no line: the time left counts down, or holds, every second all the same) + var tick = new Timer(_ => { lock (pushGate) if (ticking) Push(); }, null, 1000, 1000); + BuildResult res; + try { res = lastResult = await r.RunAsync(req, token); } + finally + { + lock (pushGate) ticking = false; + tick.Dispose(); + } Push(true); FlushLog(); logTimer?.Dispose(); logTimer = null; diff --git a/launcher/App/Backend.GameOpts.cs b/launcher/App/Backend.GameOpts.cs index 0745e60..78505aa 100644 --- a/launcher/App/Backend.GameOpts.cs +++ b/launcher/App/Backend.GameOpts.cs @@ -23,16 +23,21 @@ public sealed partial class Backend if (!CanChangeGame(out var f) || GameOptionsSchema.ByKey(key) is not { } row) return; var now = f!.Get(key); var text = H(row.Help ?? "") + (row.Restart ? "
Takes effect the next time the game starts." : ""); - if (key == Benchmark.KPreset) - text += "
" + string.Join("
", GameOptionsSchema.PresetSummaries.Select((s, i) => i == 0 ? "" : "" + H(f.ChoicesOf(key).ElementAtOrDefault(i)?.Label) + " " + H(s)).Where(x => x.Length > 0)); var choices = f.ChoicesOf(key).AsEnumerable(); if (key == Benchmark.KUpscaler) { - // (as the game's menu offers them: DLSS and DLAA with an NVIDIA card and DLSS in the build; FSR 4 on AMD cards) + // (as the game's menu offers them: DLSS (and DLAA, its Native) with an NVIDIA card and DLSS in the build; FSR 4 on AMD cards) var dlss = Gpus.HasNvidia() && new Game(ctx).HasDlss(GameDir!); var amd = Gpus.List().Any(x => x.Vendor == GpuVendor.Amd); choices = choices.Where(c => c.Value == now || (c.Value is not (1 or 2) || dlss) && (c.Value is not (4 or 5) || amd)); } + if (key == GameOptionsSchema.FrameGenKey) + { + // ((Triforce, options_version 8) NVIDIA DLSS 2x to 4x, merged into the row, where DLSS frame generation can run: + // an NVIDIA card and DLSS in the build, as the game's menu offers them (vk_menu.cpp dlssg_offered)) + var dlssg = Gpus.HasNvidia() && new Game(ctx).HasDlss(GameDir!); + choices = choices.Where(c => c.Value == now || c.Value < 10 || dlssg); + } Choice(row.Name, text, "gopt.set", choices.Select(c => (c.Value.ToString(), c.Label, (string?)null, c.Value == now)), "Choose", key); }); On("gopt.set", m => @@ -74,7 +79,7 @@ public sealed partial class Backend { var g = GameDir; if (g == null) return null; - if (!gpusHooked) { GameOptionsSchema.UseGameGpus(() => GameDir); gpusHooked = true; } // ((Triforce) 11-19: the game's own order) + if (!gpusHooked) { GameOptionsSchema.UseGameGpus(() => GameDir); GameOptionsSchema.UseSystemLatency(); gpusHooked = true; } // ((Triforce) 11-19: the game's own order; options_version 8: Low latency by the card's maker) GameOptionsFile f; try { f = GameOptionsFile.Load(g, MainScreen()); } catch (Exception e) { Log.Write("game options: " + e.Message); return null; } diff --git a/launcher/App/Host/WinHost.cs b/launcher/App/Host/WinHost.cs index 3e1e5a3..76be58e 100644 --- a/launcher/App/Host/WinHost.cs +++ b/launcher/App/Host/WinHost.cs @@ -305,11 +305,11 @@ public sealed class WinHost : IHost [DllImport("user32", CharSet = CharSet.Unicode)] static extern ushort RegisterClassEx(ref WNDCLASSEX c); [DllImport("user32", CharSet = CharSet.Unicode)] static extern IntPtr CreateWindowEx(uint ex, string cls, string name, uint style, int x, int y, int w, int h, IntPtr parent, IntPtr menu, IntPtr inst, IntPtr p); - [DllImport("user32")] static extern IntPtr DefWindowProc(IntPtr h, uint m, IntPtr w, IntPtr l); - [DllImport("user32")] static extern int GetMessage(out MSG m, IntPtr h, uint a, uint b); + [DllImport("user32", CharSet = CharSet.Unicode)] static extern IntPtr DefWindowProc(IntPtr h, uint m, IntPtr w, IntPtr l); + [DllImport("user32", CharSet = CharSet.Unicode)] static extern int GetMessage(out MSG m, IntPtr h, uint a, uint b); [DllImport("user32")] static extern bool TranslateMessage(ref MSG m); - [DllImport("user32")] static extern IntPtr DispatchMessage(ref MSG m); - [DllImport("user32")] static extern bool PostMessage(IntPtr h, uint m, IntPtr w, IntPtr l); + [DllImport("user32", CharSet = CharSet.Unicode)] static extern IntPtr DispatchMessage(ref MSG m); + [DllImport("user32", CharSet = CharSet.Unicode)] static extern bool PostMessage(IntPtr h, uint m, IntPtr w, IntPtr l); [DllImport("user32")] static extern void PostQuitMessage(int c); [DllImport("user32")] static extern bool ShowWindow(IntPtr h, int c); [DllImport("user32")] static extern bool UpdateWindow(IntPtr h); @@ -318,8 +318,8 @@ public sealed class WinHost : IHost [DllImport("user32")] static extern bool SetForegroundWindow(IntPtr h); [DllImport("user32")] static extern bool GetClientRect(IntPtr h, out RECT r); [DllImport("user32")] static extern bool SetWindowPos(IntPtr h, IntPtr after, int x, int y, int w, int hh, uint f); - [DllImport("user32")] static extern IntPtr LoadCursor(IntPtr i, int id); - [DllImport("user32")] static extern IntPtr LoadIcon(IntPtr i, IntPtr id); + [DllImport("user32", CharSet = CharSet.Unicode)] static extern IntPtr LoadCursor(IntPtr i, int id); + [DllImport("user32", CharSet = CharSet.Unicode)] static extern IntPtr LoadIcon(IntPtr i, IntPtr id); [DllImport("user32")] static extern uint GetDpiForSystem(); [DllImport("user32")] static extern bool SetProcessDpiAwarenessContext(IntPtr v); [DllImport("user32")] static extern bool SystemParametersInfo(uint a, uint b, ref RECT r, uint c); diff --git a/launcher/App/Triforce.Launcher.csproj b/launcher/App/Triforce.Launcher.csproj index e33f15d..0551987 100644 --- a/launcher/App/Triforce.Launcher.csproj +++ b/launcher/App/Triforce.Launcher.csproj @@ -12,6 +12,9 @@ enable true 1.0.0 + + Triforce + Triforce diff --git a/launcher/Logic/Benchmark.cs b/launcher/Logic/Benchmark.cs index 16cc90f..b33f8fb 100644 --- a/launcher/Logic/Benchmark.cs +++ b/launcher/Logic/Benchmark.cs @@ -71,9 +71,11 @@ public sealed class Benchmark : IBenchmarkRunner // the options file's keys (gpu/vk/vk_menu.cpp ini_key; values gpu/vk/vk_menu.h, upscaler_rows.h, fps_cap.h) public const string KCap = "frame_rate_cap", KVsync = "vsync", KSteady = "steady_frame_rate", KUpscaler = "dlss", - KResolution = "resolution", KPreset = "quality_preset", KFrameGen = "frame_generation"; + KResolution = "resolution", KFrameGen = "frame_generation"; // frame_rate_cap: 0 60, 1 90, 2 120, 3 144, 4 165, 5 240, 6 the monitor's. dlss (the Upscaler row): 0 the game's, 1 DLSS, - // 3 FSR 3.1, 4 FSR 4. resolution: 3 = 67 % (Quality), 5 = 58 % (Balanced), 4 = 50 % (Performance). quality_preset 0 Custom. + // 3 FSR 3.1, 4 FSR 4. resolution: 3 = 67 % (Quality), 5 = 58 % (Balanced), 4 = 50 % (Performance). ((Triforce, options_version 8) + // steady_frame_rate and frame_generation are the saved values of the merged Frame pacing and Frame generation rows; the + // Quality preset is gone: the game drops its line) static readonly (int value, int fps)[] Caps = { (0, 60), (2, 120), (3, 144), (5, 240) }; public const int UpGame = 0, UpDlss = 1, UpFsr3 = 3, ResQuality = 3, ResBalanced = 5, CapUncapped = 5; public const int SteadyRefreshSteps = 2; // (fps_cap.h: 0 Off, 1 Any rate, 2 Refresh steps) @@ -172,7 +174,7 @@ public sealed class Benchmark : IBenchmarkRunner try { var keys = new Dictionary { [KCap] = CapUncapped, [KVsync] = 0, [KSteady] = 0, [KFrameGen] = 0 }; - if (upscaler is { } u) { keys[KUpscaler] = u.upscaler; keys[KResolution] = ResQuality; keys[KPreset] = 0; } + if (upscaler is { } u) { keys[KUpscaler] = u.upscaler; keys[KResolution] = ResQuality; } BenchIni.Set(ini, keys); var si = new ProcessStartInfo(exe) { UseShellExecute = false, WorkingDirectory = game, RedirectStandardError = true, RedirectStandardOutput = true }; foreach (var a in new[] { Path.Combine(game, "image.bin"), "--romfs", Path.Combine(game, "romfs"), "--user", userCopy }) si.ArgumentList.Add(a); @@ -297,8 +299,6 @@ public sealed class Benchmark : IBenchmarkRunner recs.Add(new SettingChange(KSteady, "Steady frame rate", Now(KSteady, 0), uneven ? SteadyRefreshSteps : Now(KSteady, 0))); // frame generation: never turned on by this (DESIGN 12: only below 60 with DLSS-G / FSR 4 FG, not measured yet) recs.Add(new SettingChange(KFrameGen, "Frame generation", Now(KFrameGen, 0), Now(KFrameGen, 0))); - if (recs.Any(r => r.Changes && (r.Key == KUpscaler || r.Key == KResolution))) - recs.Add(new SettingChange(KPreset, "Quality preset", Now(KPreset, 0), 0)); return recs; } diff --git a/launcher/Logic/Build/BuildProtocol.cs b/launcher/Logic/Build/BuildProtocol.cs index 194c6a6..b8231e9 100644 --- a/launcher/Logic/Build/BuildProtocol.cs +++ b/launcher/Logic/Build/BuildProtocol.cs @@ -111,7 +111,7 @@ public static class BuildProtocol } /// The build's state as its lines come (old Shell.cs Line/Ended; the Tk builder's parser): the stage it is in, that -/// stage's progress, when each stage began, the plan, the time left the compile said, the folder @@done named, the last +/// stage's progress (and how it moved), when each stage began, the plan, the time left the compile said, the folder @@done named, the last /// @@error. Fed line by line by BuildRunner, or by a test from a transcript. public sealed class BuildTracker { @@ -149,10 +149,20 @@ public sealed class BuildTracker Stage = s.Id; StageStarted = t; StageProgress = 0; + lock (Gate) { recent.Clear(); recent.Add((t, 0)); } if (s.Id != "compile") EtaSeconds = null; StageChanged?.Invoke(was, s.Id); break; - case BuildEvent.Progress p: StageProgress = p.Fraction; break; + case BuildEvent.Progress p: + StageProgress = p.Fraction; + lock (Gate) + { + if (recent.Count == 0) recent.Add((StageStarted, 0)); + if (p.Fraction < recent[^1].Progress) { recent.Clear(); recent.Add((now(), p.Fraction)); } // (it went back: start over) + else if (p.Fraction > recent[^1].Progress) recent.Add((now(), p.Fraction)); + if (recent.Count > KeepSamples) recent.RemoveRange(0, recent.Count - KeepSamples / 2); + } + break; case BuildEvent.Plan p: Plan = (p.Changed, p.Total); break; case BuildEvent.FileBuilt f when f.How is "new" or "again": FilesBuilt++; break; case BuildEvent.Eta x: EtaSeconds = x.Seconds; break; @@ -177,4 +187,15 @@ public sealed class BuildTracker public TimeSpan Elapsed => now() - Started; public TimeSpan InStage => Stage == null ? TimeSpan.Zero : now() - StageStarted; + public DateTime Now => now(); + + /// This stage's progress as it moved: (when, how far), from (the stage's start, 0), a sample each time it went up (the + /// last 2000), for BuildClock's rate over the last minutes. + public IReadOnlyList<(DateTime At, double Progress)> Recent { get { lock (Gate) return recent.ToArray(); } } + + const int KeepSamples = 2000; + readonly List<(DateTime At, double Progress)> recent = new(); + /// The time left last shown and when (BuildClock.Remaining's smoothing), under Gate: the App asks from more than one thread. + internal (double Seconds, DateTime At)? Shown; + internal readonly object Gate = new(); } diff --git a/launcher/Logic/Build/BuildRunner.cs b/launcher/Logic/Build/BuildRunner.cs index fc19340..158f20c 100644 --- a/launcher/Logic/Build/BuildRunner.cs +++ b/launcher/Logic/Build/BuildRunner.cs @@ -1,5 +1,6 @@ // nx2/make_game.py as a child process (old Setup.cs Runner, Shell.cs Run/Line/Ended; hookshot_builder.py start_make/on_exit): -// -I \nx2\make_game.py cwd , stdout+stderr UTF-8, below-normal priority (Windows) +// -I \nx2\make_game.py cwd , stdout+stderr UTF-8, normal priority (below normal on +// Windows only with HOOKSHOT_LOW_PRIORITY=1) // env: PYTHONIOENCODING=utf-8, HOOKSHOT_BUILDER_EXE (written into the game folder's builder= line), HOOKSHOT_NGX_SDK when // DLSS is on, EMSDK / RX_CPM / WGPU_NATIVE / HOOKSHOT_MINGW for tool folders that exist (Windows); Linux: DLSS on drops // HOOKSHOT_VK_PREBUILT (the Vulkan back end is compiled with NGX), DLSS off drops HOOKSHOT_NGX_SDK @@ -195,7 +196,12 @@ public sealed class BuildRunner var si = StartInfo(ctx, tools, args, ctx.NgxSdkFolder, dlss); using var p = ctx.Platform.StartTree(si); process = p; - try { if (ctx.IsWindows) p.PriorityClass = ProcessPriorityClass.BelowNormal; } catch (Exception) { } + // (Triforce, 11-22, the performance audit's OS-57, after Source A's) The build tools run at normal priority; + // below normal, as every build was before, only with HOOKSHOT_LOW_PRIORITY=1. Source A's reason, a report it + // did not measure: below normal, Windows kept the steps that are one process (the read, the lift) on a hybrid + // Intel CPU's efficiency cores while the performance cores idled. + if (Environment.GetEnvironmentVariable("HOOKSHOT_LOW_PRIORITY") == "1") + try { if (ctx.IsWindows) p.PriorityClass = ProcessPriorityClass.BelowNormal; } catch (Exception) { } try { p.StandardInput.Close(); } catch (Exception) { } p.ErrorDataReceived += (_, e) => { if (e.Data != null) Take(e.Data); }; p.BeginErrorReadLine(); diff --git a/launcher/Logic/Build/BuildTimes.cs b/launcher/Logic/Build/BuildTimes.cs index 975faba..0a83786 100644 --- a/launcher/Logic/Build/BuildTimes.cs +++ b/launcher/Logic/Build/BuildTimes.cs @@ -115,6 +115,10 @@ public static class BuildEstimator { GpuVendor.Nvidia => (TimeSpan.FromMinutes(5), TimeSpan.FromMinutes(20)), GpuVendor.Amd => (TimeSpan.FromMinutes(30), TimeSpan.FromMinutes(120)), + // ((Triforce, 11-22) Intel's Arc cards: an A750's preparation took 4.3 h on v0.6.0 (#bug-tracker "Intel Arc A750 current + // state", 12 threads), an A770's 11 h; with the compute warm cut to the listed programs (RX_VK_WARM_COMPUTES) a + // re-preparation is over an hour shorter. The 10 to 40 minutes before had the time left show minutes for hours) + GpuVendor.Intel => (TimeSpan.FromMinutes(60), TimeSpan.FromMinutes(240)), _ => (TimeSpan.FromMinutes(10), TimeSpan.FromMinutes(40)), }; @@ -136,6 +140,13 @@ public static class BuildEstimator { if (!shaders) continue; if (stages != null && stages.Seconds.TryGetValue("shaders", out var ss) && ss > 0) { list.Add(Exact(st.Id, ss)); continue; } + // ((Triforce, 11-22) an update with no update of its own on record: the shader step this PC measured in its + // last build of any kind, before the stated range (an update after a full build prepares the same shaders) + if (history.Stages.LastOrDefault(r => r.Seconds.TryGetValue("shaders", out var any) && any > 0) is { } anyRec) + { + list.Add(Exact(st.Id, anyRec.Seconds["shaders"])); + continue; + } var (sl, sh) = ShaderRange(gpu); list.Add(new StageEstimate(st.Id, sl, sh, false)); continue; @@ -171,9 +182,34 @@ public static class BuildClock return Unit(low) == Unit(high) ? M(low) + " to " + M(high) + " " + Unit(high) : M(low) + " " + Unit(low) + " to " + M(high) + " " + Unit(high); } - /// The time left: the stages after this one by the estimate, and this one by @@eta (compile), else by its progress so - /// far (from 5 %), else its estimate less what it has run. Never below zero; "Almost done" is the App's words under a minute. + /// The time left as shown: Estimated, smoothed. Each call counts the shown time down by the time since the last one and + /// moves it toward the new estimate: down within seconds (FallSeconds), up slowly (RiseSeconds) and never faster than half a + /// second a second, so one odd moment never makes it jump. The first call of a build shows the estimate itself. Never below + /// zero; "Almost done" is the App's words under a minute. public static TimeSpan Remaining(BuildEstimate estimate, BuildTracker tracker) + { + var raw = Estimated(estimate, tracker).TotalSeconds; + lock (tracker.Gate) + { + var now = tracker.Now; + var shown = raw; + if (tracker.Shown is { } was) + { + var dt = Math.Max(0, (now - was.At).TotalSeconds); + var counted = Math.Max(0, was.Seconds - dt); + shown = raw < counted ? counted + (raw - counted) * (1 - Math.Exp(-dt / FallSeconds)) + : Math.Min(counted + (raw - counted) * (1 - Math.Exp(-dt / RiseSeconds)), was.Seconds + MaxRise * dt); + } + tracker.Shown = (shown, now); + return TimeSpan.FromSeconds(Math.Max(0, shown)); + } + } + + const double FallSeconds = 15, RiseSeconds = 30, MaxRise = 0.5; + + /// The time left now, unsmoothed: the stages after this one by the estimate, and this one by @@eta (compile), else by + /// StageLeft. Never below zero. + public static TimeSpan Estimated(BuildEstimate estimate, BuildTracker tracker) { var at = tracker.Stage is { } s ? BuildStages.IndexOf(s) : -1; var left = TimeSpan.Zero; @@ -181,15 +217,63 @@ public static class BuildClock if (BuildStages.IndexOf(st.Stage) > at) left += st.Mid; if (at >= 0 && estimate.Of(tracker.Stage!) is { } cur) { - TimeSpan here; - if (tracker.Stage == "compile" && tracker.EtaSeconds is { } eta) here = TimeSpan.FromSeconds(eta); - else if (tracker.StageProgress >= 0.05) here = tracker.InStage * ((1 - tracker.StageProgress) / tracker.StageProgress); - else here = cur.Mid - tracker.InStage; + var here = tracker.Stage == "compile" && tracker.EtaSeconds is { } eta ? TimeSpan.FromSeconds(eta) + : StageLeft(cur.Mid, tracker.Recent, tracker.StageStarted, tracker.Now); left += here < TimeSpan.Zero ? TimeSpan.Zero : here; } return left; } + /// A stage's time left by its progress. The work per step is uneven (the shader step: on AMD and Intel cards the first + /// part, the compute pipelines, can be most of its time, an Arc A750 spent 5175 s of 5858 s on them; one slow shader holds it + /// for minutes), so the rate since the stage began says hours where there is an hour. Instead: + /// the rate of the last WindowSeconds of progress, each wait between two steps counted as at most 4 times the usual one (and + /// at least half a minute), so one slow item is one step late, not a new pace; + /// blended with the estimate's own pace (what it leaves of the stage, at least a tenth of it, and past it a quarter of the + /// time over too), trusting the measured rate more as the stage goes (all of it from TrustFrom of the way); + /// reckoned at the last step: between steps it counts down for one usual wait, then holds, and after a long stall (2 + /// minutes, or 4 waits) rises by a quarter of the time. + /// No progress yet: the estimate less the time run (the same floor). + public static TimeSpan StageLeft(TimeSpan expected, IReadOnlyList<(DateTime At, double Progress)> samples, DateTime began, DateTime now) + { + var e = expected.TotalSeconds; + static double Prior(double e, double run) => Math.Max(e - run, 0.1 * e + 0.25 * Math.Max(0, run - e)); + if (samples.Count < 2 || samples[^1].Progress <= 0) return TimeSpan.FromSeconds(Prior(e, Math.Max(0, (now - began).TotalSeconds))); + var (last, p) = samples[^1]; + if (p >= 1) return TimeSpan.Zero; + var run = Math.Max(0, (last - began).TotalSeconds); + var rateHere = (1 - p) / Math.Max(1, Prior(e, run)); + var recent = RecentRate(samples); + var w = recent == null ? 0 : Math.Clamp(p / TrustFrom, 0, 1); + var rate = w * (recent?.Rate ?? 0) + (1 - w) * rateHere; + var atStep = (1 - p) / rate; + var wait = recent?.Wait ?? 0; + var stall = Math.Max(0, (now - last).TotalSeconds); + var left = atStep - Math.Min(stall, wait) + 0.25 * Math.Max(0, stall - Math.Max(120, 4 * wait)); + return TimeSpan.FromSeconds(Math.Max(0, left)); + } + + const double WindowSeconds = 180, TrustFrom = 0.3; + + /// Progress a second over the last WindowSeconds (each wait capped), and the usual wait between steps (the median of + /// the last 64); null with no step yet. + static (double Rate, double Wait)? RecentRate(IReadOnlyList<(DateTime At, double Progress)> s) + { + if (s.Count < 2) return null; + var waits = new List(); + for (var i = s.Count - 1; i >= 1 && waits.Count < 64; i--) waits.Add((s[i].At - s[i - 1].At).TotalSeconds); + waits.Sort(); + var usual = waits[waits.Count / 2]; + var cap = Math.Max(30, 4 * usual); + double span = 0, moved = 0; + for (var i = s.Count - 1; i >= 1 && span < WindowSeconds; i--) + { + span += Math.Min(cap, (s[i].At - s[i - 1].At).TotalSeconds); + moved += s[i].Progress - s[i - 1].Progress; + } + return span <= 0 || moved <= 0 ? null : (moved / span, usual); + } + /// "About 31 minutes left"; "Almost done" under a minute (DESIGN 6 Build). public static string LeftText(TimeSpan left) => left.TotalSeconds < 60 ? "Almost done" : left.TotalMinutes < 90 ? "About " + (int)Math.Round(left.TotalMinutes) + " minutes left" : "About " + (int)left.TotalHours + " h " + left.Minutes + " min left"; diff --git a/launcher/Logic/Game.cs b/launcher/Logic/Game.cs index 03ec69e..e27cddd 100644 --- a/launcher/Logic/Game.cs +++ b/launcher/Logic/Game.cs @@ -148,6 +148,9 @@ public sealed class Game if (DropPrefixes.Any(p => key.StartsWith(p, StringComparison.Ordinal)) || DropNames.Contains(key)) si.Environment.Remove(key); si.RedirectStandardOutput = si.RedirectStandardError = true; si.RedirectStandardInput = true; + // ((Triforce, 11-21) the game's log into triforce_log.txt all the same: with stderr a pipe the game kept it there, + // and the pipe's lines are dropped below, so a session started with Play wrote no log (host/hos_main.c HOS_LOG_FILE) + si.Environment["HOS_LOG_FILE"] = "1"; } si.Environment["HOOKSHOT_NO_UPDATE_CHECK"] = "1"; var p = ctx.IsWindows ? Process.Start(si) : ctx.Platform.StartTree(si); diff --git a/launcher/Logic/GameOptions/GameOptionRow.cs b/launcher/Logic/GameOptions/GameOptionRow.cs index 923ccf2..8143078 100644 --- a/launcher/Logic/GameOptions/GameOptionRow.cs +++ b/launcher/Logic/GameOptions/GameOptionRow.cs @@ -1,6 +1,8 @@ // The game's in-game Options (gpu/vk/vk_menu.cpp, the Row table) as plain data for the launcher: one GameOptionRow per row the -// game keeps in triforce_options.ini, with its tab, section, name, key, range, default, help and the labels its value shows. -// GameOptionsSchema.All lists them; GameOptionsFile reads and writes the file the way the game does. +// game shows with a value, with its tab, section, name, key, range, default, help and the labels its value shows. +// GameOptionsSchema.All lists them; GameOptionsFile reads and writes the file the way the game does. ((Triforce, options_version +// 8) A merged row (HDR, Frame generation, Frame pacing, Low latency: gpu/vk/merged_rows.h) has no line of its own: it is a view +// of two saved values, its Parts, each kept under its own key as before.) namespace Triforce.Logic; /// The Options screen's tabs, as the game's header shows them. (The game's Debug tab is not ported: its rows' @@ -15,8 +17,8 @@ public enum GameOptionKind { Choice, Slider } public sealed record GameOptionChoice(int Value, string Label); /// The values of the other rows, for the labels that depend on them (the Render resolution's sizes follow the -/// Upscaler, the Output resolution and the Handheld profile; the Anti-aliasing row shows the upscaler's own while one -/// draws; the Microphone shows the saved device's name). GameOptionsFile is one. +/// Upscaler and the Output resolution; the Anti-aliasing row shows the upscaler's own while one draws; the Microphone shows +/// the saved device's name). GameOptionsFile is one. public interface IGameOptionValues { /// A row's value as the game reads it (its default when the file has no line). @@ -25,37 +27,59 @@ public interface IGameOptionValues string? Raw(string key); } -/// A row of the game's Options menu that has a value in triforce_options.ini. +/// ((Triforce, options_version 8) A saved value that a merged row stands for (vk_menu.cpp's TAB_HIDDEN rows): its key +/// in triforce_options.ini, its range and default (vk_menu.h Options), and its own value's label (the hidden row's tx_*). +public sealed record GameOptionPart(string Key, int Lo, int Hi, int Default, Func Label) +{ + /// A value as the game reads it from the file: inside Lo..Hi. + public int AsRead(int value) => Math.Clamp(value, Lo, Hi); +} + +/// A merged row's rules (merged_rows.h): its value from its parts' (Of), and its parts' from a new value of it and +/// their values now (Split). +public sealed record GameOptionMerge(Func Of, Func Split); + +/// A row of the game's Options menu that has a value in triforce_options.ini (its own line, or its Parts' lines). public sealed record GameOptionRow { /// The tab it is on. The Motion page's rows and the Controls page's Button prompts are on Controls, in /// the sections "Motion" and "Controls" (the rows that open those pages). public required GameOptionTab Tab { get; init; } - /// The heading above it on its tab (the menu's section rows), null where the tab has none (Audio). + /// The heading above it on its tab (the menu's MENU_HEADING rows), null where the tab has none. public required string? Section { get; init; } /// The row's name, exactly as the menu shows it. public required string Name { get; init; } - /// Its key in triforce_options.ini (vk_menu.cpp ini_key). + /// Its key in triforce_options.ini (vk_menu.cpp ini_key). A merged row has no line: its key here starts with '@' + /// and is the launcher's alone (its values are its Parts' keys). public required string Key { get; init; } public required int Lo { get; init; } public required int Hi { get; init; } public required int Step { get; init; } - /// The game's default (vk_menu.h Options): the value when the file has no line for it. - public required int Default { get; init; } + readonly int dflt; + /// The game's default (vk_menu.h Options): the value when the file has no line for it. A merged row's: the view + /// of its parts' defaults (Low latency's follows the card's maker). + public int Default { get => Merge != null ? Merge.Of(Parts!.Select(p => p.Default).ToArray()) : dflt; init => dflt = value; } public required GameOptionKind Kind { get; init; } /// The help under the list while the row is selected, exactly the menu's. public required string Help { get; init; } /// Read once at the game's start: a change applies after a restart (the menu tags it "After restart"). public required bool Restart { get; init; } - /// A change here leaves the Quality preset alone; any other row changed by hand turns it back to Custom. + /// The table's preset_exempt flag. (Triforce, options_version 8) The Quality preset is gone and nothing reads it; + /// the rows keep their shape, so the flag is kept as the table has it. public required bool PresetExempt { get; init; } + /// A merged row's saved values, in merged_rows.h's order (null for a row with a line of its own). + public IReadOnlyList? Parts { get; init; } + /// A merged row's rules (null for a row with a line of its own). + public GameOptionMerge? Merge { get; init; } + /// A merged row: a view of its Parts, no line of its own. + public bool IsMerged => Merge != null; internal Func Format { get; init; } = (v, _) => v.ToString(System.Globalization.CultureInfo.InvariantCulture); internal Func>? ChoiceList { get; init; } /// The values a Choice row offers, in the menu's order, with their labels as the menu shows them where nothing /// is known of this PC (fps cap 6 "Monitor's rate", GPU device from GameOptionsSchema.GpuNames). Empty for a Slider. - /// GameOptionsFile.ChoicesOf has the ones that follow the file's other rows (the Render resolution). + /// GameOptionsFile.ChoicesOf has the ones that follow the file's other rows (the Upscaler, the Render resolution). public IReadOnlyList Choices => ChoiceList?.Invoke() ?? Array.Empty(); /// The label of a value, as the menu shows it with the other rows at their defaults. diff --git a/launcher/Logic/GameOptions/GameOptionsFile.cs b/launcher/Logic/GameOptions/GameOptionsFile.cs index 72605f4..ba89a54 100644 --- a/launcher/Logic/GameOptions/GameOptionsFile.cs +++ b/launcher/Logic/GameOptions/GameOptionsFile.cs @@ -81,14 +81,17 @@ public sealed class GameOptionsFile : IGameOptionValues return (int)Math.Clamp(v, int.MinValue, int.MaxValue); } - /// Each row's value as the game reads the file at its start (menu_init): the last line's, in the row's range; - /// then the rules for a file of an older options_version (the defaults that changed: such a file wrote the old one - /// whether chosen or not), the old dynamic_resolution key, the Microphone without a device's name. + /// Each saved value as the game reads the file at its start (menu_init), by its key (every row's with a line of its + /// own, and the merged rows' parts): the last line's, in its range; then the rules for a file of an older options_version + /// (the defaults that changed: such a file wrote the old one whether chosen or not; version 8's moves of the rows that + /// went), the old dynamic_resolution key, the Microphone without a device's name. Dictionary Effective(Dictionary lines, bool exists) { var v = new Dictionary(StringComparer.Ordinal); - foreach (var r in GameOptionsSchema.All) + foreach (var r in GameOptionsSchema.All.Where(r => !r.IsMerged)) v[r.Key] = exists && lines.TryGetValue(r.Key, out var s) ? r.AsRead(Atoi(s)) : r.Default; + foreach (var p in GameOptionsSchema.Parts.Values) + v[p.Key] = exists && lines.TryGetValue(p.Key, out var s) ? p.AsRead(Atoi(s)) : p.Default; if (!exists) return v; foreach (var (key, value) in Migrated(lines)) if (v.ContainsKey(key)) v[key] = value; if (!lines.ContainsKey(GameOptionsSchema.ResolutionKey) && lines.TryGetValue("dynamic_resolution", out var dyn)) @@ -108,6 +111,20 @@ public sealed class GameOptionsFile : IGameOptionValues { if (!lines.TryGetValue(key, out var s) || Atoi(s) != value) list.Add((key, value)); } + // ((Triforce, options_version 8) merged_rows.h migrate: the handheld profile is the Output resolution's 1536x864, the + // shared stick dead zone each stick's own where it had none of its own. The other rows that went move nothing; their + // lines stay until the game's next save drops them. A 436f9ac-based build's 7 has none of these lines: nothing moves) + if (version < 8) + { + if (lines.TryGetValue("handheld_profile", out var hh) && Atoi(hh) != 0) Force(GameOptionsSchema.OutputResKey, GameOptionsSchema.HandheldOutput); + var old = lines.TryGetValue("stick_dead_zone", out var sz) ? Atoi(sz) : 0; + var shared = old is > 0 and < 6 ? GameOptionsSchema.SharedZone[old] : -1; + foreach (var key in new[] { "left_stick_dead_zone", "right_stick_dead_zone" }) + { + var own = lines.TryGetValue(key, out var s) ? GameOptionsSchema.Get(key).AsRead(Atoi(s)) : -1; + if (own < 0 && shared != own) list.Add((key, shared)); + } + } if (!hasVersion) Force("game_speed", 1); if (version < 4) Force("descriptor_reuse", 1); if (version < 5) @@ -142,12 +159,21 @@ public sealed class GameOptionsFile : IGameOptionValues // ---- the values ---- - /// A row's value: the launcher's change, else what the game reads from the file (its default without a line). - /// KeyNotFoundException for a key that is no row (Raw has those). + /// A saved value by its key (a row's own line or a merged row's part): the launcher's change, else what the game + /// reads from the file (its default without a line). + int Value(string key) => set.TryGetValue(key, out var v) ? v : read[key]; + + /// A row's value as the game's menu shows it: the launcher's change, else what the game reads from the file (its + /// default without a line). A merged row's: the view of its parts (merged_rows.h); the Render resolution's: NativeStep while + /// the Upscaler is an engine's native AA (its saved value is then left for the engine's other steps). A merged row's part + /// by its own key: its saved value. KeyNotFoundException for any other key (Raw has those). public int Get(string key) { - GameOptionsSchema.Get(key); - return set.TryGetValue(key, out var v) ? v : read[key]; + if (GameOptionsSchema.Part(key) != null) return Value(key); + var row = GameOptionsSchema.Get(key); + if (row.Merge != null) return row.Merge.Of(row.Parts!.Select(p => Value(p.Key)).ToArray()); + if (key == GameOptionsSchema.ResolutionKey && GameOptionsSchema.FullScale(Value(GameOptionsSchema.UpscalerKey))) return GameOptionsSchema.NativeStep; + return Value(key); } /// A line's value as the file has it (the last line with that key, trimmed), null without one. Any key. @@ -157,20 +183,22 @@ public sealed class GameOptionsFile : IGameOptionValues /// sizes, the Anti-aliasing while an upscaler draws, the Microphone's device). public string Label(string key) => GameOptionsSchema.Get(key).Label(Get(key), this); - /// The values a row offers with the other rows as they are here: the Render resolution's steps for the Upscaler - /// chosen (none with DLAA or a native AA), the Microphone's saved device, the row's own Choices otherwise. The value it has - /// now is always in the list (a hand-edited one where it would sit). + /// The values a row offers with the other rows as they are here: the Upscaler's engines (the one chosen by its + /// saved value: an engine's native AA is its engine here), the Render resolution's steps for the Upscaler chosen (Native + /// (100%), NativeStep, among DLSS's and FSR's), the Microphone's saved device, the row's own Choices otherwise. The value it + /// has now is always in the list (a hand-edited one where it would sit). public IReadOnlyList ChoicesOf(string key) { var row = GameOptionsSchema.Get(key); if (row.Kind == GameOptionKind.Slider) return Array.Empty(); + var now = Get(key); var values = key switch { - GameOptionsSchema.ResolutionKey => GameOptionsSchema.ResolutionSteps(Get(GameOptionsSchema.UpscalerKey)).ToList(), + GameOptionsSchema.ResolutionKey => GameOptionsSchema.ResolutionSteps(Value(GameOptionsSchema.UpscalerKey)).ToList(), + GameOptionsSchema.UpscalerKey => GameOptionsSchema.UpscalerOrder.Select(e => e == GameOptionsSchema.Engine(now) ? now : e).ToList(), GameOptionsSchema.MicKey => Raw(GameOptionsSchema.MicDeviceKey) is { Length: > 0 } ? new List { 0, 1, 2 } : new List { 0, 1 }, _ => row.Choices.Select(c => c.Value).ToList(), }; - var now = Get(key); if (!values.Contains(now)) values.Add(now); if (key == GameOptionsSchema.GpuKey) { @@ -182,57 +210,93 @@ public sealed class GameOptionsFile : IGameOptionValues return values.Select(v => new GameOptionChoice(v, row.Label(v, this))).ToList(); } - /// The rows the launcher changed since the file was read, with their new values (Save writes these). + /// The saved values the launcher changed since the file was read, by their keys in the file (a merged row's are + /// its parts'), with their new values (Save writes these). public IReadOnlyDictionary Changes => set; - /// The file's lines that are no row here, in order, as (key, value): the KeptKeys and pad_/key_ lines, and + /// The file's lines that are no value here, in order, as (key, value): the KeptKeys and pad_/key_ lines, and /// any key nobody knows (kept as it is). Lines without a key (blank, a comment) are kept too but not listed. public IReadOnlyList<(string Key, string Value)> OtherLines => - lines.Select(l => (Key: KeyOf(l), Line: l)).Where(x => x.Key != null && GameOptionsSchema.ByKey(x.Key) == null) + lines.Select(l => (Key: KeyOf(l), Line: l)).Where(x => x.Key != null && !GameOptionsSchema.IsValueKey(x.Key)) .Select(x => (x.Key!, x.Line[(x.Key!.Length + 1)..])).ToList(); /// OtherLines' keys that neither the schema nor KeptKeys knows. public IReadOnlyList UnknownKeys => OtherLines.Select(o => o.Key).Where(k => !GameOptionsSchema.IsKept(k)).Distinct().ToList(); - /// Sets a row as the game's menu would (set_row): the value inside its range and on its steps (GameOptionRow.Clamp); - /// the Quality preset sets its rows (PresetValues, the Render resolution then on the Upscaler's nearest step); another - /// Upscaler moves the Render resolution to its nearest step; a row that is not PresetExempt, changed, turns the preset - /// back to Custom. Returns the value set. Nothing is written until Save. + /// Sets a row as the game's menu would (set_row): the value inside its range and on its steps (GameOptionRow.Clamp). + /// A merged row sets its parts (merged_rows.h *_set: one of a pair chosen, the other Off). The Upscaler takes an engine as + /// upscaler_rows.h step_engine does: the Render resolution's Native (100%) goes with it (the game's own at 100% becomes the + /// engine's native AA and back), the Render resolution then on the engine's nearest step; an engine's native AA value + /// (DLAA 2, FSR 4's 5, FSR 3.1's 6) is set as it is. The Render resolution's NativeStep makes the Upscaler its engine's + /// native AA (the game's own: 100%); another step with a native AA makes it the engine again (step_resolution). A merged + /// row's part by its own key: in its range. Returns the row's value afterwards. Nothing is written until Save. + /// (options_version 8: no Quality preset turns back to Custom.) public int Set(string key, int value) { - var row = GameOptionsSchema.Get(key); - var v = row.Clamp(value); - if (key == GameOptionsSchema.PresetKey) + if (GameOptionsSchema.Part(key) is { } part) { - Put(key, v); - foreach (var (k, pv) in GameOptionsSchema.PresetValues(v)) Put(k, pv); - if (v > 0) Put(GameOptionsSchema.ResolutionKey, GameOptionsSchema.SnapResolution(Get(GameOptionsSchema.UpscalerKey), Get(GameOptionsSchema.ResolutionKey))); - return v; + Put(key, part.AsRead(value)); + return Value(key); } - var changed = Get(key) != v; - Put(key, v); - if (key == GameOptionsSchema.UpscalerKey) - Put(GameOptionsSchema.ResolutionKey, GameOptionsSchema.SnapResolution(v, Get(GameOptionsSchema.ResolutionKey))); - if (changed && !row.PresetExempt) Put(GameOptionsSchema.PresetKey, 0); - return v; + var row = GameOptionsSchema.Get(key); + if (row.Merge != null) + { + var parts = row.Parts!; + var split = row.Merge.Split(row.Clamp(value), parts.Select(p => Value(p.Key)).ToArray()); + for (var i = 0; i < parts.Count; i++) Put(parts[i].Key, parts[i].AsRead(split[i])); + return Get(key); + } + const string Up = GameOptionsSchema.UpscalerKey, Res = GameOptionsSchema.ResolutionKey; + if (key == Up) + { + var v = row.Clamp(value); + var e = GameOptionsSchema.Engine(v); + if (!GameOptionsSchema.FullScale(v) && e != GameOptionsSchema.Engine(Value(Up)) && GameOptionsSchema.NativeSize(Value(Up), Value(Res))) + { + if (e == 0) Put(Res, 1); // (the game's own at 100%) + else v = GameOptionsSchema.NativeOf(e); + } + else if (!GameOptionsSchema.FullScale(v) && e == GameOptionsSchema.Engine(Value(Up))) v = Value(Up); // (its engine already: as it is) + Put(Up, v); + Put(Res, GameOptionsSchema.SnapResolution(v, Value(Res))); + return Get(key); + } + if (key == Res) + { + var up = Value(Up); + if (value == GameOptionsSchema.NativeStep) + { + if (GameOptionsSchema.Engine(up) == 0) Put(Res, 1); + else Put(Up, GameOptionsSchema.NativeOf(GameOptionsSchema.Engine(up))); + return Get(key); + } + if (GameOptionsSchema.FullScale(up)) Put(Up, GameOptionsSchema.Engine(up)); + Put(Res, row.Clamp(value)); + return Get(key); + } + Put(key, row.Clamp(value)); + return Get(key); } void Put(string key, int v) { - if (read[key] == v) set.Remove(key); // (what the game reads from the file already: nothing to write) + // (what the game reads from the file already: nothing to write. ((Triforce) 11-22) But for the GPU device row's card chosen + // again while the file has no name for it, or another one (a number saved before 11-20, in Windows' order): its number and + // name are written, as the game finds the card by the name first; nothing was written before and the number decided) + if (read[key] == v && !(key == GameOptionsSchema.GpuKey && GpuNameDiffers(v))) set.Remove(key); else set[key] = v; } - /// The benchmark's recommendations (key to value): the Quality preset first when it is one of them, then each - /// other row with Set's rules (so a row the preset set to the same value leaves it as it is; one that differs, and is not - /// PresetExempt, makes it Custom, as in the game). KeyNotFoundException, and nothing set, for a key that is no row. - /// Returns the rows' values afterwards, for the keys given. Nothing is written until Save. + bool GpuNameDiffers(int v) => + GameOptionsSchema.GpuNameOf(v) is { } name && (Raw(GameOptionsSchema.GpuNameKey) ?? "").Trim() != name; + + /// The benchmark's recommendations (key to value), each with Set's rules, in the order given. KeyNotFoundException, + /// and nothing set, for a key that is no row (nor a merged row's part). Returns the values afterwards, for the keys given. + /// Nothing is written until Save. public IReadOnlyDictionary Apply(IReadOnlyDictionary values) { - foreach (var k in values.Keys) GameOptionsSchema.Get(k); - if (values.TryGetValue(GameOptionsSchema.PresetKey, out var p)) Set(GameOptionsSchema.PresetKey, p); - foreach (var (k, v) in values) - if (k != GameOptionsSchema.PresetKey) Set(k, v); + foreach (var k in values.Keys) if (GameOptionsSchema.Part(k) == null) GameOptionsSchema.Get(k); + foreach (var (k, v) in values) Set(k, v); return values.Keys.ToDictionary(k => k, Get, StringComparer.Ordinal); } public IReadOnlyDictionary Apply(Dictionary values) => Apply((IReadOnlyDictionary)values); @@ -243,7 +307,7 @@ public sealed class GameOptionsFile : IGameOptionValues /// get its value, every other line stays exactly as the file has it (so a line changed outside since Load is kept, as /// the game's merge keeps it), a changed row the file has no line for is added before options_version. A file of an /// older options_version is brought up to the current one as the game would read it (the values its rules give are - /// written, options_version=7), so the game does not read the launcher's values by the old rules. The whole file is + /// written, options_version=8), so the game does not read the launcher's values by the old rules. The whole file is /// written beside it as .new and then put in its place: a reader never meets half a file, and a failure leaves the old /// one (IOException). Nothing to write: nothing written. The caller refuses while the game runs (Game.IsRunning). /// Afterwards this object holds the file as written. @@ -283,7 +347,7 @@ public sealed class GameOptionsFile : IGameOptionValues else outLines.Add(l); } var missing = write.Keys.Where(k => !done.Contains(k) && k != GameOptionsSchema.VersionKey) - .OrderBy(k => GameOptionsSchema.ByKey(k) is { } r ? IndexOf(r) : int.MaxValue).Select(k => k + "=" + write[k]).ToList(); + .OrderBy(k => GameOptionsSchema.RowOfLine(k) is { } r ? IndexOf(r) : int.MaxValue).Select(k => k + "=" + write[k]).ToList(); var at = outLines.FindIndex(l => KeyOf(l) == GameOptionsSchema.VersionKey); if (at < 0) at = outLines.Count; outLines.InsertRange(at, missing); diff --git a/launcher/Logic/GameOptions/GameOptionsSchema.cs b/launcher/Logic/GameOptions/GameOptionsSchema.cs index d00c9e2..2e1aa08 100644 --- a/launcher/Logic/GameOptions/GameOptionsSchema.cs +++ b/launcher/Logic/GameOptions/GameOptionsSchema.cs @@ -1,31 +1,36 @@ // The game's Options menu as data: gpu/vk/vk_menu.cpp's Row table (rows[]), the tx_* functions that label its values, the -// keys ini_key gives them, the defaults of vk_menu.h's Options, and the Quality preset (apply_preset, hn_preset). Every row -// the game keeps in triforce_options.ini is here except the ones KeptKeys names (the Debug tab, the Layout preset); the Game -// language is language.txt's, not this file's (Languages.cs). Labels are the menu's own words: players compare them. -// Where the menu's label depends on this PC (the display's refresh, a gyroscope, what the driver runs) the label here is the -// one the menu shows when it knows nothing of it. When vk_menu.cpp's table changes, change this one: the test -// "game options: schema matches vk_menu.cpp" reads the table and says what differs. +// keys ini_key gives them, the defaults of vk_menu.h's Options, and (options_version 8) the merged rows of merged_rows.h. Every +// row the game keeps in triforce_options.ini is here except the ones KeptKeys names (the Debug tab, the Layout preset, the +// lines options_version 8 dropped); the Game language is language.txt's, not this file's (Languages.cs). Labels are the menu's +// own words: players compare them. Where the menu's label depends on this PC (the display's refresh, a gyroscope, what the +// driver runs) the label here is the one the menu shows when it knows nothing of it. When vk_menu.cpp's table changes, change +// this one: the test "game options: schema matches vk_menu.cpp" reads the table and says what differs. using System.Globalization; namespace Triforce.Logic; +/// Whose low-latency library the Low latency row stands for (merged_rows.h Vendor): NVIDIA Reflex, AMD Anti-Lag 2, +/// or neither (the game hides the row there). +public enum LatencyVendor { None, Nvidia, Amd } + public static class GameOptionsSchema { /// The file the game reads at its start and writes at every change in its menu, in the game folder. public const string FileName = "triforce_options.ini"; - /// The options_version the game writes (vk_menu.cpp save): the file's format, which menu_init's rules read. - public const int CurrentVersion = 7; + /// The options_version the game writes (vk_menu.cpp save, merged_rows.h kVersion): the file's format, which + /// menu_init's rules read. + public const int CurrentVersion = 8; public const string OutputResKey = "output_res"; public const string ResolutionKey = "resolution"; public const string UpscalerKey = "dlss"; - public const string PresetKey = "quality_preset"; public const string RumbleKey = "rumble_strength"; public const string GpuKey = "gpu_device"; public const string MicKey = "microphone"; - public const string HandheldKey = "handheld_profile"; public const string VersionKey = "options_version"; public const string MicDeviceKey = "microphone_device"; + /// The merged rows' keys (the launcher's alone: no line has them, their Parts' keys are the file's). + public const string HdrKey = "@hdr", FrameGenKey = "@frame_generation", PacingKey = "@frame_pacing", LatencyKey = "@low_latency"; /// The names of the graphics cards, in the order the game counts them (its GPU device row's 1 is the first), /// for the GPU device row's labels. The launcher sets it (UseSystemGpus: Gpus.List()). The game lists the cards @@ -61,18 +66,42 @@ public static class GameOptionsSchema }; public const string GpuListFile = "triforce_gpus.txt"; + /// ((Triforce, options_version 8) Whose library the Low latency row is, as the game decides it by the card it runs + /// on (vk_menu.cpp latency_vendor: Reflex on an NVIDIA card, Anti-Lag 2 on an AMD one). Null: NVIDIA's, the row's widest + /// (Off, On, On + Boost). The launcher sets it (UseSystemLatency). + public static Func? LatencyVendorOf { get; set; } + /// LatencyVendorOf from the card that most likely runs the game (Gpus.Main(): an NVIDIA or AMD one before an + /// integrated Intel one). + public static void UseSystemLatency() => LatencyVendorOf = () => Gpus.Main()?.Vendor switch + { + GpuVendor.Nvidia => LatencyVendor.Nvidia, GpuVendor.Amd => LatencyVendor.Amd, _ => LatencyVendor.None, + }; + static LatencyVendor LatencyNow() + { + try { return LatencyVendorOf?.Invoke() ?? LatencyVendor.Nvidia; } catch (Exception) { return LatencyVendor.Nvidia; } + } + /// merged_rows.h latency_hi: the Low latency row's last value for this vendor (2 Reflex's On + Boost, 1 Anti-Lag 2's On). + public static int LatencyHi(LatencyVendor v) => v == LatencyVendor.Nvidia ? 2 : v == LatencyVendor.Amd ? 1 : 0; + // ---- the keys ---- - /// vk_menu.cpp ini_key: a few rows keep the keys they were first saved with; every other row's key is its - /// name lower-cased, its spaces as '_', cut to 31 characters. + /// vk_menu.cpp ini_key: a few rows keep the keys they were first saved with (the rows renamed since, options_version + /// 8's groups among them); every other row's key is its name lower-cased, its spaces as '_', cut to 31 characters. public static string IniKey(string name) => name switch { "Output resolution" => OutputResKey, "Render resolution" => ResolutionKey, - "Game frame rate cap" => "frame_rate_cap", - "Window scaling filter" => "upscaling", + "Frame rate cap" => "frame_rate_cap", "Upscaler" => UpscalerKey, "Start next frame early" => "tail_early", + "Aspect ratio" => "ultrawide", "HUD position" => "ultrawide_hud", "Menu sides" => "ultrawide_menus", + "Paper white" => "hdr_paper_white", "Peak brightness" => "hdr_peak_brightness", "UI brightness" => "renodx_ui_brightness", + "Tonemapper" => "renodx_tonemapper", "Gamma correction" => "renodx_gamma_correction", "Color space" => "renodx_color_space", + "Exposure" => "renodx_exposure", "Middle gray" => "renodx_midgray", "Match tone slope" => "renodx_match_tone_slope", + "Tone slope" => "renodx_tone_slope", "Hue clip" => "renodx_hue_clip", "Highlight saturation" => "renodx_highlight_saturation", + "Blowout" => "renodx_blowout", "Flare" => "renodx_flare", "Hue correction" => "renodx_hue_correction", + "Color grading" => "renodx_color_grading", "Highlights" => "renodx_highlights", "Shadows" => "renodx_shadows", + "Contrast" => "renodx_contrast", "Saturation" => "renodx_saturation", "Bloom" => "renodx_bloom", _ => Cut(new string(name.Select(c => c == ' ' ? '_' : c is >= 'A' and <= 'Z' ? (char)(c + 32) : c).ToArray()), 31), }; static string Cut(string s, int n) => s.Length > n ? s[..n] : s; @@ -86,7 +115,9 @@ public static class GameOptionsSchema static string TxOnoff(int v, IGameOptionValues? o) => v != 0 ? "On" : "Off"; static string TxMenuShortcut(int v, IGameOptionValues? o) => v != 0 ? "F1 only" : "Insert or F1"; - static string TxHdr(int v, IGameOptionValues? o) => v == 1 ? "Expanded" : v == 2 ? "Native" : "Off"; + static string TxHdr(int v) => v == 1 ? "Expanded" : v == 2 ? "Native" : "Off"; + /// tx_hdr_mode: the HDR row, opts.hdr with RenoDX HDR as its fourth value. + static string TxHdrMode(int v, IGameOptionValues? o) => v == HdrRenodx ? "Native + RenoDX" : TxHdr(Math.Clamp(v, 0, 2)); static string TxNits(int v, IGameOptionValues? o) => v.ToString(CultureInfo.InvariantCulture) + " nits"; /// tx_fps_cap. 6, Monitor's rate, shows the display's rate in the game ("Monitor's rate (144)") once it knows it. static string TxFpsCap(int v, IGameOptionValues? o) @@ -95,22 +126,22 @@ public static class GameOptionsSchema if (v >= 6) return "Monitor's rate"; return new[] { "60 (the console's)", "90", "120", "144", "165", "240" }[v]; } - static string TxSteadyFps(int v, IGameOptionValues? o) => v == 1 ? "Any rate" : v == 2 ? "Refresh steps" : "Off"; + static string TxSteadyFps(int v) => v == 1 ? "Any rate" : v == 2 ? "Refresh steps" : "Off"; + /// tx_pacing: Frame pacing and Steady frame rate in one row. + static string TxPacing(int v, IGameOptionValues? o) => Pick(new[] { "Off", "Even", "Steady, any rate", "Steady, refresh steps" }, v); /// tx_gyro without a controller's answer (the game adds "(gyro found)" and the like). static string TxGyro(int v, IGameOptionValues? o) => v != 0 ? "On" : "Off"; static string TxMousecam(int v, IGameOptionValues? o) => v != 0 ? "Native" : "As the stick"; static string TxSens(int v, IGameOptionValues? o) => (v < 0 ? "-" : "") + (Math.Abs(v) / 100).ToString(CultureInfo.InvariantCulture) + "." + (Math.Abs(v) % 100).ToString("00", CultureInfo.InvariantCulture); - static readonly string[] Upscales = { "Off", "Catmull-Rom", "Lanczos", "AMD FSR 1" }; - /// tx_filter as tx_upscale: the game shows "Not needed (window = output)" while its window is the output's size. - static string TxFilter(int v, IGameOptionValues? o) => OrFirst(Upscales, v); - /// tx_reflex as on an NVIDIA card with Reflex (elsewhere the game shows Off, and On with DLSS frame generation). - static string TxReflex(int v, IGameOptionValues? o) => Pick(new[] { "Off", "On", "On + Boost" }, v); - /// tx_antilag as on an AMD card with Anti-Lag 2 (elsewhere the game shows Off). - static string TxAntilag(int v, IGameOptionValues? o) => v != 0 ? "On" : "Off"; - static readonly string[] Upscalers = { "Game", "NVIDIA DLSS", "NVIDIA DLAA", "AMD FSR 3.1", "AMD FSR 4", "AMD FSR 4 Native AA", "AMD FSR 3.1 Native AA" }; - /// tx_upscaler by the value chosen (the game shows what runs: the game's own where the choice cannot). - static string TxUpscaler(int v, IGameOptionValues? o) => Pick(Upscalers, v); + static readonly string[] LatencyNames = { "Off", "On", "On + Boost" }; + /// tx_latency: the Low latency row by the card's maker (with DLSS frame generation the game shows Reflex's Off as + /// On: Streamline's Reflex is always on). + static string TxLatency(int v, IGameOptionValues? o) => LatencyNames[Math.Clamp(v, 0, LatencyHi(LatencyNow()))]; + static readonly string[] Engines = { "Game", "NVIDIA DLSS", "NVIDIA DLSS", "AMD FSR 3.1", "AMD FSR 4", "AMD FSR 4", "AMD FSR 3.1" }; + /// tx_upscaler: the engine alone (options_version 8: DLAA and the native AAs are the Render resolution's Native + /// (100%)); the game names FSR 4 by the version AMD's library runs once it has asked ("AMD FSR 4.1"). + static string TxUpscaler(int v, IGameOptionValues? o) => Engines[Engine(v)]; /// tx_dlss_model without RX_VK_DLSS_PRESET. static string TxDlssModel(int v, IGameOptionValues? o) => Pick(new[] { "Transformer (K)", "Transformer 2 (M)", "Transformer 2 (L)", "NVIDIA default" }, v); static string TxAniso(int v, IGameOptionValues? o) => OrFirst(new[] { "Game", "2x", "4x", "8x", "16x" }, v); @@ -134,19 +165,20 @@ public static class GameOptionsSchema static string TxMenuSounds(int v, IGameOptionValues? o) => v != 0 ? Pct(v) : "Off"; static string TxSprint(int v, IGameOptionValues? o) => v == 1 ? "Left stick click" : v == 2 ? "B" : "Off"; static string TxSprintMode(int v, IGameOptionValues? o) => v != 0 ? "Hold" : "Toggle"; - static string TxStickDeadZone(int v, IGameOptionValues? o) => OrFirst(new[] { "Default (24%)", "5%", "10%", "15%", "20%", "30%" }, v); - static string TxOwnDeadZone(int v, IGameOptionValues? o) => v < 0 ? "As Stick dead zone" : Pct(v); + /// tx_own_dead_zone: -1 is XInput's own (~24%), the Default (options_version 8: the shared Stick dead zone is gone). + static string TxOwnDeadZone(int v, IGameOptionValues? o) => v < 0 ? "Default (24%)" : Pct(v); static string TxWalkKey(int v, IGameOptionValues? o) => OrFirst(new[] { "Off", "Caps Lock", "Shift", "Ctrl", "Tab" }, v); static string TxAudioOutput(int v, IGameOptionValues? o) => OrFirst(new[] { "Auto", "Stereo", "5.1" }, v); static string TxDeadzone(int v, IGameOptionValues? o) => Pick(new[] { "Low", "Standard", "High" }, v); - static string TxRobust(int v, IGameOptionValues? o) => OrFirst(new[] { "Auto", "Off", "On" }, v); static string TxMesh(int v, IGameOptionValues? o) => v == 1 ? "Compute fallback" : "Auto"; static string TxPerf(int v, IGameOptionValues? o) => v == 1 ? "On" : v >= 2 ? "Full" : "Off"; - static string TxDlssg(int v, IGameOptionValues? o) => v == 1 ? "2x" : v == 2 ? "3x" : v == 3 ? "4x" : "Off"; - /// tx_framegen where AMD's library can make the frames (elsewhere the game shows a saved AMD FSR 4 as 2x). - static string TxFramegen(int v, IGameOptionValues? o) => Pick(new[] { "Off", "Auto", "2x", "3x", "4x", "5x", "6x", "7x", "8x", "AMD FSR 4" }, v); - static readonly string[] Presets = { "Custom", "Minimum (6GB)", "Low (8GB)", "Balanced (12GB)", "High (16GB)", "Ultra (20GB+)" }; - static string TxPreset(int v, IGameOptionValues? o) => Pick(Presets, v); + static string TxDlssg(int v) => v == 1 ? "2x" : v == 2 ? "3x" : v == 3 ? "4x" : "Off"; + /// tx_framegen where AMD's library can make the frames, before it has said its version (the game then shows + /// "AMD FSR (4.0.2)" and the like; elsewhere a saved 9 as 2x). + static string TxFramegen(int v) => Pick(new[] { "Off", "Auto", "2x", "3x", "4x", "5x", "6x", "7x", "8x", "AMD FSR" }, v); + /// tx_framegen_row: the renderer's own and AMD's (opts.framegen), then NVIDIA DLSS 2x to 4x (opts.dlssg). + static string TxFramegenRow(int v, IGameOptionValues? o) => + v >= FgDlss ? new[] { "NVIDIA DLSS 2x", "NVIDIA DLSS 3x", "NVIDIA DLSS 4x" }[Math.Clamp(v - FgDlss, 0, 2)] : TxFramegen(v); /// tx_gpu: 0 "Auto", then "<n>: <name>" counted from 0 (GpuNames), "<n>: Device <n>" without a name. static string TxGpu(int v, IGameOptionValues? o) { @@ -183,30 +215,45 @@ public static class GameOptionsSchema // ---- the Output resolution and the Render resolution ---- - /// The Output resolution's sizes by their saved value (kOutputs); 6 came after the others so that saved values - /// keep their sizes. - public static readonly IReadOnlyList<(int W, int H)> Outputs = new[] { (2048, 1152), (2560, 1440), (3840, 2160), (1920, 1080), (5120, 1440), (5120, 2880), (5120, 2160) }; + /// The Output resolution's sizes by their saved value (kOutputs); 6 and 7 came after the others so that saved + /// values keep their sizes. (options_version 8) 7 is 1536x864, the game's handheld profile, once a row of its own. + public static readonly IReadOnlyList<(int W, int H)> Outputs = new[] { (2048, 1152), (2560, 1440), (3840, 2160), (1920, 1080), (5120, 1440), (5120, 2880), (5120, 2160), (1536, 864) }; + /// The Output resolution's handheld profile (merged_rows.h kHandheldOutput). + public const int HandheldOutput = 7; /// The order the menu shows them in, smallest to largest (kOutputOrder). - public static readonly IReadOnlyList OutputOrder = new[] { 3, 0, 1, 2, 4, 6, 5 }; + public static readonly IReadOnlyList OutputOrder = new[] { 7, 3, 0, 1, 2, 4, 6, 5 }; static int OutputChoice(int v) => v >= 0 && v < Outputs.Count ? v : 0; static string TxOutputRes(int v, IGameOptionValues? o) { - var (w, h) = Outputs[OutputChoice(v)]; + v = OutputChoice(v); + if (v == HandheldOutput) return "1536x864 (handheld)"; + var (w, h) = Outputs[v]; return w + "x" + h; } // (upscaler_rows.h) const int Game = 0, Dlss = 1, Dlaa = 2, Fsr3 = 3, Fsr4 = 4, Fsr4Aa = 5, Fsr3Aa = 6; - /// The Upscaler row's order in the menu: each engine's native anti-aliasing after it (kOrder). - public static readonly IReadOnlyList UpscalerOrder = new[] { Game, Dlss, Dlaa, Fsr3, Fsr3Aa, Fsr4, Fsr4Aa }; + /// The engines in the order the Upscaler row shows them (kEngines): an engine's native anti-aliasing (DLAA, FSR's + /// Native AA) is the Render resolution's Native (100%) from options_version 8, its saved value as before. + public static readonly IReadOnlyList UpscalerOrder = new[] { Game, Dlss, Fsr3, Fsr4 }; + /// upscaler_rows.h engine: DLAA is DLSS's, FSR 4's and FSR 3.1's native AA their own. + public static int Engine(int up) => up == Dlaa ? Dlss : up == Fsr4Aa ? Fsr4 : up == Fsr3Aa ? Fsr3 : up is >= Game and <= Fsr3Aa ? up : Game; + /// upscaler_rows.h native_of: an engine's native anti-aliasing (the game's own has none: itself). + public static int NativeOf(int engine) => engine == Dlss ? Dlaa : engine == Fsr4 ? Fsr4Aa : engine == Fsr3 ? Fsr3Aa : engine; + /// The Render resolution row's Native (100%) with DLSS or FSR (upscaler_rows.h kNativeStep): its value here while + /// the Upscaler is an engine's native AA (GameOptionsFile.Get). Chosen, the Upscaler becomes that native AA and the + /// saved resolution stays as it is. + public const int NativeStep = -1; static readonly int[] Percents = { 0, 100, 75, 67, 50, 58, 33 }; static int Percent(int v) => v > 0 && v < Percents.Length ? Percents[v] : 0; - static bool FullScale(int up) => up == Dlaa || up == Fsr4Aa || up == Fsr3Aa; + public static bool FullScale(int up) => up == Dlaa || up == Fsr4Aa || up == Fsr3Aa; static bool ScalesByMode(int up) => up == Dlss || up == Fsr3 || up == Fsr4; - static readonly int[] GameSteps = { 0, 1, 2, 3, 4 }, DlssSteps = { 0, 3, 5, 4, 6 }; - /// The Render resolution values the menu steps through with this Upscaler (none with DLAA or a native AA: - /// the 3D is drawn at full size). - public static IReadOnlyList ResolutionSteps(int upscaler) => FullScale(upscaler) ? Array.Empty() : ScalesByMode(upscaler) ? DlssSteps : GameSteps; + /// upscaler_rows.h native_size: the 3D at the output's full size (an engine's native AA, or the game's own at 100%). + public static bool NativeSize(int up, int resolution) => FullScale(up) || (up == Game && Percent(resolution) == 100); + static readonly int[] GameSteps = { 0, 1, 2, 3, 4 }, DlssSteps = { 0, 3, 5, 4, 6 }, TemporalSteps = { 0, NativeStep, 3, 5, 4, 6 }; + /// The Render resolution values the menu steps through with this Upscaler: the game's own its shares; DLSS and + /// FSR (and their native AAs) Auto, Native (100%) and their modes (upscaler_rows.h kTemporalSteps). + public static IReadOnlyList ResolutionSteps(int upscaler) => ScalesByMode(Engine(upscaler)) ? TemporalSteps : GameSteps; /// upscaler_rows.h snap: v as one of the upscaler's steps, the nearest in size (the larger on a tie); Auto stays, /// and with a full-size upscaler v is kept for the next. public static int SnapResolution(int upscaler, int v) @@ -225,17 +272,15 @@ public static class GameOptionsSchema } static string ModeName(int v) => Percent(v) switch { 67 => "Quality", 58 => "Balanced", 50 => "Performance", 33 => "Ultra Performance", _ => "" }; /// tx_resolution: by the Upscaler chosen, and the size the game draws the 3D at for a fixed share of the - /// Output resolution (the handheld profile's 1536x864), a twelfth of it as the game takes it. + /// Output resolution (the handheld profile's 1536x864 among them), a twelfth of it as the game takes it. static string TxResolution(int v, IGameOptionValues? o) { var up = V(o, UpscalerKey, 0); - if (up == Dlaa) return "Native (DLAA)"; - if (up == Fsr4Aa) return "Native (FSR 4)"; - if (up == Fsr3Aa) return "Native (FSR 3.1)"; + if (FullScale(up) || v == NativeStep) return "Native (100%)"; var p = Percent(v); if (p == 0) return "Auto (dynamic)"; if (ScalesByMode(up) && ModeName(v).Length > 0) return ModeName(v) + " (" + p + "%)"; - var (ow, oh) = V(o, HandheldKey, 0) != 0 ? (1536, 864) : Outputs[OutputChoice(V(o, OutputResKey, 0))]; + var (ow, oh) = Outputs[OutputChoice(V(o, OutputResKey, 0))]; var k = Math.Min(Math.Max((int)(p / 100.0f * 12.0f + 0.5f), 1), 12); return p + "% (" + (ow * k + 11) / 12 + "x" + oh * k / 12 + ")"; } @@ -249,6 +294,38 @@ public static class GameOptionsSchema return OrFirst(AaNames, v); } + // ---- the merged rows (merged_rows.h) ---- + + const int HdrRenodx = 3, FgDlss = 10; + /// The saved values the merged rows stand for (vk_menu.cpp's TAB_HIDDEN rows), by key: each keeps its line. + public static readonly IReadOnlyDictionary Parts = new GameOptionPart[] + { + new("hdr", 0, 2, 0, TxHdr), new("renodx_hdr", 0, 1, 0, v => v != 0 ? "On" : "Off"), + new("frame_generation", 0, 9, 0, TxFramegen), new("dlss_frame_generation", 0, 3, 0, TxDlssg), + new("frame_pacing", 0, 1, 1, v => v != 0 ? "On" : "Off"), new("steady_frame_rate", 0, 2, 0, TxSteadyFps), + new("nvidia_reflex", 0, 2, 2, v => Pick(LatencyNames, v)), new("amd_anti-lag_2", 0, 1, 0, v => v != 0 ? "On" : "Off"), + }.ToDictionary(p => p.Key, StringComparer.Ordinal); + /// A saved value a merged row stands for, by its key; null for any other key. + public static GameOptionPart? Part(string key) => Parts.GetValueOrDefault(key); + + static (string[] Parts, GameOptionMerge Merge) MergeHdr => (new[] { "hdr", "renodx_hdr" }, new GameOptionMerge( + p => p[0] == 2 && p[1] != 0 ? HdrRenodx : Math.Clamp(p[0], 0, 2), + (row, _) => new[] { row >= HdrRenodx ? 2 : Math.Max(row, 0), row >= HdrRenodx ? 1 : 0 })); + static (string[] Parts, GameOptionMerge Merge) MergeFramegen => (new[] { "frame_generation", "dlss_frame_generation" }, new GameOptionMerge( + p => p[1] is >= 1 and <= 3 ? FgDlss + p[1] - 1 : p[0], + (row, _) => row >= FgDlss ? new[] { 0, Math.Min(row - FgDlss + 1, 3) } : new[] { Math.Max(row, 0), 0 })); + static (string[] Parts, GameOptionMerge Merge) MergePacing => (new[] { "frame_pacing", "steady_frame_rate" }, new GameOptionMerge( + p => p[1] == 1 ? 2 : p[1] == 2 ? 3 : p[0] != 0 ? 1 : 0, + (row, _) => new[] { row != 0 ? 1 : 0, row == 2 ? 1 : row == 3 ? 2 : 0 })); + static (string[] Parts, GameOptionMerge Merge) MergeLatency => (new[] { "nvidia_reflex", "amd_anti-lag_2" }, new GameOptionMerge( + p => LatencyNow() switch { LatencyVendor.Nvidia => Math.Clamp(p[0], 0, 2), LatencyVendor.Amd => p[1] != 0 ? 1 : 0, _ => 0 }, + (row, now) => LatencyNow() switch + { + LatencyVendor.Nvidia => new[] { Math.Clamp(row, 0, 2), now[1] }, + LatencyVendor.Amd => new[] { now[0], row > 0 ? 1 : 0 }, + _ => now.ToArray(), + })); + // ---- the rows ---- static GameOptionRow Row(GameOptionTab tab, string? section, string name, int lo, int hi, int step, int dflt, GameOptionKind kind, @@ -282,184 +359,182 @@ public static class GameOptionsSchema }; } - /// Every row, in the menu's order (Display, Graphics, System, Audio, Controls as the table has them; the - /// Motion page's rows and Button prompts after the Controls tab's own). + /// A merged row (merged_rows.h): a Choice row with no line of its own, a view of its parts. Low latency's choices + /// follow the card's maker (LatencyVendorOf). + static GameOptionRow Merged(GameOptionTab tab, string section, string name, int lo, int hi, int step, + (string[] Parts, GameOptionMerge Merge) merge, Func format, bool restart, bool exempt, string help) + { + var key = name switch { "HDR" => HdrKey, "Frame generation" => FrameGenKey, "Frame pacing" => PacingKey, _ => LatencyKey }; + Func> list = key == LatencyKey + ? () => Enumerable.Range(0, LatencyHi(LatencyNow()) + 1).Select(v => new GameOptionChoice(v, format(v, null))).ToList() + : () => Enumerable.Range(lo, hi - lo + 1).Select(v => new GameOptionChoice(v, format(v, null))).ToList(); + return new GameOptionRow + { + Tab = tab, Section = section, Name = name, Key = key, Lo = lo, Hi = hi, Step = step, Kind = GameOptionKind.Choice, + Format = format, ChoiceList = list, Restart = restart, PresetExempt = exempt, Help = help, + Parts = merge.Parts.Select(k => Parts[k]).ToList(), Merge = merge.Merge, + }; + } + + /// Every row, in the menu's order (as vk_menu.cpp's table has them: Display, Graphics, System's Game, Audio, + /// System's Online, Controls; the Motion page's rows and Button prompts after the Controls tab's own). public static readonly IReadOnlyList All = new[] { Row(GameOptionTab.Display, "Screen", "Fullscreen", 0, 1, 1, 0, GameOptionKind.Choice, TxOnoff, false, true, - "Fills the whole screen with the game, without the window's border. The same as F10 or Alt+Enter."), - Row(GameOptionTab.Display, "Screen", "Output resolution", 0, 6, 1, 0, GameOptionKind.Choice, TxOutputRes, true, false, - "The game's whole frame, HUD included, before it is scaled to the window. Docked only. Applies after a restart."), + "The game fills the whole screen, without the window's border. F10 and Alt+Enter do the same."), + Row(GameOptionTab.Display, "Screen", "Output resolution", 0, 7, 1, 0, GameOptionKind.Choice, TxOutputRes, true, false, + "The size of the game's whole frame, HUD included, before it is scaled to the window. Applies after a restart."), Row(GameOptionTab.Display, "Screen", "GPU device", 0, 8, 1, 0, GameOptionKind.Choice, TxGpu, true, true, "Which graphics card the game runs on. Applies after a restart."), - Row(GameOptionTab.Display, "Widescreen", "Ultrawide", 0, 3, 1, 0, GameOptionKind.Choice, TxUltrawide, true, false, - "The camera's shape: the game's own 16:9, or a fixed 16:10, 21:9 or 32:9 for a screen or a window of that shape. A screen of another shape shows the picture between black bars. Applies after a restart."), - Row(GameOptionTab.Display, "Widescreen", "Ultrawide HUD", 0, 1, 1, 0, GameOptionKind.Choice, TxWideHud, true, true, - "With Ultrawide: where the hearts, the item slot and the map sit on a screen wider than 16:9. Middle: in the middle 16:9, as on a normal screen. Screen edges: out in the screen's corners. Applies after a restart."), - Row(GameOptionTab.Display, "Widescreen", "Ultrawide menus", 0, 2, 1, 1, GameOptionKind.Choice, TxWideMenus, false, false, - "With Ultrawide: the pause, item and map menus, and what shows beside them. Loading screens and pre-rendered movies, which draw no world either, get the same sides; a fade always covers the whole screen. Changes at once."), - Row(GameOptionTab.Display, "Frame rate", "Game frame rate cap", -3, 6, 1, 0, GameOptionKind.Choice, TxFpsCap, true, true, - "Limits frames rendered by the game. Generated frames are additional. Takes a restart. 30, 40 and 45 are for a PC or handheld that holds one of them steadily where 60 swings; the game keeps its speed. Past 60 is experimental: the game keeps real time, but some cutscenes and bosses can misbehave (Gohma, the intro's narration). 60 is the safe choice. Monitor's rate is the refresh rate of the screen the game starts on, from 30 to 240."), - Row(GameOptionTab.Display, "Frame rate", "Steady frame rate", 0, 2, 1, 0, GameOptionKind.Choice, TxSteadyFps, true, true, - "Paces the game to a frame rate this PC holds where you are, under the Game frame rate cap, and shows each frame a period after the last, so that frames come evenly instead of each as soon as it is ready. It goes down quickly where frames run late and back up slowly where there is room; loads and single hitches do not move it. Gives up some frame rate for steadiness. Any rate suits a screen with variable refresh (G-SYNC, FreeSync) or VSync off; Refresh steps keeps to whole divisions of the refresh rate (at 240 Hz: 120, 80, 60 ...) for VSync without variable refresh. Takes a restart."), - Row(GameOptionTab.Display, "Frame rate", "Reflex frame limit", 0, 1, 1, 0, GameOptionKind.Choice, TxOnoff, false, true, - "With DLSS frame generation: NVIDIA Reflex holds the Game frame rate cap where each frame starts, in place of the game's own wait, so a frame waits once and the controls are read later. The game's own wait is back whenever no frame is being generated. Where RivaTuner Statistics Server is the game's limiter it holds the frames instead. Changes at once."), - Row(GameOptionTab.Display, "Frame rate", "VSync", 0, 1, 1, 1, GameOptionKind.Choice, TxOnoff, false, true, - "Whether a finished frame waits for the screen's next refresh. Off shows it at once: a little less delay, and the picture can tear. Neither changes the frame rate, which the Game frame rate cap limits. Changes at once."), - Row(GameOptionTab.Display, "Frame rate", "FPS counter", 0, 2, 1, 0, GameOptionKind.Choice, TxPerf, false, true, - "On: frames per second in the top right corner. Full: a performance overlay in its place, with the GPU's and the CPU's load, temperatures and memory, the frame rate and its 1% and 0.1% lows, the frame times and their graph, the waits per frame and the picture's settings."), - Row(GameOptionTab.Display, "High dynamic range", "HDR", 0, 2, 1, 0, GameOptionKind.Choice, TxHdr, false, true, - "HDR10 out on an HDR display (Windows: Use HDR on in the display settings). Expanded: the game's SDR picture, its white at the paper white and its highlights up to the peak. Native: the game's own HDR, told the display's peak and paper white."), - Row(GameOptionTab.Display, "High dynamic range", "HDR paper white", 80, 400, 10, 200, GameOptionKind.Slider, TxNits, false, true, - "How bright the picture's white is with HDR on. Around 200 for a normal room; higher in a bright one."), - Row(GameOptionTab.Display, "High dynamic range", "HDR peak brightness", 400, 10000, 100, 1000, GameOptionKind.Slider, TxNits, false, true, - "The brightest highlights with HDR on: set it to your display's peak brightness."), - Row(GameOptionTab.Display, "RenoDX", "RenoDX HDR", 0, 1, 1, 0, GameOptionKind.Choice, TxOnoff, true, true, - "With HDR set to Native: RenoDX's tone mapping, eye adaptation and color grading in place of the game's own, and its rows below. Off: the game's own HDR. Applies after a restart. RenoDX is by Carlos Lopez (clshortfuse), under the MIT license."), - Row(GameOptionTab.Display, "RenoDX", "RenoDX tonemapper", 0, 2, 1, 1, GameOptionKind.Choice, TxRdxTonemap, false, true, - "With HDR Native: how the scene's light is brought to the display. Vanilla: the game's own curve. PsychoV35: RenoDX's, which keeps hue and saturation into the highlights, up to the peak brightness. RenoDRT: RenoDX's, which carries the game's SDR look up to the peak by brightness. RenoDX is by Carlos Lopez (clshortfuse), under the MIT license."), - Row(GameOptionTab.Display, "RenoDX", "RenoDX gamma correction", 0, 2, 1, 0, GameOptionKind.Choice, TxRdxGamma, false, true, - "Shows the picture as a 2.2 or BT.1886 (2.4) display would, as the game was made to be seen: darker shadows. Off keeps the sRGB curve. With every tonemapper. PsychoV35 and RenoDRT still end at the peak brightness; Vanilla's highlights are the game's own, which 2.2 dims a little and 2.4 brightens a little."), - Row(GameOptionTab.Display, "RenoDX", "RenoDX midgray", 0, 2, 1, 1, GameOptionKind.Choice, TxRdxMidgray, false, true, - "With PsychoV35: what the scene's middle grey is held to. Render: as rendered. Vanilla: where the game's own curve puts it. Eye adaptation: it follows what is on screen over a few seconds, weighted towards the middle of the view, as eyes adjust."), - Row(GameOptionTab.Display, "RenoDX", "RenoDX match tone slope", 0, 1, 1, 1, GameOptionKind.Choice, TxOnoff, false, true, - "With PsychoV35: gives the mid tones the contrast of the game's own curve. Off: PsychoV35's own, softer."), - Row(GameOptionTab.Display, "RenoDX", "RenoDX tone slope", 0, 100, 1, 50, GameOptionKind.Slider, TxRdxGrade, false, true, - "With PsychoV35: the steepness of the tone curve through middle grey. 50 leaves it unchanged."), - Row(GameOptionTab.Display, "RenoDX", "RenoDX hue clip", 0, 100, 5, 50, GameOptionKind.Slider, TxPercent, false, true, - "With PsychoV35: how much bright colors turn as they do where the game's SDR picture clips: fire towards yellow and white. 0% keeps their hue."), - Row(GameOptionTab.Display, "RenoDX", "RenoDX UI brightness", 80, 400, 10, 200, GameOptionKind.Slider, TxNits, false, true, - "How bright the menus' and the HUD's white is with RenoDX's tonemapper. The scene's follows HDR paper white."), - Row(GameOptionTab.Display, "RenoDX", "RenoDX color space", 0, 2, 1, 1, GameOptionKind.Choice, TxRdxSpace, false, true, - "The colors the picture is taken to be in before HDR10's. Vanilla: the game's own matrix. BT.709: as the SDR picture. DCI-P3: more saturated."), - Row(GameOptionTab.Display, "RenoDX", "RenoDX exposure", 0, 200, 5, 100, GameOptionKind.Slider, TxPercent, false, true, - "The scene's brightness before RenoDX's tone mapping. 100% is the game's."), - Row(GameOptionTab.Display, "RenoDX", "RenoDX highlights", 0, 100, 5, 50, GameOptionKind.Slider, TxRdxGrade, false, true, - "RenoDX's color grading: the bright parts of the scene. 50 is neutral."), - Row(GameOptionTab.Display, "RenoDX", "RenoDX shadows", 0, 100, 5, 50, GameOptionKind.Slider, TxRdxGrade, false, true, - "RenoDX's color grading: the dark parts of the scene. 50 is neutral."), - Row(GameOptionTab.Display, "RenoDX", "RenoDX contrast", 0, 100, 5, 50, GameOptionKind.Slider, TxRdxGrade, false, true, - "RenoDX's color grading: the scene's contrast. 50 is neutral."), - Row(GameOptionTab.Display, "RenoDX", "RenoDX saturation", 0, 100, 5, 50, GameOptionKind.Slider, TxRdxGrade, false, true, - "RenoDX's color grading: the scene's saturation. 50 is neutral."), - Row(GameOptionTab.Display, "RenoDX", "RenoDX highlight saturation", 0, 100, 1, 50, GameOptionKind.Slider, TxRdxGrade, false, true, - "With RenoDRT: the strength of the colors of the brightest parts. 50 leaves it unchanged."), - Row(GameOptionTab.Display, "RenoDX", "RenoDX blowout", 0, 100, 1, 0, GameOptionKind.Slider, TxRdxGrade, false, true, - "With RenoDRT: turns the brightest parts towards white. 0 keeps their color."), - Row(GameOptionTab.Display, "RenoDX", "RenoDX flare", 0, 100, 1, 0, GameOptionKind.Slider, TxRdxGrade, false, true, - "With RenoDRT: lifts the darkest parts as a little stray light would. 0 keeps black black."), - Row(GameOptionTab.Display, "RenoDX", "RenoDX hue correction", 0, 100, 5, 100, GameOptionKind.Slider, TxPercent, false, true, - "With RenoDRT: how much bright colors keep the hue they have before the tonemapper. 0% lets them shift as the curve takes them."), - Row(GameOptionTab.Display, "RenoDX", "RenoDX color grading", 0, 100, 5, 100, GameOptionKind.Slider, TxPercent, false, true, - "With RenoDRT: how much of the game's own color grading the HDR picture keeps. 0% is the scene without it."), - Row(GameOptionTab.Display, "RenoDX", "RenoDX bloom", 0, 100, 5, 50, GameOptionKind.Slider, TxRdxGrade, false, true, - "How strong the glow around bright things is. 50 is the game's."), - Row(GameOptionTab.Graphics, "Quality", "Quality preset", 0, 5, 1, 0, GameOptionKind.Choice, TxPreset, false, false, - "Sets the scaling rows and memory limits for a GPU with that much memory. The Upscaler and Frame generation stay yours."), - Row(GameOptionTab.Graphics, "Quality", "Render resolution", 0, 6, 1, 0, GameOptionKind.Choice, TxResolution, true, false, - "The share of the output the 3D is drawn at; the Upscaler fills in the rest. Applies after a restart."), - Row(GameOptionTab.Graphics, "Quality", "Texture detail", 0, 3, 1, 0, GameOptionKind.Choice, TxTexDetail, true, false, - "How large the game's textures are kept in video memory. Automatic suits the graphics card; a Quality preset sets it for its card's size. Applies after a restart."), - Row(GameOptionTab.Graphics, "Quality", "Texture filtering", 0, 4, 1, 0, GameOptionKind.Choice, TxAniso, false, false, - "Anisotropic filtering: keeps ground, paths and walls sharp where they slope away from the camera. Game's is what the game asks for; 16x is the sharpest, at a small cost."), - Row(GameOptionTab.Graphics, "Upscaling and edges", "Upscaler", 0, 6, 1, 0, GameOptionKind.Choice, TxUpscaler, false, false, - "What scales the 3D up to the Output resolution."), - Row(GameOptionTab.Graphics, "Upscaling and edges", "DLSS model", 0, 3, 1, 0, GameOptionKind.Choice, TxDlssModel, false, true, + Row(GameOptionTab.Display, "Ultrawide", "Aspect ratio", 0, 3, 1, 0, GameOptionKind.Choice, TxUltrawide, true, false, + "The shape of the picture: the game's own 16:9, or 16:10, 21:9 or 32:9 for a screen of that shape. A screen of another shape shows it between black bars. Applies after a restart."), + Row(GameOptionTab.Display, "Ultrawide", "HUD position", 0, 1, 1, 0, GameOptionKind.Choice, TxWideHud, true, true, + "Where the hearts, the item buttons and the map sit. Middle: in the middle 16:9, as on a normal screen. Screen edges: in the screen's corners. Applies after a restart."), + Row(GameOptionTab.Display, "Ultrawide", "Menu sides", 0, 2, 1, 1, GameOptionKind.Choice, TxWideMenus, false, false, + "What is beside the pause, item and map menus, loading screens and movies, which are 16:9. Changes at once."), + Merged(GameOptionTab.Display, "HDR", "HDR", 0, 3, 1, MergeHdr, TxHdrMode, true, true, + "HDR10 on an HDR display (Windows: Use HDR on in the display settings). Native + RenoDX applies after a restart."), + Row(GameOptionTab.Display, "HDR", "Paper white", 80, 400, 10, 200, GameOptionKind.Slider, TxNits, false, true, + "How bright white is: around 200 nits for a normal room, higher for a bright one."), + Row(GameOptionTab.Display, "HDR", "Peak brightness", 400, 10000, 100, 1000, GameOptionKind.Slider, TxNits, false, true, + "The brightest highlights: set it to your display's peak brightness."), + Row(GameOptionTab.Display, "HDR", "UI brightness", 80, 400, 10, 200, GameOptionKind.Slider, TxNits, false, true, + "How bright the HUD's and the menus' white is with RenoDX. The scene's white is Paper white."), + Row(GameOptionTab.Display, "RenoDX", "Tonemapper", 0, 2, 1, 1, GameOptionKind.Choice, TxRdxTonemap, false, true, + "How the scene's light is brought to the display. Vanilla: the game's own curve. PsychoV35: keeps hue and saturation into the highlights. RenoDRT: carries the game's SDR look up to the peak brightness."), + Row(GameOptionTab.Display, "RenoDX", "Gamma correction", 0, 2, 1, 0, GameOptionKind.Choice, TxRdxGamma, false, true, + "Shows the picture as a 2.2 or BT.1886 (2.4) display would, as the game was made to be seen: darker shadows. Off keeps the sRGB curve."), + Row(GameOptionTab.Display, "RenoDX", "Color space", 0, 2, 1, 1, GameOptionKind.Choice, TxRdxSpace, false, true, + "The colors the picture is taken to be in. Vanilla: the game's own. BT.709: as the SDR picture. DCI-P3: more saturated."), + Row(GameOptionTab.Display, "RenoDX", "Exposure", 0, 200, 5, 100, GameOptionKind.Slider, TxPercent, false, true, + "The scene's brightness before tone mapping. 100% is the game's."), + Row(GameOptionTab.Display, "RenoDX", "Middle gray", 0, 2, 1, 1, GameOptionKind.Choice, TxRdxMidgray, false, true, + "What the scene's middle gray is held to. Render: as rendered. Vanilla: where the game's curve puts it. Eye adaptation: it follows what is on screen over a few seconds."), + Row(GameOptionTab.Display, "RenoDX", "Match tone slope", 0, 1, 1, 1, GameOptionKind.Choice, TxOnoff, false, true, + "PsychoV35: the mid tones get the contrast of the game's own curve. Off: PsychoV35's own, softer."), + Row(GameOptionTab.Display, "RenoDX", "Tone slope", 0, 100, 1, 50, GameOptionKind.Slider, TxRdxGrade, false, true, + "PsychoV35: how steep the tone curve is through middle gray. 50 leaves it unchanged."), + Row(GameOptionTab.Display, "RenoDX", "Hue clip", 0, 100, 5, 50, GameOptionKind.Slider, TxPercent, false, true, + "PsychoV35: how much bright colors turn as they do where the SDR picture clips, fire towards yellow and white. 0% keeps their hue."), + Row(GameOptionTab.Display, "RenoDX", "Highlight saturation", 0, 100, 1, 50, GameOptionKind.Slider, TxRdxGrade, false, true, + "RenoDRT: how strong the colors of the brightest parts are. 50 leaves them unchanged."), + Row(GameOptionTab.Display, "RenoDX", "Blowout", 0, 100, 1, 0, GameOptionKind.Slider, TxRdxGrade, false, true, + "RenoDRT: turns the brightest parts towards white. 0 keeps their color."), + Row(GameOptionTab.Display, "RenoDX", "Flare", 0, 100, 1, 0, GameOptionKind.Slider, TxRdxGrade, false, true, + "RenoDRT: lifts the darkest parts as a little stray light would. 0 keeps black black."), + Row(GameOptionTab.Display, "RenoDX", "Hue correction", 0, 100, 5, 100, GameOptionKind.Slider, TxPercent, false, true, + "RenoDRT: how much bright colors keep the hue they had before tone mapping."), + Row(GameOptionTab.Display, "RenoDX", "Color grading", 0, 100, 5, 100, GameOptionKind.Slider, TxPercent, false, true, + "RenoDRT: how much of the game's own color grading is kept. 0% is the scene without it."), + Row(GameOptionTab.Display, "RenoDX", "Highlights", 0, 100, 5, 50, GameOptionKind.Slider, TxRdxGrade, false, true, + "The bright parts of the scene. 50 is neutral."), + Row(GameOptionTab.Display, "RenoDX", "Shadows", 0, 100, 5, 50, GameOptionKind.Slider, TxRdxGrade, false, true, + "The dark parts of the scene. 50 is neutral."), + Row(GameOptionTab.Display, "RenoDX", "Contrast", 0, 100, 5, 50, GameOptionKind.Slider, TxRdxGrade, false, true, + "The scene's contrast. 50 is neutral."), + Row(GameOptionTab.Display, "RenoDX", "Saturation", 0, 100, 5, 50, GameOptionKind.Slider, TxRdxGrade, false, true, + "The scene's color. 50 is neutral."), + Row(GameOptionTab.Display, "RenoDX", "Bloom", 0, 100, 5, 50, GameOptionKind.Slider, TxRdxGrade, false, true, + "The glow around bright things. 50 is the game's. RenoDX is by Carlos Lopez (clshortfuse), under the MIT license."), + Row(GameOptionTab.Display, "Overlay", "FPS counter", 0, 2, 1, 0, GameOptionKind.Choice, TxPerf, false, true, + "Frames per second in the top right corner. Full: a performance panel in its place: the GPU's and the CPU's load, temperatures and memory, the frame rate and its lows, the frame times and their graph, the waits per frame and the picture's settings."), + Row(GameOptionTab.Graphics, "Upscaling", "Upscaler", 0, 6, 1, 0, GameOptionKind.Choice, TxUpscaler, false, false, + "What draws the 3D up to the Output resolution."), + Row(GameOptionTab.Graphics, "Upscaling", "DLSS model", 0, 3, 1, 0, GameOptionKind.Choice, TxDlssModel, false, true, "Which of NVIDIA's DLSS networks draws the picture."), - Row(GameOptionTab.Graphics, "Upscaling and edges", "Upscale before bloom", 0, 1, 1, 0, GameOptionKind.Choice, TxOnoff, true, true, - "With NVIDIA DLSS, DLAA or AMD FSR as the Upscaler: On, it works on the scene before the game's glow and tone mapping, so both are made from the sharpened scene. Off: on the finished picture, as before. Applies after a restart."), - Row(GameOptionTab.Graphics, "Upscaling and edges", "Anti-aliasing", 0, 3, 1, 0, GameOptionKind.Choice, TxAa, false, true, - "How the 3D's edges are smoothed before it is scaled up: by the game's own SMAA, not at all, by FXAA, or by SMAA at its Ultra preset. Changes at once."), - Row(GameOptionTab.Graphics, "Upscaling and edges", "Sharpness", 0, 100, 10, 0, GameOptionKind.Slider, TxPercent, false, false, - "Sharpens the picture as it goes to the window, with any filter. Too much shows halos around edges."), - Row(GameOptionTab.Graphics, "Upscaling and edges", "Window scaling filter", 0, 3, 1, 0, GameOptionKind.Choice, TxFilter, false, false, - "How the finished picture is scaled to the window, when its size differs from the Output resolution."), - Row(GameOptionTab.Graphics, "Smoothness", "Frame generation", 0, 9, 1, 0, GameOptionKind.Choice, TxFramegen, false, true, - "Makes frames between the game's own, for smoother motion, at a little input delay. Fast motion or camera cuts can show artifacts."), - Row(GameOptionTab.Graphics, "Smoothness", "DLSS frame generation", 0, 3, 1, 0, GameOptionKind.Choice, TxDlssg, true, true, - "NVIDIA RTX 40 and 50 cards: frames made between the game's own by NVIDIA DLSS, from the scene's depth and motion, with NVIDIA Reflex. The Frame generation row above is not used with it. Applies after a restart."), - Row(GameOptionTab.Graphics, "Smoothness", "Frame pacing", 0, 1, 1, 1, GameOptionKind.Choice, TxOnoff, false, false, - "When each frame is shown."), - Row(GameOptionTab.Graphics, "Latency", "NVIDIA Reflex", 0, 2, 1, 2, GameOptionKind.Choice, TxReflex, false, false, - "NVIDIA GPUs: each frame starts as late as it can, for less input delay. With DLSS frame generation it runs through NVIDIA Streamline; not used with the other frame generation."), - Row(GameOptionTab.Graphics, "Latency", "AMD Anti-Lag 2", 0, 1, 1, 0, GameOptionKind.Choice, TxAntilag, false, false, - "AMD GPUs: each frame starts as late as it can, for less input delay. Not used with frame generation."), - Row(GameOptionTab.Graphics, "Advanced", "Handheld profile", 0, 1, 1, 0, GameOptionKind.Choice, TxOnoff, true, false, - "Runs the game as if the console were undocked. Applies after a restart."), - Row(GameOptionTab.Graphics, "Advanced", "Robust buffers", 0, 2, 1, 1, GameOptionKind.Choice, TxRobust, true, false, - "Bounds-checked buffer reads: a read past a buffer's end is dropped instead of stopping the GPU. Applies after a restart."), + Row(GameOptionTab.Graphics, "Upscaling", "Render resolution", 0, 6, 1, 0, GameOptionKind.Choice, TxResolution, true, false, + "The size the 3D is drawn at; the upscaler fills in the rest. Applies after a restart."), + Row(GameOptionTab.Graphics, "Upscaling", "Upscale before bloom", 0, 1, 1, 0, GameOptionKind.Choice, TxOnoff, true, true, + "On: the upscaler works on the scene before the game's glow and tone mapping, which are then made at full size. Applies after a restart."), + Row(GameOptionTab.Graphics, "Upscaling", "Sharpness", 0, 100, 10, 0, GameOptionKind.Slider, TxPercent, false, false, + "With DLSS, DLAA or AMD FSR as the Upscaler, the 3D before the HUD (FSR's own sharpening; AMD's RCAS for DLSS); with the game's own upscaler, the picture at the window. 0 is off. Too much shows halos around edges."), + Row(GameOptionTab.Graphics, "Anti-aliasing", "Anti-aliasing", 0, 3, 1, 0, GameOptionKind.Choice, TxAa, false, true, + "How the 3D's edges are smoothed: the game's own SMAA, none, FXAA, or SMAA at its Ultra preset. Changes at once."), + Merged(GameOptionTab.Graphics, "Frame generation", "Frame generation", 0, 12, 1, MergeFramegen, TxFramegenRow, true, true, + "Makes frames between the game's own, for smoother motion, at a little input delay. Fast motion and camera cuts can show artifacts. NVIDIA DLSS applies after a restart."), + Row(GameOptionTab.Graphics, "Frame rate & latency", "Frame rate cap", -3, 6, 1, 0, GameOptionKind.Choice, TxFpsCap, true, true, + "The most frames the game renders a second; made frames come on top. 60 is the safe choice: past it some cutscenes and bosses can misbehave. 30, 40 and 45 are for a PC that holds one of them where 60 swings. Monitor's rate: the refresh rate of the screen the game starts on. Applies after a restart."), + Row(GameOptionTab.Graphics, "Frame rate & latency", "VSync", 0, 1, 1, 1, GameOptionKind.Choice, TxOnoff, false, true, + "Whether a finished frame waits for the screen's next refresh. Off: a little less delay, and the picture can tear. Changes at once."), + Merged(GameOptionTab.Graphics, "Frame rate & latency", "Frame pacing", 0, 3, 1, MergePacing, TxPacing, true, false, + "When each frame is shown. Steady paces the game to a rate this PC holds where you are: steadier, at some frame rate."), + Merged(GameOptionTab.Graphics, "Frame rate & latency", "Low latency", 0, 2, 1, MergeLatency, TxLatency, false, false, + "Each frame starts as late as it can, for less input delay: NVIDIA Reflex on NVIDIA cards, AMD Anti-Lag 2 on AMD ones."), + Row(GameOptionTab.Graphics, "Textures", "Texture filtering", 0, 4, 1, 0, GameOptionKind.Choice, TxAniso, false, false, + "Keeps ground, paths and walls sharp where they slope away from the camera. Game: what the game asks for; 16x is the sharpest, at a small cost."), + Row(GameOptionTab.Graphics, "Textures", "Texture detail", 0, 3, 1, 0, GameOptionKind.Choice, TxTexDetail, true, false, + "How large the game's textures are kept in video memory. Automatic suits the graphics card. Applies after a restart."), Row(GameOptionTab.Graphics, "Advanced", "Mesh shaders", 0, 1, 1, 0, GameOptionKind.Choice, TxMesh, true, false, "How the game's mesh-shader draws run. Try Compute fallback on an AMD card that shows glowing walls or loses the GPU. Applies after a restart."), - Row(GameOptionTab.System, "Game", "Game speed", 0, 1, 1, 1, GameOptionKind.Choice, TxSpeed, false, false, - "How the game keeps time when it runs below 60 fps."), Row(GameOptionTab.System, "Game", "Text speed", 0, 3, 1, 0, GameOptionKind.Choice, TxTextSpeed, false, false, "How fast the letters of dialogue and other messages appear."), - Row(GameOptionTab.System, "Game", "Discord status", 0, 1, 1, 1, GameOptionKind.Choice, TxOnoff, false, true, - "Shows \"Playing Triforce\" and the time played in your Discord status, while Discord runs on this PC."), - Row(GameOptionTab.Audio, null, "Master volume", 0, 100, 5, 100, GameOptionKind.Slider, TxVolume, false, true, - "How loud the game is, in percent of its own volume. The game's sounds and music keep their balance."), - Row(GameOptionTab.Audio, null, "Menu sounds", 0, 100, 5, 80, GameOptionKind.Slider, TxMenuSounds, false, true, - "How loud these options' own sounds are: the game's interface sounds, played from your game files as you move through the tabs and rows. Off is silent. The Master volume is over them too."), - Row(GameOptionTab.Audio, null, "Audio output", 0, 2, 1, 0, GameOptionKind.Choice, TxAudioOutput, true, true, + Row(GameOptionTab.System, "Game", "Game speed", 0, 1, 1, 1, GameOptionKind.Choice, TxSpeed, false, false, + "How the game keeps time when it runs below 60 fps."), + Row(GameOptionTab.Audio, "Sound", "Master volume", 0, 100, 5, 100, GameOptionKind.Slider, TxVolume, false, true, + "How loud the game is, in percent of its own volume."), + Row(GameOptionTab.Audio, "Sound", "Menu sounds", 0, 100, 5, 80, GameOptionKind.Slider, TxMenuSounds, false, true, + "How loud these options' own sounds are, the game's interface sounds from your game files."), + Row(GameOptionTab.Audio, "Sound", "Audio output", 0, 2, 1, 0, GameOptionKind.Choice, TxAudioOutput, true, true, "The speakers the game mixes its sound for. Applies after a restart."), - Row(GameOptionTab.Audio, null, "Microphone", 0, 18, 1, 1, GameOptionKind.Choice, TxMic, false, true, + Row(GameOptionTab.Audio, "Microphone", "Microphone", 0, 18, 1, 1, GameOptionKind.Choice, TxMic, false, true, "What the ocarina hears when the game's microphone option is on (playing an instrument, or humming)."), - Row(GameOptionTab.Controls, "Camera", "Camera speed", 25, 300, 5, 100, GameOptionKind.Slider, TxSens, true, true, - "How fast the right stick turns the camera: times the game's own speed, at any Camera sensitivity of the game's Options. The mouse keeps its own sensitivity. Applies after a restart."), - Row(GameOptionTab.Controls, "Camera", "Invert camera X", 0, 1, 1, 0, GameOptionKind.Choice, TxOnoff, false, true, - "Inverts horizontal look (left/right) on the right analog stick."), - Row(GameOptionTab.Controls, "Camera", "Invert camera Y", 0, 1, 1, 0, GameOptionKind.Choice, TxOnoff, false, true, - "Inverts vertical look (up/down) on the right analog stick."), - Row(GameOptionTab.Controls, "Camera", "Mouse camera", 0, 1, 1, 0, GameOptionKind.Choice, TxMousecam, false, true, - "How the captured mouse turns the camera."), - Row(GameOptionTab.Controls, "Camera", "Mouse sensitivity", 10, 500, 10, 100, GameOptionKind.Slider, TxSens, false, true, - "How far the camera turns for a mouse movement, with either Mouse camera: at 1.00 native look turns 0.1 degree a count. The - and = keys change it in game too."), - Row(GameOptionTab.Controls, "Camera", "Invert mouse vertical", 0, 1, 1, 0, GameOptionKind.Choice, TxOnoff, false, true, - "Swaps up and down for the mouse."), - Row(GameOptionTab.Controls, "Sticks", "Stick dead zone", 0, 5, 1, 0, GameOptionKind.Choice, TxStickDeadZone, false, true, - "How far a stick moves before the game sees it, on each axis. Lower for a precise stick, higher for one that drifts. The game's own dead zone still follows."), - Row(GameOptionTab.Controls, "Sticks", "Left stick dead zone", -1, 50, 1, -1, GameOptionKind.Slider, TxOwnDeadZone, false, true, - "The left stick's own dead zone, in percent of its travel on each axis, in place of Stick dead zone: a worn stick can have a larger one than the other. 0% cuts nothing; the game's own dead zone still follows."), - Row(GameOptionTab.Controls, "Sticks", "Right stick dead zone", -1, 50, 1, -1, GameOptionKind.Slider, TxOwnDeadZone, false, true, - "The right stick's own dead zone, in percent of its travel on each axis, in place of Stick dead zone: a worn stick can have a larger one than the other. 0% cuts nothing; the game's own dead zone still follows."), - Row(GameOptionTab.Controls, "Movement", "Sprint button", 0, 2, 1, 0, GameOptionKind.Choice, TxSprint, false, true, - "Link sprints from a button press, without rolling first. Left stick click: the game has no other use for it. B: sprints only while Link moves, and puts the weapon away as usual otherwise (with the game's Swap Jump Controls B is the jump: use the stick's click)."), - Row(GameOptionTab.Controls, "Movement", "Sprint mode", 0, 1, 1, 0, GameOptionKind.Choice, TxSprintMode, false, true, - "Toggle: a press starts the sprint and the next press, or stopping, ends it. Hold: Link sprints while the button is held."), - Row(GameOptionTab.Controls, "Movement", "Walk key", 0, 4, 1, 0, GameOptionKind.Choice, TxWalkKey, false, true, - "A key that makes Link walk on the keyboard: held with the movement keys, they push the left stick only as far as Walk scale. While it is chosen the key presses nothing else."), - Row(GameOptionTab.Controls, "Movement", "Walk scale", 20, 90, 1, 51, GameOptionKind.Slider, TxSens, false, true, - "How far the movement keys push the left stick while the Walk key is held: 1.00 would be all the way, lower walks slower. A controller's stick and the camera's keys are not changed."), - Row(GameOptionTab.Controls, "Vibration", "Rumble strength", 0, 300, 25, 100, GameOptionKind.Slider, TxRumble, false, true, - "How strongly the controller rumbles, in percent of the game's rumble. Off turns it off. Over 100% a rumble that would pass the motors' full strength stops there."), - Row(GameOptionTab.Controls, "Vibration", "Vibration curve", 20, 150, 5, 100, GameOptionKind.Slider, TxSens, false, true, - "Shapes the rumble of a controller's two motors. 1.00 is the game's. Lower lifts the many weak rumbles so they can be felt and hardly changes the strong ones; higher quiets them. A Switch controller's HD rumble is not changed by it."), - Row(GameOptionTab.Controls, "Vibration", "Trigger vibration", 0, 1, 1, 0, GameOptionKind.Choice, TxOnoff, false, true, - "Xbox One and Series controllers: the motors in the triggers rumble too, with the lighter of the two motors. Controllers without them rumble as before."), - Row(GameOptionTab.Controls, "Other", "Controller in background", 0, 1, 1, 0, GameOptionKind.Choice, TxOnoff, false, false, + Row(GameOptionTab.System, "Online", "Discord status", 0, 1, 1, 1, GameOptionKind.Choice, TxOnoff, false, true, + "Shows \"Playing Triforce\" and the time played in your Discord status, while Discord runs on this PC."), + Row(GameOptionTab.Controls, "Controller", "Controller in background", 0, 1, 1, 0, GameOptionKind.Choice, TxOnoff, false, false, "Whether the controller plays the game while another window has the focus."), - Row(GameOptionTab.Controls, "Other", "Menu shortcut", 0, 1, 1, 0, GameOptionKind.Choice, TxMenuShortcut, false, true, + Row(GameOptionTab.Controls, "Controller", "Left stick dead zone", -1, 50, 1, -1, GameOptionKind.Slider, TxOwnDeadZone, false, true, + "How far the left stick moves before the game sees it, in percent of its travel on each axis: lower for a precise stick, higher for one that drifts. The game's own dead zone still follows."), + Row(GameOptionTab.Controls, "Controller", "Right stick dead zone", -1, 50, 1, -1, GameOptionKind.Slider, TxOwnDeadZone, false, true, + "How far the right stick moves before the game sees it, in percent of its travel on each axis: lower for a precise stick, higher for one that drifts. The game's own dead zone still follows."), + Row(GameOptionTab.Controls, "Controller", "Camera speed", 25, 300, 5, 100, GameOptionKind.Slider, TxSens, true, true, + "How fast the right stick turns the camera, times the game's own speed. Applies after a restart."), + Row(GameOptionTab.Controls, "Controller", "Invert camera X", 0, 1, 1, 0, GameOptionKind.Choice, TxOnoff, false, true, + "Swaps left and right for the right stick's camera."), + Row(GameOptionTab.Controls, "Controller", "Invert camera Y", 0, 1, 1, 0, GameOptionKind.Choice, TxOnoff, false, true, + "Swaps up and down for the right stick's camera."), + Row(GameOptionTab.Controls, "Mouse & keyboard", "Mouse camera", 0, 1, 1, 0, GameOptionKind.Choice, TxMousecam, false, true, + "How the captured mouse turns the camera."), + Row(GameOptionTab.Controls, "Mouse & keyboard", "Mouse sensitivity", 10, 500, 10, 100, GameOptionKind.Slider, TxSens, false, true, + "How far the camera turns for a mouse movement. The - and = keys change it in game too."), + Row(GameOptionTab.Controls, "Mouse & keyboard", "Invert mouse vertical", 0, 1, 1, 0, GameOptionKind.Choice, TxOnoff, false, true, + "Swaps up and down for the mouse."), + Row(GameOptionTab.Controls, "Mouse & keyboard", "Walk key", 0, 4, 1, 0, GameOptionKind.Choice, TxWalkKey, false, true, + "A key that makes Link walk: held with the movement keys, they push the stick only as far as Walk scale. While it is chosen the key presses nothing else."), + Row(GameOptionTab.Controls, "Mouse & keyboard", "Walk scale", 20, 90, 1, 51, GameOptionKind.Slider, TxSens, false, true, + "How far the movement keys push the stick while the Walk key is held: 1.00 would be all the way."), + Row(GameOptionTab.Controls, "Mouse & keyboard", "Menu shortcut", 0, 1, 1, 0, GameOptionKind.Choice, TxMenuShortcut, false, true, "Which keys open and close these options from anywhere. F1 always does; F1 only leaves Insert free for an overlay."), + Row(GameOptionTab.Controls, "Rumble", "Rumble strength", 0, 300, 25, 100, GameOptionKind.Slider, TxRumble, false, true, + "How strongly the controller rumbles, in percent of the game's rumble. Off turns it off."), + Row(GameOptionTab.Controls, "Rumble", "Vibration curve", 20, 150, 5, 100, GameOptionKind.Slider, TxSens, false, true, + "Shapes the rumble: 1.00 is the game's. Lower makes the many weak rumbles easier to feel; higher quiets them. A Switch controller's HD rumble is not changed."), + Row(GameOptionTab.Controls, "Rumble", "Trigger vibration", 0, 1, 1, 0, GameOptionKind.Choice, TxOnoff, false, true, + "Xbox One and Series controllers: the motors in the triggers rumble too."), + Row(GameOptionTab.Controls, "Gameplay", "Sprint button", 0, 2, 1, 0, GameOptionKind.Choice, TxSprint, false, true, + "Link sprints from a button press, without rolling first. Left stick click: the game has no other use for it. B: sprints while Link moves (with the game's Swap Jump Controls B is the jump: use the stick's click)."), + Row(GameOptionTab.Controls, "Gameplay", "Sprint mode", 0, 1, 1, 0, GameOptionKind.Choice, TxSprintMode, false, true, + "Toggle: a press starts the sprint, the next press or stopping ends it. Hold: Link sprints while the button is held."), Row(GameOptionTab.Controls, "Motion", "Motion controls", 0, 1, 1, 1, GameOptionKind.Choice, TxGyro, false, true, - "Aims and looks with the controller's gyroscope where the game uses motion, as on the Switch. Works with Switch Pro controllers and pads in Switch mode, DualSense and DualShock 4. Shown beside it: whether the connected pad has one."), + "Aims and looks with the controller's gyroscope where the game uses motion, as on the Switch: Switch Pro controllers and pads in Switch mode, DualSense and DualShock 4."), Row(GameOptionTab.Controls, "Motion", "Motion sensitivity X", 10, 600, 10, 100, GameOptionKind.Slider, TxPercent, false, true, - "How far the aim moves left and right for a given turn of the controller. 100% is the console's own."), + "How far the aim moves left and right for a turn of the controller. 100% is the console's."), Row(GameOptionTab.Controls, "Motion", "Motion sensitivity Y", 10, 300, 10, 100, GameOptionKind.Slider, TxPercent, false, true, - "How far the aim moves up and down for a given tilt of the controller. 100% is the console's own."), + "How far the aim moves up and down for a tilt of the controller. 100% is the console's."), Row(GameOptionTab.Controls, "Motion", "Invert motion X", 0, 1, 1, 0, GameOptionKind.Choice, TxOnoff, false, true, - "Turns the motion aim's left and right around."), + "Swaps the motion aim's left and right."), Row(GameOptionTab.Controls, "Motion", "Invert motion Y", 0, 1, 1, 0, GameOptionKind.Choice, TxOnoff, false, true, - "Turns the motion aim's up and down around."), + "Swaps the motion aim's up and down."), Row(GameOptionTab.Controls, "Motion", "Motion deadzone", 0, 2, 1, 1, GameOptionKind.Choice, TxDeadzone, false, true, - "How slow a turn counts as none, the console's three settings: Standard is its own. High steadies a pad that jitters when held still; Low keeps the smallest movements."), + "How slow a turn counts as none, the console's three settings: Standard is its own. High steadies a pad that jitters; Low keeps the smallest movements."), Row(GameOptionTab.Controls, "Controls", "Button prompts", 0, 6, 1, 0, GameOptionKind.Choice, TxPrompts, true, true, - "The button icons the game shows, in its texts and menus: those of this controller, or the keyboard's and the mouse's, for the buttons or keys your controls put each action on."), + "The button icons the game shows in its texts and menus: those of this controller, or the keyboard's and the mouse's."), }; static readonly Dictionary byKey = All.ToDictionary(r => r.Key, StringComparer.Ordinal); static readonly Dictionary byName = All.ToDictionary(r => r.Name, StringComparer.Ordinal); - /// The row of a triforce_options.ini key (exact, as the game compares them), null for any other key. + /// The row of a key (a triforce_options.ini key, exact as the game compares them, or a merged row's own '@' key), + /// null for any other key. A merged row's parts' keys are no row's: Part has them. public static GameOptionRow? ByKey(string key) => byKey.GetValueOrDefault(key); /// The row of a key; KeyNotFoundException for a key no row has. public static GameOptionRow Get(string key) => ByKey(key) ?? throw new KeyNotFoundException("no Options row has the key " + key); @@ -467,8 +542,14 @@ public static class GameOptionsSchema public static GameOptionRow? ByName(string name) => byName.GetValueOrDefault(name); /// A tab's rows, in order. public static IReadOnlyList RowsOf(GameOptionTab tab) => All.Where(r => r.Tab == tab).ToList(); - /// A tab's sections that have rows here, in order (Audio has none: its rows' Section is null). + /// A tab's sections that have rows here, in order. public static IReadOnlyList Sections(GameOptionTab tab) => RowsOf(tab).Select(r => r.Section).OfType().Distinct().ToList(); + /// A key the launcher reads and writes as a value of its own line: a row's (not a merged row's), or a merged + /// row's part. + public static bool IsValueKey(string key) => ByKey(key) is { IsMerged: false } || Parts.ContainsKey(key); + /// The row a file key belongs to: its own row, or the merged row a part is of (null for any other key). + public static GameOptionRow? RowOfLine(string key) => + ByKey(key) is { IsMerged: false } r ? r : All.FirstOrDefault(m => m.Parts?.Any(p => p.Key == key) == true); // ---- the keys of the file that are not rows here ---- @@ -476,7 +557,7 @@ public static class GameOptionsSchema /// Besides these: every pad_* and key_* line (KeptPrefixes). public static readonly IReadOnlyDictionary KeptKeys = new Dictionary(StringComparer.Ordinal) { - [VersionKey] = "the file's format: the rules the game reads an older file by (GameOptionsFile.Save writes 7 when it brings an older file up to date)", + [VersionKey] = "the file's format: the rules the game reads an older file by (GameOptionsFile.Save writes 8 when it brings an older file up to date)", [MicDeviceKey] = "the Microphone row's device, by its name: the game's to choose from the devices it lists", [GpuNameKey] = "the GPU device row's card by its name (11-20): Save writes it with the row; the game finds the card by it first", ["layout_preset"] = "the Controls page's Layout preset: choosing one in the game rebinds the pad_ lines, which a changed value alone would not do", @@ -488,6 +569,12 @@ public static class GameOptionsSchema ["replay_helper_threads"] = "the Debug tab's Developer rows (not ported)", ["tail_early"] = "the Debug tab's Start next frame early (not ported)", ["opaque_textures_as_bc1"] = "the Debug tab's Developer rows (not ported)", + ["robust_buffers"] = "the Debug tab's Developer rows from options_version 8, the Graphics tab's before (not ported)", + ["quality_preset"] = "a row gone in options_version 8 (merged_rows.h kDropped: its memory limits are the card's own): the game reads nothing from it and drops the line at its next save", + ["upscaling"] = "the Window scaling filter, gone in options_version 8 (the sizes choose the filter: upscaler_rows.h auto_filter): dropped at the game's next save", + ["reflex_frame_limit"] = "a row gone in options_version 8 (off, its old default): dropped at the game's next save", + ["handheld_profile"] = "a row gone in options_version 8: a file before 8 with it on gets the Output resolution's 1536x864 (handheld) once (GameOptionsFile), then the game drops the line", + ["stick_dead_zone"] = "a row gone in options_version 8: a file before 8 gets its share in each stick's own dead zone that had none (GameOptionsFile), then the game drops the line", ["menu_size"] = "the old Menu size row, gone from the menu (the game's menus have one size): the game keeps the line and reads nothing from it", ["dynamic_resolution"] = "the Render resolution's old key: read once by the game where the file has no resolution line", ["debug_tab"] = "an old line (Alt+G's Debug tab): the game drops it at its next save", @@ -496,43 +583,10 @@ public static class GameOptionsSchema }; /// The binding lines of the Controls page's grid: pad_<button>=<input> and key_<button>=<key>,<key>. public static readonly IReadOnlyList KeptPrefixes = new[] { "pad_", "key_" }; - /// A key that is not a row here but is known (KeptKeys, KeptPrefixes): kept in the file as it is. + /// A key that is not a value here but is known (KeptKeys, KeptPrefixes): kept in the file as it is. public static bool IsKept(string key) => KeptKeys.ContainsKey(key) || KeptPrefixes.Any(p => key.StartsWith(p, StringComparison.Ordinal)); - // ---- the Quality preset ---- - - /// What each Quality preset sets (vk_menu.cpp apply_preset), in the menu's own words (hn_preset). - public static readonly IReadOnlyList PresetSummaries = new[] - { - "Each row as you set it. Changing most rows yourself comes back to Custom.", - "Resolution 50% (with DLSS its nearest mode), Lanczos, Sharpness 30%, Handheld profile.", - "Resolution 75% (with DLSS its nearest mode), Lanczos, Sharpness 30%, Handheld profile.", - "Resolution 100% (with DLSS its nearest mode), Catmull-Rom, Sharpness 30%.", - "Resolution 100% (with DLSS its nearest mode), Lanczos, Sharpness 40%.", - "Resolution 100% (with DLSS its nearest mode), Lanczos, Sharpness 50%.", - }; - - /// The rows a Quality preset sets and their values (apply_preset): the Render resolution (then moved to the - /// Upscaler's nearest step: GameOptionsFile.Set does that), the Window scaling filter, Sharpness, Frame pacing, Game speed, - /// the Handheld profile and the Texture detail. Custom (0) sets nothing. The Upscaler and Frame generation stay the - /// player's. (The game also sets its memory budgets from the preset; they are not kept in the file.) - public static IReadOnlyDictionary PresetValues(int preset) - { - var (res, filter, sharp, handheld) = preset switch - { - 1 => (4, 2, 30, 1), - 2 => (2, 2, 30, 1), - 3 => (1, 1, 30, 0), - 4 => (1, 2, 40, 0), - 5 => (1, 2, 50, 0), - _ => (-1, 0, 0, 0), - }; - if (res < 0) return new Dictionary(); - int[] detail = { 0, 3, 2, 1, 1, 1 }; - return new Dictionary(StringComparer.Ordinal) - { - [ResolutionKey] = res, ["upscaling"] = filter, ["sharpness"] = sharp, ["frame_pacing"] = 1, ["game_speed"] = 1, - [HandheldKey] = handheld, ["texture_detail"] = detail[preset], - }; - } + /// merged_rows.h kSharedZone: the old shared Stick dead zone's shares in percent, by its saved value (-1 XInput's + /// own, each stick's Default), for a file before options_version 8. + public static readonly IReadOnlyList SharedZone = new[] { -1, 5, 10, 15, 20, 30 }; } diff --git a/launcher/Logic/Install.cs b/launcher/Logic/Install.cs index a3c41ac..5fe3296 100644 --- a/launcher/Logic/Install.cs +++ b/launcher/Logic/Install.cs @@ -85,26 +85,90 @@ public sealed class Uninstaller readonly TriforceContext ctx; public Uninstaller(TriforceContext ctx) { this.ctx = ctx; } + // ((Triforce, 11-21, ticket "Downloads folder got deleted while uninstalling the Triforce launcher") The uninstall took its + // folder away whole (rmdir /s), and that folder was the install folder or, for a Setup never installed, the folder it ran + // from: a TriforceSetup.exe run from Downloads took the player's whole Downloads folder with it. Now only the launcher's own + // entries go (OwnFiles, and OwnFolders where the folder is the launcher's alone: Dedicated), then the folder itself only + // if nothing else is left in it (rmdir without /s), and a Setup that is not the installed one removes nothing of its folder. + + /// The launcher's own files in its folder (del patterns): its exe, the copies an update or a handover leaves + /// (TriforceSetup.old.exe, TriforceSetup.exe.partial), and its log. + public static readonly string[] OwnFiles = { "TriforceSetup*.exe", "TriforceSetup*.exe.*", "setup.log", "setup.log.*" }; + /// The folders it makes there (TriforceContext ToolsFolder, DownloadsFolder, NgxSdkFolder); save-backups stays (saves). + public static readonly string[] OwnFolders = { "tools", "downloads", "ngx-sdk" }; + + /// Is dir a folder of the launcher's alone, where its folders' names (downloads, tools) can only be its own? The + /// default install folder, or one named Triforce that is not a drive's root, the account's folder or one above it, nor one + /// of the account's known folders. Elsewhere (a player who installed into Downloads, D:\Games or their own folder) its + /// folders' names may be the player's: only its files go there. + public static bool Dedicated(string dir, string defaultDir, IEnumerable known) + { + try + { + var full = Path.GetFullPath(dir).TrimEnd('\\', '/'); + if (TriforceContext.SamePath(full, defaultDir)) return true; + if (Path.GetPathRoot(full) is { } root && TriforceContext.SamePath(full, root)) return false; + foreach (var k in known) + if (!string.IsNullOrEmpty(k) && (TriforceContext.SamePath(full, k) || TriforceContext.Inside(full, k))) return false; + return string.Equals(Path.GetFileName(full), "Triforce", StringComparison.OrdinalIgnoreCase); + } + catch (Exception) { return false; } + } + + /// The account's folders an install folder must never be (nor hold): its own, Desktop, Documents, Downloads and the + /// other known ones, AppData's roots, Program Files. + public static IEnumerable KnownFolders() + { + var profile = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); + foreach (var f in new[] { Environment.SpecialFolder.UserProfile, Environment.SpecialFolder.DesktopDirectory, Environment.SpecialFolder.MyDocuments, + Environment.SpecialFolder.MyMusic, Environment.SpecialFolder.MyPictures, Environment.SpecialFolder.MyVideos, + Environment.SpecialFolder.ApplicationData, Environment.SpecialFolder.LocalApplicationData, Environment.SpecialFolder.ProgramFiles, + Environment.SpecialFolder.ProgramFilesX86, Environment.SpecialFolder.Windows }) + yield return Environment.GetFolderPath(f); + if (!string.IsNullOrEmpty(profile)) + { + yield return Path.Combine(profile, "Downloads"); + yield return Path.Combine(profile, "OneDrive"); + } + } + + /// The short-lived cmd's line that removes the launcher from dir once this process is gone: its files, its folders + /// where dir is Dedicated, then dir itself only when empty. + public static string RemoveCommand(string dir, bool dedicated) + { + var d = Path.GetFullPath(dir).TrimEnd('\\', '/'); + var line = "/c timeout /t 2 /nobreak >nul"; + foreach (var f in OwnFiles) line += " & del /f /q \"" + Path.Combine(d, f) + "\" 2>nul"; + if (dedicated) + foreach (var f in OwnFolders) line += " & rmdir /s /q \"" + Path.Combine(d, f) + "\" 2>nul"; + return line + " & rmdir \"" + d + "\" 2>nul"; + } + public UninstallPlan Describe() { + var dir = ctx.InstalledDir; var removes = new List { - "Triforce and its tools (" + ctx.Home + ")", + dir != null ? "Triforce and its tools (" + dir + ")" : "Triforce's settings and Apps-list entry", "the shortcuts", "the build's work folder and compile cache (" + ctx.WorkFolder + ", made again by a build)", "the launcher's settings", }; if (!ctx.IsWindows) removes[0] = "the launcher's updates and NVIDIA's SDK (" + ctx.Platform.DefaultInstallDir + ")"; var keeps = new List { "your game folders" + (ctx.Settings.GameFolder is { } g ? " (" + g + ")" : ""), "your saves" }; + if (ctx.IsWindows && dir == null) keeps.Add("this Setup file and everything beside it (" + ctx.SelfDir + "): delete the file yourself"); + else if (ctx.IsWindows && dir != null && !Dedicated(dir, ctx.Platform.DefaultInstallDir, KnownFolders())) + keeps.Add("everything in " + dir + " that is not Triforce's own files"); return new UninstallPlan(removes, keeps); } - /// Does it. The launcher's own folder goes once this process is gone (Windows: a short-lived cmd waits 2 s, then - /// rmdir; Linux: removed now, a running binary can be unlinked). + /// Does it. The launcher's own files go once this process is gone (Windows: a short-lived cmd waits 2 s, then + /// removes them: RemoveCommand; Linux: removed now, a running binary can be unlinked). public void Run() { - var dir = ctx.InstalledDir ?? ctx.Home; - ctx.Log.Write("uninstall: " + dir); + // (the installed launcher's folder only: never the folder a Setup that is not installed runs from, Downloads) + var dir = ctx.IsWindows ? ctx.InstalledDir : ctx.Home; + ctx.Log.Write("uninstall: " + (dir ?? "(not installed: nothing removed from " + ctx.SelfDir + ")")); ctx.Platform.RemoveShortcuts(); // (the build's work folder and compile cache, nx2/make_game.py --work: several GB, made again by a build) var work = ctx.WorkFolder; @@ -114,9 +178,13 @@ public sealed class Uninstaller if (ctx.IsWindows) { // (this exe is running from the folder: a short-lived cmd removes it once we are gone) - if (File.Exists(Path.Combine(dir, BuildInfo.SetupAsset)) && ctx.Allowed(dir)) - Process.Start(new ProcessStartInfo("cmd.exe", "/c timeout /t 2 /nobreak >nul & rmdir /s /q \"" + dir + "\"") + if (dir != null && File.Exists(Path.Combine(dir, BuildInfo.SetupAsset)) && ctx.Allowed(dir)) + { + var dedicated = Dedicated(dir, ctx.Platform.DefaultInstallDir, ctx.TestRoot != null ? Array.Empty() : KnownFolders()); + ctx.Log.Write("uninstall: " + (dedicated ? "its files and folders" : "its files alone (the folder is not Triforce's alone)")); + Process.Start(new ProcessStartInfo("cmd.exe", RemoveCommand(dir, dedicated)) { CreateNoWindow = true, UseShellExecute = false, WorkingDirectory = Path.GetTempPath() })?.Dispose(); + } } else { diff --git a/launcher/Logic/Tools.cs b/launcher/Logic/Tools.cs index 6fe3fde..44f7c60 100644 --- a/launcher/Logic/Tools.cs +++ b/launcher/Logic/Tools.cs @@ -229,12 +229,24 @@ public sealed class Tools } /// Whether the tools in folder hold base b whole (progress: 0..1 while it reads the files). Remembers a match in base.have. + /// ((Triforce, 11-21, ticket "Every tools update re-downloads the whole base, even when it didn't change") A file of the + /// base that the tools rewrite as they run: the bundled Python's bytecode caches (__pycache__\*.pyc), which Python writes + /// again whenever its sources are newer or the cache's interpreter differs. Their sizes and hashes in base.list were those of + /// the packed folder, so after one build 352 of the 417 differed and every update fetched the whole base (~95% of the tools) + /// again. Not compared: one that is stale or missing is made again by Python itself. + public static bool Regenerated(string path) + { + var p = path.Replace('\\', '/'); + return p.EndsWith(".pyc", StringComparison.OrdinalIgnoreCase) || p.Contains("/__pycache__/", StringComparison.OrdinalIgnoreCase); + } + public static bool HasBase(Base b, string folder, Action? progress) { foreach (var d in b.Dirs) if (!Directory.Exists(Path.Combine(folder, d))) return false; foreach (var f in b.Files) // (every file there, at its size: quick, and enough once base.have says the rest was read) { + if (Regenerated(f.Path)) continue; var fi = new FileInfo(Path.Combine(folder, f.Path)); if (!fi.Exists || fi.Length != f.Size) return false; } @@ -244,6 +256,7 @@ public sealed class Tools long total = Math.Max(1, b.Files.Sum(f => f.Size)), done = 0; foreach (var f in b.Files) { + if (Regenerated(f.Path)) { done += f.Size; continue; } string got; using (var s = File.OpenRead(Path.Combine(folder, f.Path))) got = Convert.ToHexStringLower(SHA256.HashData(s)); if (got != f.Sha) return false; diff --git a/launcher/LogicTests/Tests.Build.cs b/launcher/LogicTests/Tests.Build.cs index b4d0c0d..ee58063 100644 --- a/launcher/LogicTests/Tests.Build.cs +++ b/launcher/LogicTests/Tests.Build.cs @@ -58,6 +58,8 @@ static partial class Tests Check(none.Text.Contains(" to "), "a range in words: " + none.Text); var amd = BuildEstimator.Estimate(BuildHistory.Load(root), false, 16, GpuVendor.Amd); Equal(TimeSpan.FromMinutes(120), amd.Of("shaders")!.High, "AMD's shader step up to 2 hours"); + var intel = BuildEstimator.Estimate(BuildHistory.Load(root), false, 12, GpuVendor.Intel); + Equal(TimeSpan.FromMinutes(240), intel.Of("shaders")!.High, "Intel's shader step up to 4 hours (an A750: 4.3 h on v0.6.0)"); File.WriteAllText(Path.Combine(root, "build_times.json"), new JsonObject { ["format"] = 1, @@ -74,6 +76,8 @@ static partial class Tests var est2 = BuildEstimator.Estimate(BuildHistory.Load(root), false, 8, GpuVendor.Nvidia); Equal(TimeSpan.FromSeconds(1500), est2.Of("compile")!.Low, "the stage record wins"); Equal(TimeSpan.FromSeconds(600), est2.Of("shaders")!.High, "the shader step measured too"); + var upd = BuildEstimator.Estimate(BuildHistory.Load(root), true, 8, GpuVendor.Intel); + Equal(TimeSpan.FromSeconds(600), upd.Of("shaders")!.High, "an update with no update on record: the build's measured shader step, not the stated range"); Equal("About 40 minutes", est2.Text, "one time in words"); var tr = new BuildTracker(); tr.Feed("@@stage compile Compiling"); @@ -85,6 +89,112 @@ static partial class Tests Equal("1 h 5 min", BuildClock.Span(TimeSpan.FromMinutes(65)), "span"); } + static BuildEstimate OneStage(string stage, double low, double high, bool measured) => + new(new[] { new StageEstimate(stage, TimeSpan.FromSeconds(low), TimeSpan.FromSeconds(high), measured) }, measured); + + /// A stage run second by second on a fake clock: progress(t) said as the game says it (each time the tenth of a percent + /// moves), the shown time left asked each second as the App's tick does. The shown seconds at each second. + static double[] RunStage(BuildEstimate est, string stage, Func progress, int seconds) + { + var t0 = new DateTime(2026, 11, 22, 10, 0, 0, DateTimeKind.Utc); + var now = t0; + var tr = new BuildTracker(() => now); + tr.Feed("@@stage " + stage + " x"); + var shown = new double[seconds + 1]; + var said = 0.0; + for (var t = 0; t <= seconds; t++) + { + now = t0.AddSeconds(t); + var p = Math.Floor(Math.Clamp(progress(t), 0, 1) * 1000) / 1000; + if (p > said) { said = p; tr.Feed("@@progress " + p.ToString("0.000", System.Globalization.CultureInfo.InvariantCulture)); } + shown[t] = BuildClock.Remaining(est, tr).TotalSeconds; + } + return shown; + } + + static void EtaSlowStart() + { + // (an Intel-like shader step of an hour: the compute pipelines, the first 14 %, take 52 minutes; the rest 8. The rate since + // the stage began said 4 hours and more there) + static double P(double t) => t < 3120 ? 0.14 * t / 3120 : 0.14 + 0.86 * (t - 3120) / 480; + foreach (var (est, what) in new[] { (OneStage("shaders", 600, 2400, false), "the stated range"), (OneStage("shaders", 3600, 3600, true), "this PC's last time") }) + { + var shown = RunStage(est, "shaders", P, 3600); + for (var t = 300; t <= 3600; t++) + { + Check(shown[t] < 90 * 60, what + ": no hours after the first minutes (" + BuildClock.LeftText(TimeSpan.FromSeconds(shown[t])) + " at " + t + " s)"); + Check(shown[t] <= shown[t - 1] + 1, what + ": never up by more than a second a second (" + shown[t - 1] + " to " + shown[t] + " at " + t + " s)"); + } + Check(shown[3540] < 180, what + ": near the end it says so: " + shown[3540]); + } + // (this PC's own time knows the hour: it is told within a quarter of an hour all the way) + var measured = RunStage(OneStage("shaders", 3600, 3600, true), "shaders", P, 3600); + for (var t = 0; t <= 3600; t += 60) Check(Math.Abs(measured[t] - (3600 - t)) < 15 * 60, "measured: " + measured[t] + " at " + t + " s"); + } + + static void EtaStall() + { + // (a 20 minute stage, steady, that sits 10 minutes on one item at 400 s) + static double P(double t) => t < 400 ? t / 1200 : t < 1000 ? 400.0 / 1200 : (t - 600) / 1200; + var shown = RunStage(OneStage("shaders", 1200, 1200, true), "shaders", P, 1800); + var before = shown[400]; + var most = shown.Skip(400).Take(900).Max(); + Check(most - before <= 4 * 60, "the stall and the steps after it add at most 4 minutes: " + before + " to " + most); + Check(shown.Skip(400).Take(600).Min() >= before - 120, "it holds while nothing moves: " + shown.Skip(400).Take(600).Min()); + Check(Math.Abs(shown[1500] - 300) < 120, "back on time once it moves again: " + shown[1500]); + Check(shown[1790] < 60, "done at the end: " + shown[1790]); + } + + static void EtaSteady() + { + var shown = RunStage(OneStage("shaders", 1200, 1200, true), "shaders", t => t / 1200, 1200); + for (var t = 60; t + 60 <= 1200; t += 60) + { + var fell = shown[t] - shown[t + 60]; + Check(fell is > 45 and < 75, "a minute a minute (" + fell + " s at " + t + " s)"); + Check(Math.Abs(shown[t] - (1200 - t)) < 60, "on time: " + shown[t] + " at " + t + " s"); + } + // (slower than the stated range said: the measured rate takes over and it is told right by the middle) + var slow = RunStage(OneStage("shaders", 600, 2400, false), "shaders", t => t / 2400.0, 2400); + for (var t = 1200; t <= 2340; t += 60) Check(Math.Abs(slow[t] - (2400 - t)) < 0.2 * (2400 - t) + 60, "slow but steady: " + slow[t] + " at " + t + " s"); + for (var t = 1; t <= 2400; t++) Check(slow[t] <= slow[t - 1] + 0.5, "never up by more than half a second a second at " + t + " s"); + } + + static void EtaOtherStages() + { + var est = new BuildEstimate(new[] + { + new StageEstimate("read", TimeSpan.FromSeconds(90), TimeSpan.FromSeconds(90), true), new StageEstimate("lift", TimeSpan.FromSeconds(130), TimeSpan.FromSeconds(130), true), + new StageEstimate("compile", TimeSpan.FromSeconds(1500), TimeSpan.FromSeconds(1500), true), new StageEstimate("finish", TimeSpan.FromSeconds(70), TimeSpan.FromSeconds(70), true), + new StageEstimate("shaders", TimeSpan.FromSeconds(600), TimeSpan.FromSeconds(600), true), + }, true); + var t0 = new DateTime(2026, 11, 22, 10, 0, 0, DateTimeKind.Utc); + var now = t0; + var tr = new BuildTracker(() => now); + Equal(TimeSpan.FromSeconds(2390), BuildClock.Estimated(est, tr), "before the first stage: every stage"); + tr.Feed("@@stage lift Translating"); + now = t0.AddSeconds(30); + Equal(TimeSpan.FromSeconds(100 + 1500 + 70 + 600), BuildClock.Estimated(est, tr), "no progress: its estimate less its time"); + Equal(TimeSpan.FromSeconds(100 + 1500 + 70 + 600), BuildClock.Remaining(est, tr), "shown as it is"); + now = t0.AddSeconds(40); + Equal(TimeSpan.FromSeconds(90 + 1500 + 70 + 600), BuildClock.Remaining(est, tr), "and counted down"); + now = t0.AddSeconds(400); + Equal(TimeSpan.FromSeconds(13 + 0.25 * 270 + 1500 + 70 + 600), BuildClock.Estimated(est, tr), "past its estimate: a tenth of it and a quarter of the time over"); + // (steady progress: by its rate) + var lift = new BuildTracker(() => now); + now = t0; + lift.Feed("@@stage lift Translating"); + for (var t = 1; t <= 65; t++) { now = t0.AddSeconds(t); lift.Feed("@@progress " + (t / 130.0).ToString("0.000", System.Globalization.CultureInfo.InvariantCulture)); } + Check(Math.Abs(BuildClock.Estimated(est, lift).TotalSeconds - (65 + 1500 + 70 + 600)) < 3, "half way at its pace: " + BuildClock.Estimated(est, lift)); + // (the compile by its @@eta, as before; the eta goes with the stage) + lift.Feed("@@stage compile Compiling"); + lift.Feed("@@eta 300"); + Equal(TimeSpan.FromSeconds(300 + 70 + 600), BuildClock.Estimated(est, lift), "the compile's eta"); + lift.Feed("@@stage finish Putting"); + Equal(TimeSpan.FromSeconds(70 + 600), BuildClock.Estimated(est, lift), "the eta ends with the compile"); + Equal(1, lift.Recent.Count, "a new stage starts its samples over"); + } + /// A test root with fake tools: tools\nx2\make_game.py, and this program as tools\python\python.exe. static TriforceContext FakeTools(string root) { diff --git a/launcher/LogicTests/Tests.Core.cs b/launcher/LogicTests/Tests.Core.cs index cf49dac..7338d61 100644 --- a/launcher/LogicTests/Tests.Core.cs +++ b/launcher/LogicTests/Tests.Core.cs @@ -32,6 +32,10 @@ static partial class Tests yield return ("tools: update from the fake server with progress", ToolsUpdate); yield return Sync("build: @@ protocol from a recorded transcript", Protocol); yield return Sync("build: estimates from history and without", Estimates); + yield return Sync("build: time left, slow first part of the shaders then fast", EtaSlowStart); + yield return Sync("build: time left, one item stalls for 10 minutes", EtaStall); + yield return Sync("build: time left, a steady stage counts down", EtaSteady); + yield return Sync("build: time left, other stages and the compile eta", EtaOtherStages); yield return ("build: runner with a fake make_game (done, args, log, times)", RunnerDone); yield return ("build: runner stop kills the tree", RunnerStop); yield return ("build: runner failure and test-root refusal", RunnerFail); @@ -382,6 +386,60 @@ static partial class Tests Check(!Directory.Exists(Path.Combine(root, "build")), "work folder gone"); Equal(null, Ctx(root, self).Platform.Settings.GetUninstall("InstallLocation"), "uninstall entry gone"); Equal(null, Ctx(root, self).Settings.GameFolder, "settings gone"); + UninstallKeepsOthers(); + } + + // (11-21, ticket "Downloads folder got deleted while uninstalling the Triforce launcher": the whole folder went, rmdir /s, + // and for a Setup run from Downloads, never installed, that folder was Downloads) + static void UninstallKeepsOthers() + { + var def = @"C:\Users\Ann\AppData\Local\Programs\Triforce"; + var known = new[] { @"C:\Users\Ann", @"C:\Users\Ann\Downloads", @"C:\Users\Ann\Desktop", @"C:\Users\Ann\AppData\Local" }; + Check(Uninstaller.Dedicated(def, def, known), "the default install folder is the launcher's"); + Check(Uninstaller.Dedicated(@"D:\Triforce", def, known), "a folder named Triforce is"); + Check(!Uninstaller.Dedicated(@"C:\Users\Ann\Downloads", def, known), "Downloads is not"); + Check(!Uninstaller.Dedicated(@"C:\Users\Ann", def, known), "the account's folder is not"); + Check(!Uninstaller.Dedicated(@"C:\Users", def, known), "a folder above it is not"); + Check(!Uninstaller.Dedicated(@"D:\", def, known), "a drive's root is not"); + Check(!Uninstaller.Dedicated(@"D:\Games", def, known), "a folder of the player's own is not"); + var c = Uninstaller.RemoveCommand(@"C:\Users\Ann\Downloads", false); + Check(!c.Contains("rmdir /s"), "outside a folder of its own, no folder is removed whole: " + c); + Check(c.Contains(@"del /f /q ""C:\Users\Ann\Downloads\TriforceSetup*.exe"""), "its exe goes"); + Check(c.EndsWith(@"rmdir ""C:\Users\Ann\Downloads"" 2>nul"), "the folder itself only when empty (no /s)"); + var c2 = Uninstaller.RemoveCommand(def, true); + Check(c2.Contains(@"rmdir /s /q """ + def + @"\tools""") && c2.Contains(@"rmdir /s /q """ + def + @"\downloads"""), "its own folders go in its own folder"); + Check(!c2.Contains(@"rmdir /s /q """ + def + @""""), "but never the folder whole"); + + // (a Setup that is not installed, run from Downloads beside the player's files: nothing of that folder goes) + var root = NewRoot("uninstall-downloads"); + var downloads = Path.Combine(root, "Downloads"); + var self = Path.Combine(downloads, BuildInfo.SetupAsset); + Directory.CreateDirectory(downloads); + File.WriteAllText(self, "launcher bytes"); + File.WriteAllText(Path.Combine(downloads, "photo.jpg"), "the player's"); + var ctx = Ctx(root, self); + Equal(null, ctx.InstalledDir, "not installed"); + var plan = new Uninstaller(ctx).Describe(); + Check(plan.Keeps.Any(k => k.Contains(downloads)), "the plan says the Setup's folder stays"); + new Uninstaller(ctx).Run(); + Thread.Sleep(3500); + Check(File.Exists(Path.Combine(downloads, "photo.jpg")) && File.Exists(self), "the Setup's folder is left as it was"); + + // (installed: its files and folders go, the player's file put beside them stays, and so the folder) + var root2 = NewRoot("uninstall-installed"); + var self2 = Path.Combine(root2, "Downloads", BuildInfo.SetupAsset); + Directory.CreateDirectory(Path.GetDirectoryName(self2)!); + File.WriteAllText(self2, "launcher bytes"); + var ctx2 = Ctx(root2, self2); + var exe = new Installer(ctx2).Run(new Installer(ctx2).DefaultDir, desktop: false, startMenu: false); + var idir = Path.GetDirectoryName(exe)!; + Directory.CreateDirectory(Path.Combine(idir, "tools", "x")); + File.WriteAllText(Path.Combine(idir, "notes.txt"), "the player's"); + new Uninstaller(ctx2).Run(); + Thread.Sleep(3500); + Check(!File.Exists(exe) && !Directory.Exists(Path.Combine(idir, "tools")), "installed: its exe and tools gone"); + Check(File.Exists(Path.Combine(idir, "notes.txt")), "and a file of the player's there stays"); + Check(File.Exists(self2), "the downloaded Setup it was installed from stays"); } // ---- report, languages ---- diff --git a/launcher/LogicTests/Tests.GameOptions.cs b/launcher/LogicTests/Tests.GameOptions.cs index 3fb3bd6..f938f0a 100644 --- a/launcher/LogicTests/Tests.GameOptions.cs +++ b/launcher/LogicTests/Tests.GameOptions.cs @@ -17,7 +17,7 @@ static partial class Tests yield return Sync("game options: keys, labels, choices, no Hookshot", GameOptionsLabels); yield return Sync("game options: the real triforce_options.ini (read only)", GameOptionsRealFile); yield return Sync("game options: round trip, merge, atomic write (a copy)", GameOptionsWrite); - yield return Sync("game options: Set's rules, Quality preset, Apply", GameOptionsSetRules); + yield return Sync("game options: Set's rules, merged rows, Upscaler, Apply", GameOptionsSetRules); yield return Sync("game options: an older options_version, a new file", GameOptionsOldVersion); } @@ -55,6 +55,10 @@ static partial class Tests sb.Append(c); } body = sb.ToString(); + // (the in-game UI merge, 11-20: section titles written as MENU_HEADING(tab, "title"), vk_menu.cpp's macro for a heading + // row: expanded here as the compiler does, so the sections are read again) + body = Regex.Replace(body, @"MENU_HEADING\((TAB_[A-Z]+),\s*(""(?:[^""\\]|\\.)*"")\)", + "{$1, $2, nullptr, 0, 0, 0, nullptr, nullptr, nullptr, false, true, -1, true}"); var rows = new List(); foreach (Match m in Regex.Matches(body, @"\{(TAB_[A-Z]+),")) { @@ -117,8 +121,8 @@ static partial class Tests Check(table.Count > 100, "the table read: " + table.Count + " entries"); var symbols = new Dictionary { - ["kLastOutput"] = 6, ["ur::kValues - 1"] = 6, ["ur::kUpscalers - 1"] = 6, ["ur::kModels - 1"] = 3, ["aa_rows::kValues - 1"] = 3, - ["2 + kMaxMics"] = 18, ["HOS_PROMPT_N"] = 6, + ["kLastOutput"] = 7, ["ur::kValues - 1"] = 6, ["ur::kUpscalers - 1"] = 6, ["ur::kModels - 1"] = 3, ["aa_rows::kValues - 1"] = 3, + ["2 + kMaxMics"] = 18, ["HOS_PROMPT_N"] = 6, ["mr7::FG_DLSS_LAST"] = 12, ["framegen_rows::kRowMax"] = 9, }; int Num(string s, string what) => int.TryParse(s, out var v) ? v : symbols.TryGetValue(s, out v) ? v : throw new TestFailed(what + ": a symbol the test does not know, " + s); var tabs = new Dictionary @@ -127,13 +131,31 @@ static partial class Tests ["TAB_CONTROLS"] = GameOptionTab.Controls, ["TAB_SYSTEM"] = GameOptionTab.System, ["TAB_MOTION"] = GameOptionTab.Controls, ["TAB_PAGE"] = GameOptionTab.Controls, }; + // ((Triforce, options_version 8) the saved values the merged rows stand for: the TAB_HIDDEN rows, each a part here, with + // the hidden row's range and vk_menu.h's default; and the merged rows' own (&merged.*): their default the view of their + // parts' defaults, by the schema's own rules) + var hidden = table.Where(r => r.Tab == "TAB_HIDDEN" && r.Value != "nullptr").ToList(); + var partDefaults = new Dictionary(StringComparer.Ordinal); + foreach (var r in hidden) + { + var key = GameOptionsSchema.IniKey(r.Name); + var part = GameOptionsSchema.Part(key); + Check(part != null, "the hidden row \"" + r.Name + "\" is a merged row's part, key " + key); + Equal(Num(r.Lo, r.Name), part!.Lo, r.Name + " (hidden): lo"); + Equal(Num(r.Hi, r.Name), part.Hi, r.Name + " (hidden): hi"); + Equal(defaults[r.Value["&opts.".Length..]], part.Default, r.Name + " (hidden): default"); + Equal(1, GameOptionsSchema.All.Count(m => m.Parts?.Any(p => p.Key == key) == true), r.Name + " (hidden): the part of one merged row"); + partDefaults[key] = part.Default; + } + Equal(string.Join(",", hidden.Select(r => GameOptionsSchema.IniKey(r.Name)).OrderBy(k => k, StringComparer.Ordinal)), + string.Join(",", GameOptionsSchema.Parts.Keys.OrderBy(k => k, StringComparer.Ordinal)), "the parts are the hidden rows"); var left = new[] { "&opts.language", "&opts.pad_layout" }; // (Game language: language.txt's; Layout preset: KeptKeys) var sections = new Dictionary(); var names = new List(); foreach (var r in table) { if (r.Heading) { sections[r.Tab] = r.Name; continue; } - if (r.Value == "nullptr" || r.Tab == "TAB_DEBUG" || left.Contains(r.Value)) continue; + if (r.Value == "nullptr" || r.Tab == "TAB_DEBUG" || r.Tab == "TAB_HIDDEN" || left.Contains(r.Value)) continue; var row = GameOptionsSchema.ByName(r.Name); Check(row != null, "the menu's row \"" + r.Name + "\" is in the schema"); names.Add(r.Name); @@ -146,9 +168,24 @@ static partial class Tests Equal(r.Help, row.Help, r.Name + ": help"); Equal(r.Restart, row.Restart, r.Name + ": restart"); Equal(r.Exempt, row.PresetExempt, r.Name + ": preset exempt"); - Equal(defaults[r.Value["&opts.".Length..]], row.Default, r.Name + ": default"); + if (r.Value.StartsWith("&merged.", StringComparison.Ordinal)) + { + Check(row.IsMerged, r.Name + ": a merged row"); + Equal(row.Merge!.Of(row.Parts!.Select(p => partDefaults[p.Key]).ToArray()), row.Default, r.Name + ": default (its parts')"); + } + else + { + Check(!row.IsMerged && GameOptionsSchema.IniKey(r.Name) == row.Key, r.Name + ": its own line, key " + row.Key); + Equal(defaults[r.Value["&opts.".Length..]], row.Default, r.Name + ": default"); + } } Equal(string.Join("|", names), string.Join("|", GameOptionsSchema.All.Select(r => r.Name)), "the rows, in the menu's order"); + // (the lines the game's save drops, merged_rows.h kDropped: kept keys, read once to move them) + var mr = File.ReadAllText(Path.Combine(dir!, "merged_rows.h")); + Check(mr.Contains("kVersion = " + GameOptionsSchema.CurrentVersion + ";"), "merged_rows.h's kVersion is CurrentVersion"); + var dropped = Regex.Match(mr, @"kDropped\[\]\s*=\s*\{([^}]*)\}").Groups[1].Value; + foreach (Match d in Regex.Matches(dropped, "\"([^\"]+)\"")) + Check(GameOptionsSchema.KeptKeys.ContainsKey(d.Groups[1].Value) && GameOptionsSchema.ByKey(d.Groups[1].Value) == null, d.Groups[1].Value + ": dropped by the game, a kept key here"); // (the Debug tab's keys, and the Layout preset's, are the KeptKeys: each with its reason) foreach (var r in table.Where(r => r.Value != "nullptr" && !r.Heading && (r.Tab == "TAB_DEBUG" || r.Value == "&opts.pad_layout"))) Check(GameOptionsSchema.KeptKeys.ContainsKey(GameOptionsSchema.IniKey(r.Name)), r.Name + "'s key " + GameOptionsSchema.IniKey(r.Name) + " is a kept key"); @@ -162,9 +199,18 @@ static partial class Tests Equal(all.Count, all.Select(r => r.Key).Distinct().Count(), "keys are unique"); Check(all.All(r => r.Key.Length <= 31 && !r.Key.Contains(' ')), "keys as ini_key makes them"); Equal("frame_rate_cap", GameOptionsSchema.Get("frame_rate_cap").Key, "renamed row keeps its key"); - Equal("Game frame rate cap", GameOptionsSchema.Get("frame_rate_cap").Name, "by key"); - Equal("amd_anti-lag_2", GameOptionsSchema.ByName("AMD Anti-Lag 2")!.Key, "AMD Anti-Lag 2's key"); - Equal("upscaling", GameOptionsSchema.ByName("Window scaling filter")!.Key, "Window scaling filter's key"); + Equal("Frame rate cap", GameOptionsSchema.Get("frame_rate_cap").Name, "by key"); + // ((Triforce, options_version 8) the rows renamed into their groups keep their keys; a merged row has none of its own) + Equal("ultrawide", GameOptionsSchema.ByName("Aspect ratio")!.Key, "Aspect ratio's key"); + Equal("hdr_paper_white", GameOptionsSchema.ByName("Paper white")!.Key, "Paper white's key"); + Equal("renodx_bloom", GameOptionsSchema.ByName("Bloom")!.Key, "Bloom's key"); + Equal(GameOptionsSchema.LatencyKey, GameOptionsSchema.ByName("Low latency")!.Key, "Low latency: a merged row"); + Equal("nvidia_reflex,amd_anti-lag_2", string.Join(",", GameOptionsSchema.ByName("Low latency")!.Parts!.Select(p => p.Key)), "its parts' keys"); + Equal(null, GameOptionsSchema.ByKey("amd_anti-lag_2"), "a part is no row"); + Check(GameOptionsSchema.Part("amd_anti-lag_2") != null && GameOptionsSchema.IsValueKey("amd_anti-lag_2"), "but a value"); + Check(!GameOptionsSchema.IsValueKey(GameOptionsSchema.HdrKey), "a merged row's own key is no line's"); + foreach (var gone in new[] { "Quality preset", "Window scaling filter", "Reflex frame limit", "Handheld profile", "Stick dead zone", "Robust buffers" }) + Equal(null, GameOptionsSchema.ByName(gone), gone + ": no row (options_version 8)"); Equal("tail_early", GameOptionsSchema.IniKey("Start next frame early"), "Start next frame early's key"); Equal("export_save_for_a_bug_report", GameOptionsSchema.IniKey("Export save for a bug report"), "a long name: 28 letters"); Equal("abcdefghijklmnopqrstuvwxyzabcde", GameOptionsSchema.IniKey("ABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGH"), "cut at 31"); @@ -186,25 +232,39 @@ static partial class Tests } else Equal(0, r.Choices.Count, r.Name + ": a slider has no choice list"); } - Equal(1, all.Count(r => r.Tab == GameOptionTab.Graphics && r.Key == GameOptionsSchema.PresetKey), "the preset row"); - Equal("Screen,Widescreen,Frame rate,High dynamic range,RenoDX", string.Join(",", GameOptionsSchema.Sections(GameOptionTab.Display)), "Display's sections"); - Equal("Quality,Upscaling and edges,Smoothness,Latency,Advanced", string.Join(",", GameOptionsSchema.Sections(GameOptionTab.Graphics)), "Graphics' sections"); - Equal("Camera,Sticks,Movement,Vibration,Other,Motion,Controls", string.Join(",", GameOptionsSchema.Sections(GameOptionTab.Controls)), "Controls' sections"); - Equal(0, GameOptionsSchema.Sections(GameOptionTab.Audio).Count, "Audio has none"); + Equal("Screen,Ultrawide,HDR,RenoDX,Overlay", string.Join(",", GameOptionsSchema.Sections(GameOptionTab.Display)), "Display's sections"); + Equal("Upscaling,Anti-aliasing,Frame generation,Frame rate & latency,Textures,Advanced", string.Join(",", GameOptionsSchema.Sections(GameOptionTab.Graphics)), "Graphics' sections"); + Equal("Controller,Mouse & keyboard,Rumble,Gameplay,Motion,Controls", string.Join(",", GameOptionsSchema.Sections(GameOptionTab.Controls)), "Controls' sections"); + Equal("Sound,Microphone", string.Join(",", GameOptionsSchema.Sections(GameOptionTab.Audio)), "Audio's sections"); + Equal("Game,Online", string.Join(",", GameOptionsSchema.Sections(GameOptionTab.System)), "System's sections (with rows)"); string L(string key, int v) => GameOptionsSchema.Get(key).Label(v); string Cs(string key) => string.Join("|", GameOptionsSchema.Get(key).Choices.Select(c => c.Value + "=" + c.Label)); Equal("-3=30|-2=40|-1=45|0=60 (the console's)|1=90|2=120|3=144|4=165|5=240|6=Monitor's rate", Cs("frame_rate_cap"), "fps cap"); - Equal("3=1920x1080|0=2048x1152|1=2560x1440|2=3840x2160|4=5120x1440|6=5120x2160|5=5120x2880", Cs("output_res"), "output sizes, smallest first"); - Equal("0=Game|1=NVIDIA DLSS|2=NVIDIA DLAA|3=AMD FSR 3.1|6=AMD FSR 3.1 Native AA|4=AMD FSR 4|5=AMD FSR 4 Native AA", Cs("dlss"), "upscalers in the menu's order"); + Equal("7=1536x864 (handheld)|3=1920x1080|0=2048x1152|1=2560x1440|2=3840x2160|4=5120x1440|6=5120x2160|5=5120x2880", Cs("output_res"), "output sizes, smallest first"); + Equal("0=Game|1=NVIDIA DLSS|3=AMD FSR 3.1|4=AMD FSR 4", Cs("dlss"), "upscalers: the engines, in the menu's order"); Equal("0=Auto (dynamic)|1=100% (2048x1152)|2=75% (1536x864)|3=67% (1366x768)|4=50% (1024x576)", Cs("resolution"), "render resolution, the game's own"); - Equal("0=Custom|1=Minimum (6GB)|2=Low (8GB)|3=Balanced (12GB)|4=High (16GB)|5=Ultra (20GB+)", Cs("quality_preset"), "presets"); Equal("0=Off|1=Default", Cs("microphone"), "microphone without devices"); Equal("0=Game SMAA|1=Off|2=FXAA|3=SMAA Ultra", Cs("anti-aliasing"), "anti-aliasing"); - Equal("0=Off|1=Auto|2=2x|3=3x|4=4x|5=5x|6=6x|7=7x|8=8x|9=AMD FSR 4", Cs("frame_generation"), "frame generation"); + Equal("0=Off|1=Auto|2=2x|3=3x|4=4x|5=5x|6=6x|7=7x|8=8x|9=AMD FSR|10=NVIDIA DLSS 2x|11=NVIDIA DLSS 3x|12=NVIDIA DLSS 4x", Cs(GameOptionsSchema.FrameGenKey), "frame generation, DLSS's merged in"); + Equal("0=Off|1=Expanded|2=Native|3=Native + RenoDX", Cs(GameOptionsSchema.HdrKey), "HDR, RenoDX merged in"); + Equal("0=Off|1=Even|2=Steady, any rate|3=Steady, refresh steps", Cs(GameOptionsSchema.PacingKey), "frame pacing, steady merged in"); Equal("0=Auto|1=Switch|2=Xbox|3=PlayStation 4|4=PlayStation 5|5=Steam Deck|6=Keyboard & mouse", Cs("button_prompts"), "prompts"); - Equal("0=Auto|1=Off|2=On", Cs("robust_buffers"), "robust"); Equal("0=Insert or F1|1=F1 only", Cs("menu_shortcut"), "menu shortcut"); + // (Low latency by the card's maker: NVIDIA's Reflex where it is not known) + var vendorBefore = GameOptionsSchema.LatencyVendorOf; + try + { + GameOptionsSchema.LatencyVendorOf = null; + Equal("0=Off|1=On|2=On + Boost", Cs(GameOptionsSchema.LatencyKey), "low latency, not known: Reflex's"); + Equal(2, GameOptionsSchema.Get(GameOptionsSchema.LatencyKey).Default, "Reflex's default On + Boost"); + GameOptionsSchema.LatencyVendorOf = () => LatencyVendor.Amd; + Equal("0=Off|1=On", Cs(GameOptionsSchema.LatencyKey), "AMD: Anti-Lag 2's"); + Equal(0, GameOptionsSchema.Get(GameOptionsSchema.LatencyKey).Default, "Anti-Lag 2's default Off"); + GameOptionsSchema.LatencyVendorOf = () => LatencyVendor.None; + Equal("0=Off", Cs(GameOptionsSchema.LatencyKey), "neither"); + } + finally { GameOptionsSchema.LatencyVendorOf = vendorBefore; } Equal("1.25", L("mouse_sensitivity", 125), "sensitivity"); Equal("0.51", L("walk_scale", 51), "walk scale"); Equal("10000 nits", L("hdr_peak_brightness", 10000), "nits"); @@ -212,12 +272,15 @@ static partial class Tests Equal("85%", L("master_volume", 85), "volume"); Equal("Off", L("menu_sounds", 0), "menu sounds 0"); Equal("Off", L("rumble_strength", 0), "rumble 0"); - Equal("As Stick dead zone", L("left_stick_dead_zone", -1), "own dead zone"); + Equal("Default (24%)", L("left_stick_dead_zone", -1), "own dead zone's default"); Equal("0%", L("right_stick_dead_zone", 0), "own dead zone 0"); Equal("50", L("renodx_bloom", 50), "grade"); - Equal("Default (24%)", L("stick_dead_zone", 0), "stick dead zone"); - Equal("Off", L("upscaling", 7), "past the row: its first"); + Equal("Normal", L("text_speed", 7), "past the row: its first"); Equal("16:9", L("ultrawide", 0), "ultrawide"); + Equal("1536x864 (handheld)", L("output_res", 7), "the handheld profile is an Output resolution"); + Equal("NVIDIA DLSS", L("dlss", 2), "DLAA reads as its engine"); + Equal("AMD FSR 3.1", L("dlss", 6), "FSR 3.1's native AA too"); + Equal("Native (100%)", L("resolution", GameOptionsSchema.NativeStep), "the Render resolution's Native"); // (the GPU device row: the names the launcher gives it) var before = GameOptionsSchema.GpuNames; @@ -259,11 +322,34 @@ static partial class Tests text = File.ReadAllText(Path.Combine(nf, GameOptionsSchema.FileName)); Check(text.Contains("gpu_device=0") && text.Contains("gpu_device_name=\r\n") || text.Contains("gpu_device_name=\n"), "Auto: the name emptied: " + text.Replace("\n", " | ")); + // ((Triforce) 11-22) a number saved before 11-20 has no name: the row's card chosen again (the same value) writes it, + // where nothing was written and the number alone decided; chosen again with its name there: nothing to write + GameOptionsSchema.GpuNames = () => new[] { "AMD Radeon(TM) Graphics", "AMD Radeon RX 6600" }; + var old = NewRoot("gpuold"); + File.WriteAllText(Path.Combine(old, GameOptionsSchema.FileName), "gpu_device=2\r\noptions_version=8\r\n"); + opt = GameOptionsFile.Load(old); + opt.Set(GameOptionsSchema.GpuKey, 2); + Check(opt.Changes.ContainsKey(GameOptionsSchema.GpuKey), "the same card again without its name: a change"); + opt.Save(); + text = File.ReadAllText(Path.Combine(old, GameOptionsSchema.FileName)); + Check(text.Contains("gpu_device=2\r\n") && text.Contains("gpu_device_name=AMD Radeon RX 6600\r\n"), + "the name written beside the old number: " + text.Replace("\r\n", " | ")); + opt = GameOptionsFile.Load(old); + opt.Set(GameOptionsSchema.GpuKey, 2); + Equal(0, opt.Changes.Count, "the same card again with its name: nothing to write"); + File.WriteAllText(Path.Combine(old, GameOptionsSchema.FileName), "gpu_device=2\r\ngpu_device_name=AMD Radeon(TM) Graphics\r\noptions_version=8\r\n"); + opt = GameOptionsFile.Load(old); + opt.Set(GameOptionsSchema.GpuKey, 2); + opt.Save(); + text = File.ReadAllText(Path.Combine(old, GameOptionsSchema.FileName)); + Check(text.Contains("gpu_device_name=AMD Radeon RX 6600\r\n") && !text.Contains("Radeon(TM)"), + "another card's name beside the number: the row's card's: " + text.Replace("\r\n", " | ")); + File.WriteAllText(Path.Combine(old, GameOptionsSchema.FileName), "gpu_device=0\r\noptions_version=8\r\n"); + opt = GameOptionsFile.Load(old); + opt.Set(GameOptionsSchema.GpuKey, 0); + Equal(0, opt.Changes.Count, "Auto again: nothing to write"); } finally { GameOptionsSchema.GpuNames = before; } - - Equal(6, GameOptionsSchema.PresetSummaries.Count, "a summary for each preset"); - Equal(0, GameOptionsSchema.PresetValues(0).Count, "Custom sets nothing"); } // ---- the real file ---- @@ -290,8 +376,15 @@ static partial class Tests if (r.Kind == GameOptionKind.Choice) Check(f.ChoicesOf(r.Key).Any(c => c.Value == v && c.Label == label), r.Name + ": " + v + " = \"" + label + "\" is one of its choices"); else - Check(Regex.IsMatch(label, @"^(-?\d+( nits|%)?|\d\.\d\d|Off|Muted|As Stick dead zone)$"), r.Name + ": a slider's label \"" + label + "\""); - Check(f.Raw(r.Key) == null || v == r.AsRead(GameOptionsFile.Atoi(f.Raw(r.Key)!)), r.Name + ": the file's value as the game reads it"); + Check(Regex.IsMatch(label, @"^(-?\d+( nits|%)?|\d\.\d\d|Off|Muted|Default \(24%\))$"), r.Name + ": a slider's label \"" + label + "\""); + Check(r.IsMerged || v == GameOptionsSchema.NativeStep || f.Raw(r.Key) == null || v == r.AsRead(GameOptionsFile.Atoi(f.Raw(r.Key)!)), + r.Name + ": the file's value as the game reads it"); + } + // (the merged rows' parts: each its line's value, in its range; the game writes every one) + foreach (var p in GameOptionsSchema.Parts.Values) + { + Check(f.Raw(p.Key) != null, p.Key + ": the game wrote its line"); + Equal(p.AsRead(GameOptionsFile.Atoi(f.Raw(p.Key)!)), f.Get(p.Key), p.Key + ": as the game reads it"); } info.Refresh(); Equal(len, info.Length, "the real file is not written"); @@ -357,31 +450,52 @@ static partial class Tests Equal("DCI-P3", Lb("renodx_color_space"), "colour space"); Equal("Eye adaptation", Lb("renodx_midgray"), "midgray"); Equal("Transformer 2 (M)", Lb("dlss_model"), "dlss model"); - Equal("Lanczos", Lb("upscaling"), "filter"); Equal("50%", Lb("sharpness"), "sharpness"); - Equal("Off", Lb("robust_buffers"), "robust 1"); Equal("Default", Lb("microphone"), "microphone"); Equal("Full", Lb("texture_detail"), "texture detail"); - Equal("Off", Lb("nvidia_reflex"), "reflex 0"); + Equal("Off", Lb(GameOptionsSchema.LatencyKey), "low latency: reflex 0"); + Equal("Even", Lb(GameOptionsSchema.PacingKey), "frame pacing on, no steady"); + Equal("Off", Lb(GameOptionsSchema.HdrKey), "hdr 0"); + Equal("Off", Lb(GameOptionsSchema.FrameGenKey), "no frame generation"); Equal("Per frame", GameOptionsSchema.Get("game_speed").Label(0), "game speed 0"); Equal("Real time", Lb("game_speed"), "game speed: its default without a line"); Equal("80%", Lb("menu_sounds"), "menu sounds' default"); Equal("Middle", Lb("ultrawide_hud"), "wide HUD"); Equal("Centred, blurred sides", Lb("ultrawide_menus"), "wide menus"); - Equal("layout_preset,debug_tools,replay_helper_threads,tail_early,options_version,menu_size,pad_a,key_a,key_cam_right", + Equal("Default (24%)", Lb("left_stick_dead_zone"), "a stick's own dead zone: its default"); + // (a v0.5.1 file, options_version 7: the rows that went are other lines, kept for the game to drop) + Equal("quality_preset,upscaling,robust_buffers,layout_preset,debug_tools,replay_helper_threads,tail_early,options_version,menu_size,pad_a,key_a,key_cam_right", string.Join(",", fx.OtherLines.Select(o => o.Key)), "other lines in the file's order"); + Equal("", string.Join(",", fx.UnknownKeys), "each one known"); Equal("69,0", fx.OtherLines.Single(o => o.Key == "key_a").Value, "a binding's value as it is"); + // ((Triforce, options_version 8) the Upscaler's engines and the Render resolution's Native (upscaler_rows.h step_engine): + // the game's own at 100% is the 3D at full size, so DLSS from it is DLSS's native AA, DLAA) fx.Set("dlss", 1); - Equal("Quality (67%)", fx.Label("resolution"), "with DLSS: its mode (100% moved to Quality)"); + Equal(2, fx.Get("dlss"), "DLSS from the game's own at 100%: DLAA"); + Equal("NVIDIA DLSS", fx.Label("dlss"), "shown as its engine"); + Equal("Native (100%)", fx.Label("resolution"), "at Native"); + Equal(GameOptionsSchema.NativeStep, fx.Get("resolution"), "the row's Native step"); Equal("DLSS's own", fx.Label("anti-aliasing"), "aa with DLSS"); - Equal("0,3,5,4,6", string.Join(",", fx.ChoicesOf("resolution").Select(c => c.Value)), "DLSS's steps"); - fx.Set("dlss", 2); - Equal("Native (DLAA)", fx.Label("resolution"), "DLAA"); + Equal("0,-1,3,5,4,6", string.Join(",", fx.ChoicesOf("resolution").Select(c => c.Value)), "DLSS's steps, Native among them"); + Equal("0,2,3,4", string.Join(",", fx.ChoicesOf("dlss").Select(c => c.Value)), "the engines, DLSS by its saved value"); + fx.Set("resolution", 3); + Equal(1, fx.Get("dlss"), "a mode: DLSS again"); + Equal("Quality (67%)", fx.Label("resolution"), "DLSS's Quality"); + fx.Set("resolution", GameOptionsSchema.NativeStep); + Equal(2, fx.Get("dlss"), "Native: DLAA"); + Equal(3, fx.Changes["resolution"], "the saved resolution left as it was, for the other steps"); fx.Set("dlss", 3); + Equal(6, fx.Get("dlss"), "FSR 3.1 from DLAA: FSR 3.1's native AA"); Equal("FSR's own", fx.Label("anti-aliasing"), "aa with FSR"); fx.Set("dlss", 0); - fx.Set("handheld_profile", 1); - Equal("67% (1024x576)", fx.Label("resolution"), "handheld profile's size (100% went to DLSS's Quality, which stays with the game's own)"); + Equal(0, fx.Get("dlss"), "the game's own from a native AA"); + Equal("100% (2560x1440)", fx.Label("resolution"), "at 100%"); + fx.Set("output_res", GameOptionsSchema.HandheldOutput); + Equal("100% (1536x864)", fx.Label("resolution"), "the handheld profile's size"); + fx.Set("resolution", 4); + fx.Set("dlss", 4); + Equal(4, fx.Get("dlss"), "FSR 4 from 50%: the engine"); + Equal("Performance (50%)", fx.Label("resolution"), "its mode"); // (the merge: a line changed outside since Load stays; the launcher's own change goes in; an unknown line stays in place) var m = NewGameFolder("game-options-merge", OptionsFixture); @@ -396,7 +510,8 @@ static partial class Tests Check(merged.Contains("\r\nhdr=2\r\nzz_tool_line=5\r\n"), "the outside change and its new line kept in place"); Check(merged.StartsWith("fullscreen=1\r\n"), "the launcher's change in its line"); Check(merged.Contains("\r\nsharpness=70\r\n"), "a row changed on both sides: the launcher's (the later writer)"); - Equal(outside.Replace("fullscreen=0", "fullscreen=1").Replace("sharpness=20", "sharpness=70"), merged, "every other line as the file had it"); + Equal(outside.Replace("fullscreen=0", "fullscreen=1").Replace("sharpness=20", "sharpness=70").Replace("options_version=7", "options_version=8"), merged, + "every other line as the file had it (the version 7 file brought up to 8: nothing of it moves)"); Equal("zz_tool_line", a.UnknownKeys.Single(), "an unknown key, kept"); Equal(2, a.Get("hdr"), "this object holds the file as written"); @@ -407,7 +522,7 @@ static partial class Tests df.Set("sharpness", 30); df.Set("vsync", 0); df.Save(); - Equal("sharpness=30\nsharpness=30\n#note\nvsync=0\noptions_version=7\nmenu_size=1\n", File.ReadAllText(Path.Combine(d, GameOptionsSchema.FileName)), "LF kept, line added, comment kept"); + Equal("sharpness=30\nsharpness=30\n#note\nvsync=0\noptions_version=8\nmenu_size=1\n", File.ReadAllText(Path.Combine(d, GameOptionsSchema.FileName)), "LF kept, line added, comment kept, brought up to 8"); // (a device's name in UTF-8: its bytes as they were, its label as text) var mic = "microphone_device=Mikrofon (Ä USB)"; @@ -445,53 +560,82 @@ static partial class Tests Equal(130, f.Set("mouse_sensitivity", 125), "onto the steps: the nearest, the larger on a tie"); Equal(50, f.Set("rumble_strength", 60), "rumble on 25s"); Equal(0, f.Set("output_res", 9), "an output past the list: the first"); + Equal(7, f.Set("output_res", 7), "the handheld profile's output"); Equal(-1, f.Set("left_stick_dead_zone", -7), "a range below 0"); Equal(-3, f.Set("frame_rate_cap", -3), "the cap's 30"); - // (the preset sets its rows; a row not exempt changed by hand makes it Custom, an exempt one does not) - f.Set("quality_preset", 4); - foreach (var (k, v) in new Dictionary { ["resolution"] = 1, ["upscaling"] = 2, ["sharpness"] = 40, ["frame_pacing"] = 1, ["game_speed"] = 1, ["handheld_profile"] = 0, ["texture_detail"] = 1 }) - Equal(v, f.Get(k), "High sets " + k); - Equal("High (16GB)", f.Label("quality_preset"), "the preset's label"); - f.Set("fullscreen", 1); - Equal(4, f.Get("quality_preset"), "an exempt row leaves the preset"); - f.Set("sharpness", 40); - Equal(4, f.Get("quality_preset"), "the same value is no change"); - f.Set("sharpness", 60); - Equal(0, f.Get("quality_preset"), "a scaling row by hand: Custom"); - // (with DLSS the preset's resolution is its nearest mode) - f.Set("dlss", 1); - f.Set("quality_preset", 2); - Equal(3, f.Get("resolution"), "Low's 75% with DLSS: Quality (67%)"); - Equal(2, f.Get("texture_detail"), "Low: Reduced textures"); - f.Set("quality_preset", 1); - Equal(4, f.Get("resolution"), "Minimum's 50% is DLSS's Performance"); - Equal(3, f.Get("texture_detail"), "Minimum: Low textures"); - Equal(1, f.Get("handheld_profile"), "Minimum: handheld profile"); - // (another Upscaler moves the resolution onto its steps) - f.Set("dlss", 0); - f.Set("resolution", 2); - f.Set("dlss", 3); - Equal(3, f.Get("resolution"), "75% with FSR 3.1: 67%"); - Equal(0, f.Get("quality_preset"), "the Upscaler is not exempt"); + // ((Triforce, options_version 8) a merged row sets its parts (merged_rows.h *_set): one of a pair chosen, the other Off) + string Parts(string key) => string.Join(",", GameOptionsSchema.Get(key).Parts!.Select(p => p.Key + "=" + f.Get(p.Key))); + f.Set(GameOptionsSchema.FrameGenKey, 11); + Equal("frame_generation=0,dlss_frame_generation=2", Parts(GameOptionsSchema.FrameGenKey), "NVIDIA DLSS 3x"); + f.Set(GameOptionsSchema.FrameGenKey, 4); + Equal("frame_generation=4,dlss_frame_generation=0", Parts(GameOptionsSchema.FrameGenKey), "4x, DLSS's off"); + Equal(12, f.Set(GameOptionsSchema.FrameGenKey, 40), "past the end: DLSS 4x"); + f.Set(GameOptionsSchema.HdrKey, 3); + Equal("hdr=2,renodx_hdr=1", Parts(GameOptionsSchema.HdrKey), "Native + RenoDX"); + f.Set(GameOptionsSchema.HdrKey, 1); + Equal("hdr=1,renodx_hdr=0", Parts(GameOptionsSchema.HdrKey), "Expanded, RenoDX off"); + f.Set(GameOptionsSchema.PacingKey, 3); + Equal("frame_pacing=1,steady_frame_rate=2", Parts(GameOptionsSchema.PacingKey), "Steady, refresh steps: pacing on"); + f.Set(GameOptionsSchema.PacingKey, 0); + Equal("frame_pacing=0,steady_frame_rate=0", Parts(GameOptionsSchema.PacingKey), "Off"); + var vendorBefore = GameOptionsSchema.LatencyVendorOf; + try + { + GameOptionsSchema.LatencyVendorOf = () => LatencyVendor.Nvidia; + f.Set(GameOptionsSchema.LatencyKey, 1); + Equal("nvidia_reflex=1,amd_anti-lag_2=0", Parts(GameOptionsSchema.LatencyKey), "NVIDIA: Reflex On"); + GameOptionsSchema.LatencyVendorOf = () => LatencyVendor.Amd; + Equal(0, f.Get(GameOptionsSchema.LatencyKey), "AMD: Anti-Lag 2's Off (Reflex's line is NVIDIA's)"); + Equal(1, f.Set(GameOptionsSchema.LatencyKey, 2), "AMD: On, its last"); + Equal("nvidia_reflex=1,amd_anti-lag_2=1", Parts(GameOptionsSchema.LatencyKey), "Anti-Lag 2 on, Reflex as it was"); + GameOptionsSchema.LatencyVendorOf = () => LatencyVendor.None; + f.Set(GameOptionsSchema.LatencyKey, 1); + Equal("nvidia_reflex=1,amd_anti-lag_2=1", Parts(GameOptionsSchema.LatencyKey), "neither: nothing changes"); + } + finally { GameOptionsSchema.LatencyVendorOf = vendorBefore; } + Equal(2, f.Set("steady_frame_rate", 5), "a part by its own key: in its range"); + Equal(3, f.Get(GameOptionsSchema.PacingKey), "and the row shows it"); + // (a pair the old launcher could save both of: the row as the game shows it) + var both = GameOptionsFile.Load(NewGameFolder("game-options-both", "frame_generation=3\ndlss_frame_generation=1\nhdr=1\nrenodx_hdr=1\nframe_pacing=0\nsteady_frame_rate=1\noptions_version=8\n")); + Equal("NVIDIA DLSS 2x", both.Label(GameOptionsSchema.FrameGenKey), "DLSS's frame generation over the renderer's"); + Equal("Expanded", both.Label(GameOptionsSchema.HdrKey), "RenoDX only with Native"); + Equal("Steady, any rate", both.Label(GameOptionsSchema.PacingKey), "steady over pacing off"); - // (Apply: the preset first, then the rest by the same rules) + // (the Upscaler: an engine; the Render resolution's Native goes with it, upscaler_rows.h step_engine) + var u = GameOptionsFile.Load(NewGameFolder("game-options-up", OptionsFixture)); // (dlss=0, resolution=1: 100%) + u.Set("dlss", 3); + Equal(6, u.Get("dlss"), "FSR 3.1 from 100%: its native AA"); + u.Set("resolution", 4); + Equal(3, u.Get("dlss"), "a mode: FSR 3.1"); + u.Set("dlss", 1); + Equal(1, u.Get("dlss"), "DLSS from 50%: DLSS"); + Equal(4, u.Get("resolution"), "50% is DLSS's Performance"); + u.Set("dlss", 0); + u.Set("resolution", 2); + u.Set("dlss", 3); + Equal(3, u.Get("resolution"), "75% with FSR 3.1: 67%"); + u.Set("dlss", 0); + u.Set("resolution", GameOptionsSchema.NativeStep); + Equal(1, u.Get("resolution"), "the game's own at Native: 100%"); + Equal(2, u.Set("dlss", 2), "DLAA by its saved value: as it is"); + + // (Apply: each with Set's rules) var b = GameOptionsFile.Load(NewGameFolder("game-options-apply", OptionsFixture)); - var got = b.Apply(new Dictionary { ["frame_generation"] = 2, ["quality_preset"] = 3, ["sharpness"] = 30 }); - Equal(3, got["quality_preset"], "a preset row at the preset's own value keeps the preset"); - Equal(2, got["frame_generation"], "frame generation set"); - Equal(1, b.Get("upscaling"), "Balanced: Catmull-Rom"); - var c = b.Apply(new Dictionary { ["quality_preset"] = 5, ["sharpness"] = 80 }); - Equal(0, c["quality_preset"], "a preset row changed after it: Custom, as in the game"); - Equal(80, c["sharpness"], "and its value"); + var got = b.Apply(new Dictionary { [GameOptionsSchema.FrameGenKey] = 2, ["steady_frame_rate"] = 2, ["sharpness"] = 30 }); + Equal(2, got[GameOptionsSchema.FrameGenKey], "frame generation set"); + Equal(2, got["steady_frame_rate"], "a part by its key"); + Equal(30, got["sharpness"], "sharpness"); var changes = b.Changes.Count; Throws(() => b.Apply(new Dictionary { ["vsync"] = 0, ["descriptor_reuse"] = 0 }), "a key that is no row"); + Throws(() => b.Apply(new Dictionary { ["quality_preset"] = 3 }), "the Quality preset is gone"); Equal(changes, b.Changes.Count, "nothing set then"); Equal(1, b.Get("vsync"), "vsync untouched"); b.Save(); var back = GameOptionsFile.Load(Path.GetDirectoryName(b.Path)!); - Equal(80, back.Get("sharpness"), "saved"); + Equal(30, back.Get("sharpness"), "saved"); Equal(2, back.Get("frame_generation"), "saved"); + Equal("Steady, refresh steps", back.Label(GameOptionsSchema.PacingKey), "saved"); Equal(0, back.Changes.Count, "a fresh read has no changes"); } @@ -507,15 +651,37 @@ static partial class Tests Equal(0, GameOptionsFile.Load(g).Get("ultrawide"), "without the screen: 16:9, as the game where it cannot ask"); f.Set("sharpness", 30); f.Save(); - Equal("frame_pacing=1\r\nultrawide=2\r\nnvidia_reflex=0\r\nsharpness=30\r\noptions_version=7\r\npad_a=0\r\n", - File.ReadAllText(Path.Combine(g, GameOptionsSchema.FileName)).Replace("tail_early=1\r\n", ""), "brought up to 7 as the game would read it"); - Check(File.ReadAllText(Path.Combine(g, GameOptionsSchema.FileName)).Contains("tail_early=1\r\noptions_version=7"), "the Debug tab's tail_early too, before the version"); + Equal("frame_pacing=1\r\nultrawide=2\r\nnvidia_reflex=0\r\nsharpness=30\r\noptions_version=8\r\npad_a=0\r\n", + File.ReadAllText(Path.Combine(g, GameOptionsSchema.FileName)).Replace("tail_early=1\r\n", ""), "brought up to 8 as the game would read it"); + Check(File.ReadAllText(Path.Combine(g, GameOptionsSchema.FileName)).Contains("tail_early=1\r\noptions_version=8"), "the Debug tab's tail_early too, before the version"); var again = GameOptionsFile.Load(g); - Equal(7, again.Version, "now 7"); - again.Set("frame_pacing", 0); + Equal(8, again.Version, "now 8"); + again.Set(GameOptionsSchema.PacingKey, 0); again.Save(); Equal(0, GameOptionsFile.Load(g).Get("frame_pacing"), "and from 7 on the choice is read as it is"); + // ((Triforce, options_version 8) a v0.5.1 file (7, the old rows' lines in it): the handheld profile becomes the Output + // resolution's 1536x864, the shared stick dead zone each stick's own that had none (merged_rows.h migrate) + var h = NewGameFolder("game-options-v7", "output_res=1\nhandheld_profile=1\nstick_dead_zone=3\nright_stick_dead_zone=12\nquality_preset=4\nupscaling=2\noptions_version=7\n"); + var hf = GameOptionsFile.Load(h); + Equal(7, hf.Get("output_res"), "the handheld profile: the handheld output"); + Equal("1536x864 (handheld)", hf.Label("output_res"), "its label"); + Equal(15, hf.Get("left_stick_dead_zone"), "the shared 15% into the left stick's own"); + Equal(12, hf.Get("right_stick_dead_zone"), "the right stick's own kept"); + Equal("", string.Join(",", hf.UnknownKeys), "the old rows' lines: known"); + hf.Set("vsync", 0); + hf.Save(); + Equal("output_res=7\nhandheld_profile=1\nstick_dead_zone=3\nright_stick_dead_zone=12\nquality_preset=4\nupscaling=2\nvsync=0\nleft_stick_dead_zone=15\noptions_version=8\n", + File.ReadAllText(Path.Combine(h, GameOptionsSchema.FileName)), "brought up to 8: the moved values written, the old lines left for the game to drop"); + var h8 = GameOptionsFile.Load(h); + h8.Set("output_res", 1); + h8.Save(); + Equal(1, GameOptionsFile.Load(h).Get("output_res"), "from 8 on the old handheld_profile line moves nothing"); + // (a 436f9ac-based build's 7: none of those lines, nothing moves) + var n7 = GameOptionsFile.Load(NewGameFolder("game-options-n7", "output_res=3\nleft_stick_dead_zone=-1\noptions_version=7\n")); + Equal(3, n7.Get("output_res"), "no handheld line: the output as it is"); + Equal(-1, n7.Get("left_stick_dead_zone"), "no shared line: the default"); + // (no options_version line: Game speed real time; no resolution line but the old dynamic_resolution) var o = NewGameFolder("game-options-v0", "game_speed=0\ndynamic_resolution=0\n"); var of = GameOptionsFile.Load(o); @@ -523,20 +689,23 @@ static partial class Tests Equal(1, of.Get("resolution"), "dynamic_resolution 0: 100%"); Equal(2, of.Get("nvidia_reflex"), "Reflex On + Boost before 5"); - // (no file: the defaults; a Save makes it with the changed rows and the version) + // (no file: the defaults; a Save makes it with the changed values, in the menu's order, and the version) var n = Path.Combine(NewRoot("game-options-new"), "game"); Directory.CreateDirectory(n); var nf = GameOptionsFile.Load(n); Check(!nf.Exists, "no file"); Equal(1, nf.Get("vsync"), "default"); Equal("Off", nf.Label("fullscreen"), "default label"); + Equal("Even", nf.Label(GameOptionsSchema.PacingKey), "frame pacing's default"); nf.Save(); Check(!File.Exists(nf.Path), "nothing to write: no file made"); - nf.Set("vsync", 0); nf.Set("master_volume", 83); + nf.Set("vsync", 0); + nf.Set(GameOptionsSchema.PacingKey, 3); nf.Save(); var nl = OperatingSystem.IsWindows() ? "\r\n" : "\n"; - Equal("vsync=0" + nl + "master_volume=85" + nl + "options_version=7" + nl, File.ReadAllText(nf.Path), "a new file: the rows in the menu's order, then the version"); + Equal("vsync=0" + nl + "steady_frame_rate=2" + nl + "master_volume=85" + nl + "options_version=8" + nl, File.ReadAllText(nf.Path), + "a new file: the values in the menu's order (a merged row's parts at its place; frame_pacing at its default not written), then the version"); Check(nf.Exists, "now there"); } } diff --git a/launcher/LogicTests/Tests.Release.cs b/launcher/LogicTests/Tests.Release.cs index 96fec58..d4aadb4 100644 --- a/launcher/LogicTests/Tests.Release.cs +++ b/launcher/LogicTests/Tests.Release.cs @@ -276,6 +276,24 @@ static partial class Tests Equal("library", File.ReadAllText(Path.Combine(tools.Folder, "python", "lib.txt")), "the base moved over"); File.WriteAllText(Path.Combine(tools.Folder, "python", "lib.txt"), "changed!"); Check(!Tools.HasBase(b, tools.Folder, null), "a changed base file (other size) is not the base"); + + // (11-21, ticket "Every tools update re-downloads the whole base": Python rewrites its bytecode caches as it runs, and + // their sizes in base.list made every update fetch the base again) + var root2 = NewRoot("parts-pyc"); + var t2 = new Tools(Ctx(root2)); + var list2 = BaseList(("python/python.exe", "py"), ("python/Lib/__pycache__/os.cpython-314.pyc", "packed bytes")); + var b2 = Tools.ParseBaseList(list2)!; + Directory.CreateDirectory(Path.Combine(t2.Folder, "python", "Lib", "__pycache__")); + File.WriteAllText(Path.Combine(t2.Folder, "python", "python.exe"), "py"); + File.WriteAllText(Path.Combine(t2.Folder, "python", "Lib", "__pycache__", "os.cpython-314.pyc"), "rewritten by python, longer"); + Check(Tools.HasBase(b2, t2.Folder, null), "a bytecode cache Python rewrote is still the base"); + File.Delete(Path.Combine(t2.Folder, "python", "Lib", "__pycache__", "os.cpython-314.pyc")); + File.Delete(Path.Combine(t2.Folder, Tools.BaseHave)); + Check(Tools.HasBase(b2, t2.Folder, null), "and one missing (Python makes it again)"); + File.WriteAllText(Path.Combine(t2.Folder, "python", "python.exe"), "other"); + File.Delete(Path.Combine(t2.Folder, Tools.BaseHave)); + Check(!Tools.HasBase(b2, t2.Folder, null), "a real base file changed still is not"); + Check(Tools.Regenerated(@"python\Lib\x\__pycache__\a.pyc") && !Tools.Regenerated("python/Lib/os.py"), "what counts as rewritten"); } static async Task ToolsUpdate() diff --git a/nx2/build.py b/nx2/build.py index 38c17b0..5fe16ce 100644 --- a/nx2/build.py +++ b/nx2/build.py @@ -39,6 +39,7 @@ import time sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # (python -I leaves the script folder off) import kit +import cold_parts import lifted_c import rtgen import logo_rgba @@ -58,6 +59,26 @@ HOST_ZSTD_SOURCES = ("common/debug.c", "common/entropy_common.c", "common/error_ OBJ_CACHE = os.path.join(kit.WORK, "objcache") # (hookshot) work/hot_order.txt: the functions that run, hottest first (sampled), packed together by the linker; RX_NO_ORDER=1: off PGO = os.environ.get("RX_PGO", "") +# ((Triforce, 11-21, the performance audit's LIFT-10, after Source A's) RX_VK_PGO: the renderer's own code (gpu/vk/*.cpp, not +# the vendored recompiler, ImGui or zstd) with a play profile. The back end runs the replay on the game's render thread, and +# was built at plain -O2 whatever it does most. "gen": it counts its branches (host/pgo_rt.c in the DLL, RX_VK_PGO_FILE= +# in the run: .reset clears the counters, .dump writes them to ; llvm-profdata merge makes the profile). +# : built with that profile. Not set, empty or 0: neither, as before (nx2/make_game.py passes a shipped profile). +VK_PGO = os.environ.get("RX_VK_PGO", "") + + +def vk_pgo_flags(asked=None): + """The flags RX_VK_PGO adds to the renderer's own files. asked: its value (the environment's when not given)""" + asked = VK_PGO if asked is None else asked + if asked in ("", "0"): + return [] + if asked == "gen": + return ["-fprofile-instr-generate", "-mllvm", "-disable-vp=true"] + # (the profile's contents in the flags the object key hashes, as the hot code's: its path names no key, flag_text) + with open(asked, "rb") as f: + sha = hashlib.sha1(f.read(), usedforsecurity=False).hexdigest()[:16] + return ["-fprofile-instr-use=" + asked, "-DTRIFORCE_VK_PROFILE_SHA1=" + sha, "-Wno-profile-instr-out-of-date", + "-Wno-profile-instr-unprofiled"] # (linux) Linux builds: an ELF executable and triforce_vk.so, the Win32 calls of host/ and gpu/vk on POSIX @@ -71,6 +92,18 @@ EXE = "triforce" if LINUX else "Triforce.exe" VK_LIB = "triforce_vk.so" if LINUX else "triforce_vk.dll" +def repro_link(asked=None, linux=None): + """(Triforce, 11-22, the performance audit's IO-45, after Source A's) What HOOKSHOT_LINK_REPRO=1 adds to the Windows + links of the game's exe and of the renderer: /Brepro, with which lld writes a hash of the file where it otherwise + writes the time of the link, so that two links of the same objects are the same bytes and make_game.py's finish can + pass the unchanged exe over (INSTALLED_NOTE there). Off unless asked: it changes the bytes of every linked file once. + asked: the switch's value (the environment's when not given)""" + asked = os.environ.get("HOOKSHOT_LINK_REPRO", "") if asked is None else asked + if asked != "1" or (LINUX if linux is None else linux): + return [] + return [toolchain.link_opt("/Brepro")] + + def check_vk_symbols(so): """The back end is dlopened with RTLD_NOW, so one unresolved symbol stops the whole thing loading -- and the only sign of that at run time is "the renderer did not start: the window @@ -666,7 +699,8 @@ def mem_need(src): with open(src, "rb") as f: text = f.read() # (each function from the newline before its definition, as this was first measured) - starts = [m.start() - 1 for m in lifted_c.HEAD_BYTES.finditer(text)] + [len(text)] + # ((Triforce, 11-22, the performance audit's IO-40) lifted_c.definitions: HEAD_BYTES's matches, found by jumps) + starts = [m.start() - 1 for m in lifted_c.definitions(text)] + [len(text)] sizes = [b - a for a, b in zip(starts, starts[1:])] giant = GIANT_MB * sum((s / (1 << 20)) ** 2 for s in sizes if s >= GIANT_BYTES) if giants_counted() else 0 if name.startswith("code_hot"): @@ -676,6 +710,11 @@ def mem_need(src): return 1000 if src.endswith(".cpp") else 400 +class CompileStopped(Exception): + """((Triforce, 11-21, the performance audit's LIFT-31) The build is stopping: a compile that waited at the memory gate + is not started (MemoryGate.stop)""" + + class MemoryGate: """(issue 418) Compilers started by the memory there is, not only by --jobs: one per CPU thread ran PCs out of memory (the kernel stopped a compiler, or the whole build), and a count low enough for the biggest files and the PC's other programs left most of @@ -684,10 +723,15 @@ class MemoryGate: on Windows up to 4 (half the threads on smaller CPUs) always run, as Windows pages rather than fails (issues 502, 503, 511: 1-2 compilers on 16 GB PCs took hours); on Linux one (issue 531: Linux stops a compiler that does not fit, and it is compiled again alone). The free memory is read again as compiles start and end, so a game or browser opened during the build makes later - compiles wait""" + compiles wait + + (Triforce, 11-21, the performance audit's LIFT-31, after Source A's) stop(): the build is ending on an exception. The + pool's workers, one a CPU thread, wait here, and a compile that has left the pool's queue cannot be cancelled there: + one that waits, or comes after, raises CompileStopped and takes no lease""" def __init__(self): self.cv = threading.Condition() + self.stopped = False self.running = {} # (per-compile lease: [need, Popen or None]; one source can have several objects) self.reserve = float(os.environ.get("HOOKSHOT_MEM_RESERVE_GB") or 1.5) * 1024 self.floor = max(1, min(4, (os.cpu_count() or 2) // 2)) if os.name == "nt" else 1 @@ -695,7 +739,7 @@ class MemoryGate: def enter(self, src): need = mem_need(src) with self.cv: - while len(self.running) >= self.floor: + while not self.stopped and len(self.running) >= self.floor: free = free_mb() if free is None: break @@ -703,10 +747,17 @@ class MemoryGate: if free - owed - self.reserve >= need: break self.cv.wait(1.0) + if self.stopped: + raise CompileStopped(src) lease = object() self.running[lease] = [need, None] return lease + def stop(self): + with self.cv: + self.stopped = True + self.cv.notify_all() + def started(self, lease, proc): with self.cv: self.running[lease][1] = proc @@ -830,6 +881,10 @@ def flag_text(fl): continue if f.startswith("--sysroot="): f = "--sysroot=" + elif VK_PGO not in ("", "0", "gen") and f == "-fprofile-instr-use=" + VK_PGO: + # ((Triforce, 11-21, LIFT-10) the renderer's profile by its contents (TRIFORCE_VK_PROFILE_SHA1, beside it), not by + # where it is: another work folder's copy of the same profile is no new key) + f = "-fprofile-instr-use=" else: f = f.replace(kit.ROOT, "") kept.append(f) @@ -1075,6 +1130,14 @@ def main(): srcs = [] if a.dry_run: # (the lift's files: there is no game here) srcs = [s for s in srcs if os.path.dirname(s) != gen] + # ((Triforce, 11-22, the performance audit's LIFT-17, after Source A's) RX_COLD_OPT=O2, or a file that lists chunks: those + # lifted chunks at -O2 too (never the hot pieces), each compiled as parts of RX_COLD_PART_MB under /split in place + # of the chunk (nx2/cold_parts.py). Unset: srcs as they are, and no file takes cold_fl below + cold_stats = {} + srcs, cold = cold_parts.expand(srcs, os.path.join(out, "split"), os.environ, stats=cold_stats) + if cold: + print("RX_COLD_OPT: %d files at -O2 (parts of %d MB; %d chunks cut again, %d kept as cut before)" % ( + len(cold), cold_parts.part_bytes(os.environ) >> 20, cold_stats.get("written", 0), cold_stats.get("kept", 0)), flush=True) # the longest first, so the long ones do not end up alone at the end (compile_cost) costs = chunk_costs(gen) srcs.sort(key=lambda s: -compile_cost(s, costs)) @@ -1099,8 +1162,9 @@ def main(): # new profile kept the hot code built from the old one) with open(PGO, "rb") as f: hot_fl = hot_fl + ["-fprofile-instr-use=" + PGO, "-DHOOKSHOT_PROFILE_SHA1=" + hashlib.sha1(f.read(), usedforsecurity=False).hexdigest()[:16]] + cold_fl = cold_parts.cold_flags(fl, a.opt, os.environ) if cold else fl # ((Triforce, LIFT-17) -O2 and the part size) jobs = [(cc, s, os.path.join(obj_dir, os.path.basename(s)[:-2] + ".o"), - hot_fl if os.path.basename(s).startswith("code_hot") else fl, + hot_fl if os.path.basename(s).startswith("code_hot") else cold_fl if s in cold else fl, shim_deps if os.path.dirname(s) == NX2_RT else deps) for s in srcs] # (issue 624) what the game code's object keys are made of, as short fingerprints: two builds' logs side by side say which part # changed when an update compiled everything again (the headers, the flags, the profile, or the lifted code itself) @@ -1213,7 +1277,10 @@ def main(): # (what the code takes for granted, which newer C++ libraries bring in by themselves and older ones do not: issue 48) "-include", "algorithm", "-include", "cstdio", "-include", "cstdlib", "-include", "cstring", "-include", "functional", "-include", "string", "-include", "vector", - "-std=c++20", "-O2", "-g0", "-DNDEBUG", "-Wno-everything", "-DVK_NO_PROTOTYPES", "-DMKWVK_BUILDING", + # ((Triforce, 11-21) HOOKSHOT_VK_LINES=1: a line table in the renderer (the same code, DWARF lines only) for a profile's + # exact functions, inlined ones too, through llvm-symbolizer: the linker map lacks the file-local functions) + "-std=c++20", "-O2", "-gline-tables-only" if os.environ.get("HOOKSHOT_VK_LINES") == "1" else "-g0", "-DNDEBUG", + "-Wno-everything", "-DVK_NO_PROTOTYPES", "-DMKWVK_BUILDING", *(["-DHOOKSHOT_RELEASE"] if os.environ.get("RX_RELEASE") == "1" else []), # (the shaders precompiled before the game) # (Eden's recompiler before suyu's root: suyu has its own shader_recompiler/ headers, and a Shader::Info # of the other layout in some objects is what crashed the descriptor walk. MKWVK_INCLUDES changes the @@ -1285,11 +1352,16 @@ def main(): # (the bundled fonts, compiled in: nx2/ui_fonts.py's file of the build's own) vk_jobs.append((cxx, fonts_cpp, os.path.join(obj_dir, "vk_ui_fonts.o"), target() + (["-fPIC"] if LINUX else []) + ["-O0", "-g0", "-w"], b"")) + vk_pgo = vk_pgo_flags() # ((Triforce, 11-21, LIFT-10) the renderer's own files alone) + if VK_PGO == "gen" and not LINUX: # (what writes the counters, the DLL's own copy: host/pgo_rt.c) + vk_jobs.append((cc, os.path.join(HOST, "pgo_rt.c"), os.path.join(obj_dir, "vk_pgo_rt.o"), + target() + ["-O2", "-g0", '-DPGO_ENV="RX_VK_PGO_FILE"'], b"")) for src in vk_sources(): name = os.path.relpath(src, vk).replace(os.sep, "_")[:-4] lib = os.path.relpath(src, vk).split(os.sep)[0] in ("eden", "sirit") vk_jobs.append((cxx, src, os.path.join(obj_dir, "vk_" + name + ".o"), - vflags + ([shader_stamp, translation_def] if os.path.basename(src) == "vk_shader.cpp" else []) + + vflags + ([] if lib else vk_pgo) + + ([shader_stamp, translation_def] if os.path.basename(src) == "vk_shader.cpp" else []) + ([stamp_def] if os.path.basename(src) == "vk_menu.cpp" else []) + (ngx_flags if os.path.basename(src) == "vk_dlss.cpp" else []) + (sl_flags if os.path.basename(src) == "vk_dlssg.cpp" else []) + @@ -1351,21 +1423,33 @@ def main(): with concurrent.futures.ThreadPoolExecutor(a.jobs) as ex: # (as each ends, not in order: the builder app shows progress from these lines, and the biggest file comes first) submitted = {ex.submit(compile_one, job): job for job in jobs} - for future in concurrent.futures.as_completed(submitted): - src, secs, err = future.result() - size = os.path.getsize(src) >> 20 - if err: - failed.append(submitted[future]) - print("not compiled yet: %s (%d MB, %.0fs), tried again alone at the end:\n%s" - % (os.path.basename(src), size, secs, err[-1500:]), flush=True) - else: - print(" %-16s %4d MB %6.0fs (%.0fs)" % (os.path.basename(src), size, secs, time.time() - t0), flush=True) - if counted: - print("@@file %s %.1f %s" % (os.path.basename(src), secs, - "failed" if err else "cached" if secs <= 0 else "new"), flush=True) - if not err and expected.get(src): - pending.remove(expected[src].pop()) - print("@@eta %d" % eta_left(pending, eta_jobs), flush=True) + try: + for future in concurrent.futures.as_completed(submitted): + src, secs, err = future.result() + size = os.path.getsize(src) >> 20 + if err: + failed.append(submitted[future]) + print("not compiled yet: %s (%d MB, %.0fs), tried again alone at the end:\n%s" + % (os.path.basename(src), size, secs, err[-1500:]), flush=True) + else: + print(" %-16s %4d MB %6.0fs (%.0fs)" % (os.path.basename(src), size, secs, time.time() - t0), flush=True) + if counted: + print("@@file %s %.1f %s" % (os.path.basename(src), secs, + "failed" if err else "cached" if secs <= 0 else "new"), flush=True) + if not err and expected.get(src): + pending.remove(expected[src].pop()) + print("@@eta %d" % eta_left(pending, eta_jobs), flush=True) + except BaseException: + # ((Triforce, 11-21, the performance audit's LIFT-31, after Source A's) A compiler's error comes back as a value and + # is tried again alone. An exception does not: a compiler that is missing or blocked, an object or its stamp that + # cannot be written, a full disk in cache_put, a file gone before its size is asked, Ctrl-C. It leaves this block, + # and the pool's exit then ran every compile still queued first, the whole compile with nothing said, before the + # build stopped; and make_game deletes those objects, as no "compiled in" line follows. So: the queued ones are + # cancelled, and those waiting at the memory gate are told to go (MemoryGate.stop); the compilers running end) + if GATE: + GATE.stop() + ex.shutdown(wait=False, cancel_futures=True) + raise # (a compiler that ran out of memory beside the others dies "due to signal": a player's 24 MB code_021.c, 2026-11-08. The # files that failed are compiled again one at a time, with all the memory, before the build gives up) still = 0 @@ -1421,6 +1505,7 @@ def main(): t = time.time() r = subprocess.run([cc] + toolchain.native_target() + toolchain.native_link() + ["-o", exe, toolchain.link_opt("/map:" + exe[:-4] + ".map"), toolchain.link_opt("/stack:8388608"), "@" + rsp] + HOT_ORDER() + HOST_LIBS + + repro_link() + # ((Triforce, 11-22, IO-45) HOOKSHOT_LINK_REPRO=1: no link time in the file) ([toolchain.link_opt("/nodefaultlib:clang_rt.profile.lib")] if PGO == "gen" else []) + ([os.path.join(kit.WORK, "hookshot.res")] if os.path.exists(os.path.join(kit.WORK, "hookshot.res")) else []) + # (nx2/make_icon.py) [win_manifest.write(os.path.join(out, "hookshot_manifest.res"))] + # (UTF-8 paths: nx2/win_manifest.py) @@ -1475,7 +1560,7 @@ def main(): ([ngx_library(NGX_SDK, True)] if NGX_SDK else []) + ["-ldl", "-lpthread", "-lm"] else: # (Triforce: advapi32 for RivaTuner Statistics Server's folder in the registry, gpu/vk/rtss_sys_win.cpp) link = [cc] + toolchain.native_target() + toolchain.native_link() + \ - ["-shared", "-o", vdll, toolchain.link_opt("/map:" + vdll[:-4] + ".map"), "@" + vrsp, "-luser32", "-ladvapi32"] + (["-lc++", "-lunwind"] if toolchain.mingw() else []) + ngx_link(NGX_SDK, toolchain.mingw()) + ["-shared", "-o", vdll, toolchain.link_opt("/map:" + vdll[:-4] + ".map"), "@" + vrsp, "-luser32", "-ladvapi32"] + (["-lc++", "-lunwind"] if toolchain.mingw() else []) + ngx_link(NGX_SDK, toolchain.mingw()) + repro_link() r = subprocess.run(link, capture_output=True, text=True) if r.returncode: undefined = sorted({l.strip() for l in (r.stdout + r.stderr).splitlines() if "undefined symbol" in l}) diff --git a/nx2/chunks.py b/nx2/chunks.py index 34ce662..be1f036 100644 --- a/nx2/chunks.py +++ b/nx2/chunks.py @@ -321,7 +321,8 @@ def fn_cost(size): def text_cost(text): """The estimated compile cost of a C file of lifted functions""" - marks = [m.start() for m in HEAD.finditer(text)] + [len(text)] + # ((Triforce, 11-22, the performance audit's IO-40) lifted_c.noted_starts: where HEAD's matches start, found by jumps) + marks = list(lifted_c.noted_starts(text)) + [len(text)] return marks[0] + sum(fn_cost(b - a) for a, b in zip(marks, marks[1:])) @@ -346,7 +347,7 @@ def split_one(gen_dir, path, budget): made, costs = [], {} with open(path, encoding="utf-8", newline="") as f: text = f.read() - marks = [h.start() for h in HEAD.finditer(text)] + marks = list(lifted_c.noted_starts(text)) # ((Triforce, 11-22, IO-40) as [h.start() for h in HEAD.finditer(text)]) ends = marks[1:] + [len(text)] cost = [fn_cost(b - a) for a, b in zip(marks, ends)] total = sum(cost) diff --git a/nx2/cold_parts.py b/nx2/cold_parts.py new file mode 100644 index 0000000..9e11a7d --- /dev/null +++ b/nx2/cold_parts.py @@ -0,0 +1,164 @@ +"""(Triforce, 11-22, the performance audit's LIFT-17, after Source A's) An opt-in -O2 for every lifted chunk, each cut +into parts of a few MB for the compile. + +The build compiles the hot pieces (code_hot_*.c) at -O2 with the play profile and every other lifted chunk (code_NNN.c, +code_NNN_M.c) at -O1. RX_COLD_OPT asks for -O2 on those too: + + RX_COLD_OPT=O2 every lifted chunk but the hot pieces at -O2 + RX_COLD_OPT= only the chunks the file lists, one file name a line (code_012.c; # starts a comment) + unset, empty, O1 as before: nothing here runs + +-O2's time grows with the square of a function's size and its passes look across a whole file, so each -O2 chunk is +compiled as parts: RX_COLD_PART_MB (6 when unset; 0: no cut) is a part's size by the lift's own estimate +(nx2/chunks.py fn_cost). A part is the chunk's declarations and a run of its functions, cut only where a function +begins; the parts together hold the chunk's functions once, in order. They are written under /split as +code_NNN.p0.c, code_NNN.p1.c ..., with a key file a chunk (the chunk's text and the part size): a build that finds +the key and the parts as it wrote them writes nothing. The chunk itself, the lift's file, is not touched, so nothing +of the lift or its key changes. + +Source A's figures: +4.4% and +5.5% fps on two PCs where the game's update bounds the frame, for a lifted compile of +about twice the processor time (and 23 minutes against 5 without the cut); this build's hot list already runs most of +the lifted time at -O2. Opt-in for that cost. Inlining sees one part at a time. + +This file is not part of the lift key (nx2/make_game.py LIFT_KEY_NOT): the lift reads none of it. nx2/build.py is the +one caller; tests/test_chunk_parts.py runs the rules on made-up chunks.""" +import hashlib +import os +import re + +import lifted_c + +CHUNK = re.compile(r"code_\d{3}(?:_\d+)?\.c") # (a lifted chunk or a part of one the lift cut: never code_hot*.c) +PART_MB_DEFAULT = 6 +CUT_VERSION = "cold_parts 1" # (in every key: a change to how chunk_parts cuts writes the parts again) + + +def cold_level(env): + """What RX_COLD_OPT asks: None (nothing), "all", or the set of chunk file names its file lists""" + v = (env.get("RX_COLD_OPT") or "").strip() + if v in ("", "O1"): + return None + if v == "O2": + return "all" + if not os.path.isfile(v): # (a level this does not know, or a list that is not there: not taken for "nothing") + raise SystemExit("build: RX_COLD_OPT=%s: not O2, O1 or a file that lists chunks" % v) + with open(v, encoding="utf-8") as f: + names = {line.split("#", 1)[0].strip() for line in f} + names.discard("") + bad = sorted(n for n in names if not CHUNK.fullmatch(n)) + if bad: + raise SystemExit("build: RX_COLD_OPT=%s: not a lifted chunk's file name: %s" % (v, ", ".join(bad[:5]))) + return frozenset(names) + + +def chunk_opt_o2(name, level): + """Whether the lifted file `name` is compiled at -O2 under that level: a chunk it covers, never a hot piece""" + if level is None or not CHUNK.fullmatch(name): + return False + return level == "all" or name in level + + +def part_bytes(env): + """A part's size in bytes of estimated cost, from RX_COLD_PART_MB (6 when unset or empty); 0: no cut""" + v = (env.get("RX_COLD_PART_MB") or "").strip() + try: + mb = float(v) if v else float(PART_MB_DEFAULT) + except ValueError: + mb = -1.0 + if not 0 <= mb <= 1024: + raise SystemExit("build: RX_COLD_PART_MB=%s: not a size in MB (0: no cut)" % v) + return int(mb * (1 << 20)) + + +def cold_flags(fl, opt, env): + """The chunks' flags for a -O2 chunk: -O2 where the level was, and the part size (so it is in every object's key)""" + return ["-O2" if f == "-" + opt else f for f in fl] + ["-DRX_COLD_PART_MB=%d" % (part_bytes(env) >> 20)] + + +def chunk_parts(text, budget, fn_cost): + """A chunk's text as parts of about `budget` cost each (fn_cost of each function's size): the text before the first + function in every part, then a run of whole functions. A function dearer than the budget is a part by itself. One + part, the text itself, when the chunk fits the budget, has no two functions, or budget is 0""" + marks = [m.start() for m in lifted_c.HEAD_NOTED.finditer(text)] + if budget <= 0 or len(marks) < 2: + return [text] + ends = marks[1:] + [len(text)] + cuts, run = [0], 0 # (indexes into marks where a part starts) + for i, (a, b) in enumerate(zip(marks, ends)): + cost = fn_cost(b - a) + if i > cuts[-1] and run + cost > budget: + cuts.append(i) + run = 0 + run += cost + if len(cuts) == 1: + return [text] + preamble = text[:marks[0]] + bounds = [marks[i] for i in cuts] + [len(text)] + return [preamble + text[a:b] for a, b in zip(bounds, bounds[1:])] + + +def parts_key(text, budget): + """What a chunk's parts depend on: its text, the part size, and how this file cuts""" + return hashlib.sha1(("%s %d\n" % (CUT_VERSION, budget)).encode() + text.encode("utf-8"), usedforsecurity=False).hexdigest() + + +def part_name(chunk_name, i): + return "%s.p%d.c" % (chunk_name[:-2], i) + + +def write_parts(src, split_dir, budget, fn_cost, stats=None): + """The parts of the chunk file `src` under split_dir, written unless its key file says these are they: their paths, in + order. Parts an earlier cut of this chunk left and this one does not make are removed. stats: counts "written" and + "kept" chunks, for the build's log and the test""" + name = os.path.basename(src) + with open(src, encoding="utf-8", newline="") as f: + text = f.read() + key = parts_key(text, budget) + key_path = os.path.join(split_dir, name + ".key") + try: + with open(key_path, encoding="utf-8") as f: + had = f.read().split() + except OSError: + had = [] + if had[:1] == [key] and len(had) > 1 and all(os.path.isfile(os.path.join(split_dir, p)) for p in had[1:]): + if stats is not None: + stats["kept"] = stats.get("kept", 0) + 1 + return [os.path.join(split_dir, p) for p in had[1:]] + os.makedirs(split_dir, exist_ok=True) + if os.path.exists(key_path): # (first: a build stopped while it writes leaves no key that vouches for half the parts) + os.remove(key_path) + parts = chunk_parts(text, budget, fn_cost) + names = [part_name(name, i) for i in range(len(parts))] + for part, piece in zip(names, parts): + with open(os.path.join(split_dir, part), "w", encoding="utf-8", newline="") as f: + f.write(piece) + stale = re.compile(re.escape(name[:-2]) + r"\.p\d+\.c") + for old in os.listdir(split_dir): + if stale.fullmatch(old) and old not in names: + os.remove(os.path.join(split_dir, old)) + with open(key_path, "w", encoding="utf-8", newline="\n") as f: + f.write(key + "\n" + "".join(n + "\n" for n in names)) + if stats is not None: + stats["written"] = stats.get("written", 0) + 1 + return [os.path.join(split_dir, n) for n in names] + + +def expand(srcs, split_dir, env, fn_cost=None, stats=None): + """The build's sources with each -O2 chunk replaced by its parts, and the set of paths that take the -O2 flags (a + chunk's parts, or the chunk itself with RX_COLD_PART_MB=0). With RX_COLD_OPT unset: (srcs as given, nothing)""" + level = cold_level(env) + if level is None: + return srcs, frozenset() + budget = part_bytes(env) + if fn_cost is None: + import chunks # (the lift's own estimate; imported only here, when asked: the default build never loads the lifter) + fn_cost = chunks.fn_cost + out, cold = [], set() + for s in srcs: + if not chunk_opt_o2(os.path.basename(s), level): + out.append(s) + continue + parts = [s] if budget <= 0 else write_parts(s, split_dir, budget, fn_cost, stats) + out += parts + cold.update(parts) + return out, frozenset(cold) diff --git a/nx2/fastvec.py b/nx2/fastvec.py new file mode 100644 index 0000000..4272160 --- /dev/null +++ b/nx2/fastvec.py @@ -0,0 +1,167 @@ +"""(Triforce, 11-21, the performance audit's LIFT-02, after Source A's) Vector moves the kit's translator writes slowly, +rewritten on the lifted text: two passes over one instruction's C, each with its helpers appended to a64_fpx.h. The +kit's files stay as they are (third_party/ is not edited: CONTRIBUTING.md); install(), called from nx2/lift.py beside +insn's and lift_fixes', wraps the kit's functions: + + - a64_lift.vectorize_fpx: vectorize_bytes before it, on the emitter's own text (TBL with a one-register table, EXT), + so that widen_vstores, which comes after, no longer meets the loops' tails; + - a64_lift.op_text: vectorize_moves on its result (128-bit vector loads, DUP by element); + - a64_lift.fpx_header: VBYTE_HEADER and VMOVE_HEADER after the kit's VSET_HEADER, whose A64_OPAQUE and a64_vset64 + they use. + +Each rewrite leaves the same bytes in the same registers; tests/test_vbytes.py runs every shape both ways. Source A +counted, in its market's game code, TBL 2,995 sites, EXT 29,451, 128-bit loads 139,843 and DUP 26,529, and its game +work a frame went from 15.46 to about 14.3 ms with these and one more rewrite that Triforce had already +(a64_lift.py's vectorize_elem). + +Switches, read at lift time (the lift key holds the A64_ variables): A64_NO_VBYTES=1 and A64_NO_VMOVES=1 keep the kit's +text for that pass. The lift's emit processes run nx2/lift.py's module level again (nx2/emit_pool.py), so they have +the wrappers too.""" +import os +import re + +import a64_lift + +# TBL with a one-register table, and EXT. The emitter writes each as a loop of 8 or 16 single bytes through arrays on +# the stack, then a 16-byte reload of the result that cannot be forwarded from the byte stores. On x86 each is one SSSE3 +# instruction. TBL: an index of 16 or more gives 0, which PSHUFB does when the index byte's top bit is set; the index +# plus 0x70, saturated, keeps 0 to 15 below 0x80 with the same low four bits and takes 16 to 255 to 0x80 or more. +# EXT of 16 bytes is PALIGNR, of 8 bytes two shifts of the low halves. Both sources are read before Vd is written, so +# Vd may be Vn or Vm. +VBYTE_TBL = re.compile( + r"\{ uint8_t _t\[16\],_x\[(8|16)\],_r\[\1\]; memcpy\(_t\+0,c->vreg\[(\d+)\],16\); memcpy\(_x,c->vreg\[(\d+)\],\1\); " + r"for\(int _i=0;_i<\1;_i\+\+\)\{ unsigned _k=_x\[_i\]; if\(_k<16U\) _r\[_i\]=_t\[_k\]; else _r\[_i\]=0; \} " + r"c->vreg\[(\d+)\]\[0\]=0; c->vreg\[\4\]\[1\]=0; memcpy\(c->vreg\[\4\],_r,\1\); \}") +VBYTE_EXT = re.compile( + r"\{ uint8_t _a\[(8|16)\],_b\[\1\],_r\[\1\]; memcpy\(_a,c->vreg\[(\d+)\],\1\); memcpy\(_b,c->vreg\[(\d+)\],\1\); " + r"for\(int _i=0;_i<\1;_i\+\+\)\{ int _k=_i\+(\d+); _r\[_i\] = \(_k<\1\) \? _a\[_k\] : _b\[_k-\1\]; \} " + r"c->vreg\[(\d+)\]\[0\]=0; c->vreg\[\5\]\[1\]=0; memcpy\(c->vreg\[\5\],_r,\1\); \}") + + +def vectorize_bytes(text): + if "uint8_t _" not in text or os.environ.get("A64_NO_VBYTES"): + return text + text = VBYTE_TBL.sub(lambda m: "a64_tbl1(c->vreg[%s],c->vreg[%s],c->vreg[%s],%d);" + % (m.group(4), m.group(2), m.group(3), m.group(1) == "16"), text) + return VBYTE_EXT.sub(lambda m: "A64_EXT%s(c->vreg[%s],c->vreg[%s],c->vreg[%s],%s);" + % (m.group(1), m.group(5), m.group(2), m.group(3), m.group(4)), text) + + +VBYTE_HEADER = r""" +/* ---- (Triforce, LIFT-02) TBL with one table register, and EXT, as one SSSE3 instruction each (nx2/fastvec.py + vectorize_bytes). Without SSSE3 (a baseline x86-64 build): plain C, the same bytes ---- */ +#if defined(__SSSE3__) +#include +static inline __attribute__((always_inline)) void a64_tbl1(uint64_t *d, const uint64_t *t, const uint64_t *x, int q) { + const __m128i tv = _mm_loadu_si128((const __m128i *)t); + const __m128i xv = q ? _mm_loadu_si128((const __m128i *)x) : _mm_loadl_epi64((const __m128i *)x); + __m128i r = _mm_shuffle_epi8(tv, _mm_adds_epu8(xv, _mm_set1_epi8(0x70))); + if (!q) r = _mm_move_epi64(r); + A64_OPAQUE(r); + _mm_storeu_si128((__m128i *)d, r); +} +/* (EXT of 16 bytes: Vn's bytes from imm up, then Vm's; PALIGNR takes the high source first) */ +#define A64_EXT16(d, n, m, imm) do { \ + const __m128i a64_e_n_ = _mm_loadu_si128((const __m128i *)(n)), a64_e_m_ = _mm_loadu_si128((const __m128i *)(m)); \ + __m128i a64_e_r_ = _mm_alignr_epi8(a64_e_m_, a64_e_n_, imm); \ + A64_OPAQUE(a64_e_r_); \ + _mm_storeu_si128((__m128i *)(d), a64_e_r_); } while (0) +#else +static inline void a64_tbl1(uint64_t *d, const uint64_t *t, const uint64_t *x, int q) { + uint8_t tb[16], xb[16], r[16] = {0}; + memcpy(tb, t, 16); memcpy(xb, x, 16); + for (int i = 0; i < (q ? 16 : 8); i++) r[i] = xb[i] < 16 ? tb[xb[i]] : 0; + memcpy(d, r, 16); +} +#define A64_EXT16(d, n, m, imm) do { uint8_t a64_e_a_[32], a64_e_r_[16]; memcpy(a64_e_a_, (n), 16); \ + memcpy(a64_e_a_ + 16, (m), 16); memcpy(a64_e_r_, a64_e_a_ + (imm), 16); memcpy((d), a64_e_r_, 16); } while (0) +#endif +/* (EXT of 8 bytes: the low halves, Vn's from byte imm up, then Vm's; the top half zero) */ +#define A64_EXT8(d, n, m, imm) do { uint64_t a64_e_n_ = (n)[0], a64_e_m_ = (m)[0]; \ + const uint64_t a64_e_r_ = (imm) ? (a64_e_n_ >> (8 * (imm))) | (a64_e_m_ << ((64 - 8 * (imm)) & 63)) : a64_e_n_; \ + a64_vset64((d), a64_e_r_); } while (0) +""" + + +# Two moves the compiler leaves slow, on the final text (after widen_vstores): +# - a 128-bit vector load (LDR Q, LDP Q, LD1 of one register) is recomp_ldp64 into the register's two halves: two +# 8-byte stores, from which the next 16-byte read of the register cannot be forwarded. Now one 16-byte load and one +# 16-byte store (a64_vld16); guest memory is read the same way, recomp_load64 being a plain copy. +# - DUP Vd, Vn.T[i] is a loop through the stack and a 16-byte reload: one broadcast (a64_vdup). The element is read +# before Vd is written, so Vd may be Vn. +VMOVE_LDQ = re.compile(r"recomp_ldp64\(c,([^;]*?),&c->vreg\[(\d+)\]\[0\],&c->vreg\[\2\]\[1\]\);") +VMOVE_DUP = re.compile( + r"\{ uint8_t _s\[16\]; memcpy\(_s,c->vreg\[(\d+)\],16\); uint64_t _e=0; memcpy\(&_e,_s\+(\d+),(1|2|4|8)\); " + r"uint8_t _d\[16\]; memset\(_d,0,16\); for\(int _i=0;_i<(\d+);_i\+\+\) memcpy\(_d\+_i\*\3,&_e,\3\); " + r"a64_vst16\(c->vreg\[(\d+)\],_d\); \}") + + +def vectorize_moves(text): + if "c->vreg" not in text or os.environ.get("A64_NO_VMOVES"): + return text + text = VMOVE_LDQ.sub(lambda m: "a64_vld16(c->vreg[%s],%s);" % (m.group(2), m.group(1)), text) + + def dup(m): + s, off, w, n, d = m.groups() + if int(w) * int(n) not in (8, 16): # (a shape that fills neither half nor the whole register: left alone) + return m.group(0) + return "a64_vdup(c->vreg[%s],c->vreg[%s],%s,%s,%d);" % (d, s, off, w, int(w) * int(n) == 16) + return VMOVE_DUP.sub(dup, text) + + +VMOVE_HEADER = r""" +/* ---- (Triforce, LIFT-02) a whole vector register loaded at once, and DUP by element as one broadcast (nx2/fastvec.py + vectorize_moves). Without the SSE2 host path: plain C, the same bytes ---- */ +#if RECOMP_FPX_HOST == 2 +static inline __attribute__((always_inline)) void a64_vld16(uint64_t *d, uint64_t a) { + __m128i x = _mm_loadu_si128((const __m128i *)A64_PTR(a)); + A64_OPAQUE(x); + _mm_storeu_si128((__m128i *)d, x); +} +static inline __attribute__((always_inline)) void a64_vdup(uint64_t *d, const uint64_t *s, int off, int w, int q) { + uint64_t e = 0; + memcpy(&e, (const uint8_t *)s + off, w); + __m128i r = w == 1 ? _mm_set1_epi8((char)e) : w == 2 ? _mm_set1_epi16((short)e) : w == 4 ? _mm_set1_epi32((int)e) + : _mm_set1_epi64x((long long)e); + if (!q) r = _mm_move_epi64(r); + A64_OPAQUE(r); + _mm_storeu_si128((__m128i *)d, r); +} +#else +static inline void a64_vld16(uint64_t *d, uint64_t a) { memcpy(d, A64_PTR(a), 16); } +static inline void a64_vdup(uint64_t *d, const uint64_t *s, int off, int w, int q) { + uint8_t r[16] = {0}; + uint64_t e = 0; + memcpy(&e, (const uint8_t *)s + off, w); + for (int i = 0; i < (q ? 16 : 8) / w; i++) memcpy(r + i * w, &e, w); + memcpy(d, r, 16); +} +#endif +""" + + +_installed = False + + +def install(): + """The two passes around the kit's vectorize_fpx and op_text, and their helpers into its fpx_header. Once""" + global _installed + if _installed: + return + _installed = True + base_vfpx, base_op_text, base_header = a64_lift.vectorize_fpx, a64_lift.op_text, a64_lift.fpx_header + + def vectorize_fpx(text): + return base_vfpx(vectorize_bytes(text)) + + def op_text(text, fpx): + return vectorize_moves(base_op_text(text, fpx)) + + def fpx_header(used): + text = base_header(used) + at = a64_lift.VSET_HEADER + "\n" + if text.count(at) != 1: # (the kit changed: the helpers have nowhere known to go, and the lift must not go on) + raise SystemExit("fastvec: a64_fpx.h no longer holds the kit's VSET_HEADER once: where its helpers go") + return text.replace(at, at + VBYTE_HEADER + "\n" + VMOVE_HEADER + "\n") + + a64_lift.vectorize_fpx, a64_lift.op_text, a64_lift.fpx_header = vectorize_fpx, op_text, fpx_header diff --git a/nx2/fma_soft.py b/nx2/fma_soft.py new file mode 100644 index 0000000..519cc55 --- /dev/null +++ b/nx2/fma_soft.py @@ -0,0 +1,147 @@ +"""(Triforce, 11-22, the performance audit's LIFT-06, after Source A's) A short way out of the exact fused multiply-add +bodies on a CPU without FMA3. + +Where the build has no hardware FMA (x86-64-v2: RECOMP_FPX_FMA64 is 0) the kit's double-precision fast path declines every +FMADD, and the single-precision one declines what its binary64 form cannot prove (a binary32 midpoint, a closed gate), so +those calls run the exact bodies recomp_fpe_fma64 and recomp_fpe_fma32: integer code of many words, about 180 ns a call +by Source A's figure on an i7-3770, 13.7% of a core in its walk. Most of those calls are the plain case, which this +answers first, in the body's first lines: + + - only under the default FPCR (no bit of RECOMP_FPX_FPCR_MASK; FZ may be set), with three normal operands (no zero, + subnormal, infinity or NaN) and, for binary64, factors whose exponents leave the product and its error exact; + - binary64: the product as an exact pair (Dekker's, from Veltkamp's split), the addend added without error (TwoSum), the + two small terms added and rounded to odd, then one rounding to nearest (Boldo and Melquiond's emulation of an FMA); + - binary32: the product is exact in binary64; the addend is added there with its error, the sum rounded to odd, and one + rounding to binary32; + - kept only when the result is a normal finite number above the smallest normal (so neither overflow nor underflow, and + FZ changes nothing); FPSR.IXC is set when the result is not the exact value, and no other flag can be raised; + - anything else goes on into the exact body, unchanged. + +The kit's files stay as they are (third_party/ is not edited): install(), called from nx2/lift.py, wraps the kit's +a64_rtgen.runtime_c and puts HEADER after its `#include "a64.h"` and one EARLY line group after each body's head, each +place required to be there exactly once, so a refreshed kit that moved them stops the lift with a message. + +Switches: A64_NO_FMA_SOFT=1 at lift time (in the lift key, an A64_ name) leaves the kit's runtime text as it is. +HOS_FMA_FAST=0 when the game runs (read once) sends every call to the exact body. Where the build has FMA3 (x86-64-v3) +none of this is compiled. tests/test_fma_soft.py runs both bodies with the way out against without it.""" +import os + +import a64_rtgen + +HEADER = r""" +/* ---- (Triforce, 11-22, the performance audit's LIFT-06, after Source A's) the exact fused multiply-add bodies' short way + out where the CPU has no FMA3 (nx2/fma_soft.py): the plain case (default FPCR, FZ or not; normal operands; a normal + result) by an exact product and sum in binary64, rounded to odd, then rounded once. 1 and the result's bits when it + answers (FPSR.IXC set when inexact), 0 for the exact body. HOS_FMA_FAST=0: never. ---- */ +#if RECOMP_FPX_HOST == 2 && !RECOMP_FPX_FMA64 +#define RECOMP_FMA_SOFT 1 +#include +#if defined(__clang__) +#pragma STDC FP_CONTRACT OFF +#endif +#ifndef RECOMP_FMA_SOFT_PROBE +#define RECOMP_FMA_SOFT_PROBE() ((void)0) /* (tests/test_fma_soft.py counts the calls answered here) */ +#endif +#ifndef RECOMP_FMA_SOFT_ON +static int recomp_fma_soft_on(void) { /* (read once; a race reads the same value twice) */ + static int on = -1; + int v = __atomic_load_n(&on, __ATOMIC_RELAXED); + if (v < 0) { + const char *e = getenv("HOS_FMA_FAST"); + v = !(e && e[0] == '0' && !e[1]); + __atomic_store_n(&on, v, __ATOMIC_RELAXED); + } + return v; +} +#define RECOMP_FMA_SOFT_ON() recomp_fma_soft_on() +#endif +/* (a sum's result moved to the neighbour with an odd last bit, on the side its error lies, when it is inexact and even) */ +static inline double recomp_fma_soft_odd(double s, double err) { + uint64_t u = recomp_fpx_db(s); + if (!(u & 1u)) u += ((err > 0.0) == (s > 0.0)) ? 1u : (uint64_t)-1; + return recomp_fpx_d(u); +} +static inline int recomp_fma_soft64(GuestContext *c, uint64_t a, uint64_t b, uint64_t z, uint64_t *out) { + /* (the factors' biased exponents 543 to 1503: the split cannot overflow, the product is 2^-960 to 2^962 and its error + term is exact; the addend any normal number) */ + if ((c->fpcr & RECOMP_FPX_FPCR_MASK) || (uint32_t)(((uint32_t)(a >> 52) & 0x7ffu) - 543u) > 960u || + (uint32_t)(((uint32_t)(b >> 52) & 0x7ffu) - 543u) > 960u || (uint32_t)(((uint32_t)(z >> 52) & 0x7ffu) - 1u) > 2045u || + !RECOMP_FMA_SOFT_ON()) + return 0; + const double x = recomp_fpx_d(a), y = recomp_fpx_d(b), w = recomp_fpx_d(z); + /* x * y = ph + pl exactly */ + const double ph = x * y; + const double xt = x * 134217729.0, xh = xt - (xt - x), xl = x - xh; + const double yt = y * 134217729.0, yh = yt - (yt - y), yl = y - yh; + const double pl = (((xh * yh - ph) + xh * yl) + xl * yh) + xl * yl; + /* w + ph = th + tl exactly */ + const double th = w + ph, tv = th - w, tl = (w - (th - tv)) + (ph - tv); + /* v = tl + pl rounded to odd */ + double v = tl + pl; + const double vv = v - tl, ve = (tl - (v - vv)) + (pl - vv); + if (ve != 0.0) v = recomp_fma_soft_odd(v, ve); + /* the result, rounded once; its own error for IXC */ + const double r = th + v, rv = r - th, re = (th - (r - rv)) + (v - rv); + const uint64_t t = recomp_fpx_db(r); + if (!RECOMP_FPX_BIG64(t)) return 0; + if (ve != 0.0 || re != 0.0) c->fpsr |= 16u; + RECOMP_FMA_SOFT_PROBE(); + *out = t; + return 1; +} +static inline int recomp_fma_soft32(GuestContext *c, uint64_t a, uint64_t b, uint64_t z, uint64_t *out) { + if ((c->fpcr & RECOMP_FPX_FPCR_MASK) || (uint32_t)(((uint32_t)a & 0x7f800000u) - 0x00800000u) >= 0x7f000000u || + (uint32_t)(((uint32_t)b & 0x7f800000u) - 0x00800000u) >= 0x7f000000u || + (uint32_t)(((uint32_t)z & 0x7f800000u) - 0x00800000u) >= 0x7f000000u || ((a | b | z) >> 32) || !RECOMP_FMA_SOFT_ON()) + return 0; + const double p = (double)recomp_fpx_f(a) * (double)recomp_fpx_f(b); /* (exact: 48 bits) */ + const double w = (double)recomp_fpx_f(z); + double s = p + w; + const double sv = s - p, se = (p - (s - sv)) + (w - sv); + if (se != 0.0) s = recomp_fma_soft_odd(s, se); + const float r = (float)s; + const uint32_t t = (uint32_t)recomp_fpx_fb(r); + if (!RECOMP_FPX_BIG32(t)) return 0; + if (se != 0.0 || (double)r != s) c->fpsr |= 16u; + RECOMP_FMA_SOFT_PROBE(); + *out = t; + return 1; +} +#endif +""" + +INCLUDE = '#include "a64.h"\n' +HEAD = "RECOMP_FPE_COLD uint64_t recomp_fpe_fma%d(GuestContext* c,uint64_t _a,uint64_t _b,uint64_t _z){uint64_t _v=0;\n" +EARLY = ("#ifdef RECOMP_FMA_SOFT\n" + "{ uint64_t _s; if (recomp_fma_soft%d(c, _a, _b, _z, &_s)) return _s; } /* (Triforce, LIFT-06: nx2/fma_soft.py) */\n" + "#endif\n") + + +def patch(text): + """The kit's runtime C with the header and the two ways out; the kit's own text with A64_NO_FMA_SOFT=1""" + if os.environ.get("A64_NO_FMA_SOFT"): + return text + places = [(INCLUDE, INCLUDE + HEADER)] + [(HEAD % n, HEAD % n + EARLY % n) for n in (32, 64)] + for old, new in places: + if text.count(old) != 1: # (the kit changed: the lift must not go on with half of it) + raise SystemExit("fma_soft: %r is %d times in the kit's runtime C (expected once): update nx2/fma_soft.py" + % (old.strip()[:70], text.count(old))) + text = text.replace(old, new) + return text + + +_installed = False + + +def install(): + """patch() around the kit's a64_rtgen.runtime_c, which its write() calls by name. Once""" + global _installed + if _installed: + return + _installed = True + base = a64_rtgen.runtime_c + + def runtime_c(): + return patch(base()) + + a64_rtgen.runtime_c = runtime_c diff --git a/nx2/fma_steps.py b/nx2/fma_steps.py new file mode 100644 index 0000000..092c8c6 --- /dev/null +++ b/nx2/fma_steps.py @@ -0,0 +1,62 @@ +"""(Triforce, 11-22, the performance audit's LIFT-05, after Source A's) The single-precision FRECPS and FRSQRTS steps as +one fused instruction where the build has FMA3. + +The kit's fast path for the Newton-Raphson steps, recomp_fpx_recps32 (2 - a*b) and recomp_fpx_rsqrts32 ((3 - a*b) / 2), +has two forms: on an Arm host one fused multiply-add, the step rounded once as the guest rounds it; on x86-64 the step in +binary64 (two widenings, a multiply, a subtraction, a narrowing) and a test for a binary32 midpoint, which sends the call +to the exact body, because x86-64's baseline has no FMA. The game is built for x86-64-v3 where the CPU has it, and there +one FMA instruction is that same correctly rounded step. The kit already took this step for the fused multiply-add itself +(a64_rtgen.fma32_native, recomp_fpx_fma32); this gives the two steps the same guard. The tests that decide whether the +native result is kept (a normal result; under FZ no subnormal operand) are unchanged, and halving a normal result is +exact. + +The kit's files stay as they are (third_party/ is not edited): install(), called from nx2/lift.py, wraps the kit's +a64_rtgen.runtime_h. Each function's head must be in its text exactly once, so a refreshed kit that changed one stops the +lift with a message, not the game. + +Switches: A64_NO_FMA_STEPS=1 at lift time (in the lift key, an A64_ name) leaves the kit's text for the two steps. The +build define -DA64_FPX_NO_FMA32, the kit's own for the fused multiply-add, now returns all three to the binary64 form. +Nothing changes in a build without FMA3 (x86-64-v2). tests/test_fma_steps.py runs the guest instructions both ways.""" +import os + +import a64_rtgen + +GUARD_OLD = "#if RECOMP_FPX_HOST == 1\n" +GUARD_NEW = "#if RECOMP_FPX_HOST == 1 || (RECOMP_FPX_HOST == 2 && defined(__FMA__) && !defined(A64_FPX_NO_FMA32))\n" +# (each function's head line through its #if line, and the line after it, so the form that follows is the one known here) +RECPS32_OLD = ("static RECOMP_INLINE int recomp_fpx_recps32(uint64_t a, uint64_t b, uint32_t fz, uint64_t* v) {\n" + GUARD_OLD + + " const uint64_t t = recomp_fpx_fb(__builtin_fmaf(-recomp_fpx_f(a), recomp_fpx_f(b), 2.0f));\n") +RSQRTS32_OLD = ("static RECOMP_INLINE int recomp_fpx_rsqrts32(uint64_t a, uint64_t b, uint32_t fz, uint64_t* v) {\n" + GUARD_OLD + + " /* Halving a normal result is exact, and a normal result is all that is kept. */\n" + " const uint64_t t = recomp_fpx_fb(__builtin_fmaf(-recomp_fpx_f(a), recomp_fpx_f(b), 3.0f) * 0.5f);\n") +RECPS32_NEW = RECPS32_OLD.replace(GUARD_OLD, GUARD_NEW) +RSQRTS32_NEW = RSQRTS32_OLD.replace(GUARD_OLD, GUARD_NEW) + + +def patch(h): + """The kit's runtime header with the two steps' fused form on x86-64 with FMA3; the kit's text with A64_NO_FMA_STEPS=1""" + if os.environ.get("A64_NO_FMA_STEPS"): + return h + for name, old, new in (("recomp_fpx_recps32", RECPS32_OLD, RECPS32_NEW), ("recomp_fpx_rsqrts32", RSQRTS32_OLD, RSQRTS32_NEW)): + if h.count(old) != 1: # (the kit changed: the lift must not go on) + raise SystemExit("fma_steps: the kit's %s changed (its head is %d times in the runtime header): update " + "nx2/fma_steps.py" % (name, h.count(old))) + h = h.replace(old, new) + return h + + +_installed = False + + +def install(): + """patch() around the kit's a64_rtgen.runtime_h, which its write() calls by name. Once""" + global _installed + if _installed: + return + _installed = True + base = a64_rtgen.runtime_h + + def runtime_h(): + return patch(base()) + + a64_rtgen.runtime_h = runtime_h diff --git a/nx2/insn.py b/nx2/insn.py index b56d50d..a357888 100644 --- a/nx2/insn.py +++ b/nx2/insn.py @@ -19,6 +19,8 @@ Model: Xd unchanged, PACGA writes 0. Nothing in the game's own code can tell, since no pointer is ever signed. - MTE: tags are always 0. STG/ST2G only do their base-register writeback. """ +import os + import a64emit import a64_lift @@ -219,8 +221,22 @@ def ldar_stlr(w): % (rn, T, T, xr(rt), 1 << size)) +# (Triforce, 11-22, the performance audit's LIFT-47, after Source B's) DMB of the load or the store kind: ISHLD, ISHST and +# the OSH, NSH and full-system forms, eight words (CRm<1:0> = 01 or 10; op2 = 101, so a DSB of those kinds is not one). +# The kit writes every DMB and DSB as recomp_barrier(), an out-of-line full fence. x86-64 keeps load-load, load-store and +# store-store order in hardware, so there such a DMB needs only that the compiler moves no guest access across it: +# A64_DMB_LDST() (nx2/rtgen.py: nothing but a compiler barrier on x86-64, the full fence on any other host). A full DMB +# (ISH, SY, ...), every DSB, ISB and STLR's locked store keep what they had, so store-load order holds as before. +# Only when the lift runs with A64_DMB_LIGHT=1 (unset, empty or 0: the kit's fence; an A64_ name, so in the lift key). +def dmb_light(w): + """Whether w is a DMB of the load or store kind that A64_DMB_LIGHT=1 turns into A64_DMB_LDST()""" + return (w & 0xFFFFF0FF) == 0xD50330BF and ((w >> 8) & 3) in (1, 2) and os.environ.get("A64_DMB_LIGHT", "0") not in ("", "0") + + def emit(w): """C for one of the forms above, or None (then suyu translates it)""" + if (w & 0xFFFFF0FF) == 0xD50330BF: # DMB (LIFT-47: the LD and ST kinds, with A64_DMB_LIGHT=1) + return "A64_DMB_LDST();" if dmb_light(w) else None if (w & 0x3FBF7C00) == 0x089F7C00: # LDAR, STLR, LDLAR, STLLR (o0 = bit 15) return ldar_stlr(w) if (w & 0xBF20FC00) == 0x0E209000 or (w & 0xBF00F400) == 0x0F003000: diff --git a/nx2/lift.py b/nx2/lift.py index db7ade5..fccd8bf 100644 --- a/nx2/lift.py +++ b/nx2/lift.py @@ -29,6 +29,10 @@ import kit import nso2 # noqa: F401 (DT_RELR) import insn import lift_fixes +import fastvec +import fma_soft +import fma_steps +import vecfp import chunks import standalone import adr_entries # noqa: E402 @@ -42,6 +46,10 @@ import a64_rtgen insn.install() lift_fixes.install() +fastvec.install() # ((Triforce, 11-21, LIFT-02) TBL, EXT, 128-bit loads and DUP as one instruction each: nx2/fastvec.py) +fma_soft.install() # ((Triforce, 11-22, LIFT-06) the exact FMA bodies' short way out where the CPU has no FMA3: nx2/fma_soft.py) +fma_steps.install() # ((Triforce, 11-22, LIFT-05) FRECPS and FRSQRTS S as one fused instruction where the build has FMA3: nx2/fma_steps.py) +vecfp.install() # ((Triforce, 11-22, LIFT-03) SSE fast paths before float forms lifted lane by lane, with A64_VECFP=1: nx2/vecfp.py) def module_of(img, pc): diff --git a/nx2/lifted_c.py b/nx2/lifted_c.py index f067ca8..616fc73 100644 --- a/nx2/lifted_c.py +++ b/nx2/lifted_c.py @@ -7,10 +7,60 @@ HEAD_BYTES HEAD, for bytes (build.py's memory estimate reads the files The lift writes its C with \n line ends; a file with \r\n ones (written in text mode on Windows) is read the same: none of the patterns needs the end of the definition's line, and a note's line may end in \r\n. + +(Triforce, 11-22, the performance audit's IO-40, after Source A's) The scans themselves, for the callers: + +definitions(text) HEAD's matches (HEAD_BYTES's for bytes), one after the other, as HEAD.finditer gives them +noted_starts(text) where HEAD_NOTED's matches start, as [m.start() for m in HEAD_NOTED.finditer(text)] + +A pattern that begins with ^ is tried by finditer at every line of 1.5 GB of C. These jump from one "\nvoid f_" to the +next with str.find and try the pattern only there: the same matches, far fewer tries (Source A: two scans over 1.56 GB in +1.49 s where they took 11.49 s). HOOKSHOT_LIFTED_C_REGEX=1 (read once): both are the patterns' own finditer, as before. +tests/test_lifted_c_scan.py holds them to the patterns. """ +import os import re _DEF = r"void (f_[0-9a-f]+)\(Ctx \*__restrict A64_NOESC c\) \{" HEAD = re.compile(r"^" + _DEF, re.M) HEAD_NOTED = re.compile(r"^(?:/\* .* \*/\r?\n)?" + _DEF, re.M) HEAD_BYTES = re.compile((r"^" + _DEF).encode(), re.M) + +REGEX_SCAN = os.environ.get("HOOKSHOT_LIFTED_C_REGEX", "0") not in ("", "0") +_NOTE = re.compile(r"/\* .* \*/\r?\n") # (HEAD_NOTED's optional first line, by itself) + + +def definitions(text): + """Each definition in text (str or bytes), in order: the match HEAD (or HEAD_BYTES) has for it""" + head, mark = (HEAD_BYTES, b"\nvoid f_") if isinstance(text, (bytes, bytearray, memoryview)) else (HEAD, "\nvoid f_") + if REGEX_SCAN: + yield from head.finditer(text) + return + # (^ is the start of the text or the place after a \n: the first line, then every "\nvoid f_". A definition is one + # line and holds no "\n", so the next one is looked for from where this one's line began) + at = 0 + while at >= 0: + m = head.match(text, at) + if m: + yield m + at = text.find(mark, at) + if at >= 0: + at += 1 + + +def noted_starts(text): + """Where each definition in text (str) starts, its note's line counted in when the line just above is one: the starts + of HEAD_NOTED's matches, in order""" + if REGEX_SCAN: + for m in HEAD_NOTED.finditer(text): + yield m.start() + return + # (HEAD_NOTED matches from the note's line when the line above the definition, whole, is "/* ... */" and its line + # end; a match never reaches into the line after a definition's, so that line is free to be the next one's note) + for m in definitions(text): + start = m.start() + if start: + above = text.rfind("\n", 0, start - 1) + 1 + if _NOTE.fullmatch(text, above, start): + start = above + yield start diff --git a/nx2/make_game.py b/nx2/make_game.py index aff1308..2af8c0e 100644 --- a/nx2/make_game.py +++ b/nx2/make_game.py @@ -27,6 +27,7 @@ build's last step, the shaders' preparation, was skipped or failed and is left t import argparse import atexit import contextlib +import errno import filecmp import hashlib import importlib.util @@ -838,10 +839,100 @@ def stop_stray_compilers(work, default_work): print("stopped %d process(es) an earlier build left running on this work folder: %s" % (len(doomed), " ".join(sorted(set(doomed.values())))), flush=True) time.sleep(2) + return len(doomed) # (lock_work waits for a stopped build's lock: Windows frees it late) except Exception as e: # (not finding them is no reason to stop) print("could not look for an earlier build's processes: %s" % e, flush=True) +# (Triforce, 11-22, the performance audit's IO-47, after Source A's) One build at a time per work folder: make_game.py +# holds an OS lock on /make_game.lock for as long as it runs. A second build on the folder stopped nothing on Linux +# (two builders, a builder beside the launcher, AppRun --cli): both compiled the same objects and raced on the staged +# game folder. On Windows stop_stray_compilers still stops an earlier build first, and the lock is what is left when it +# finds none. The lock is the operating system's and ends with its process, so a build that crashed never blocks the +# next one. The first byte is the locked one; the holder's process id is written after it, for the refusal to name. +# HOOKSHOT_WORK_LOCK=0: no lock, as before. +WORK_LOCK = "make_game.lock" +WORK_LOCK_HELD = (errno.EWOULDBLOCK, errno.EAGAIN, errno.EACCES, getattr(errno, "EDEADLOCK", errno.EDEADLK)) +_work_lock = None # ((the open file, its work folder) while this process holds one) + + +def _lock_first_byte(f): + """The OS lock itself, without waiting: an OSError of WORK_LOCK_HELD when another process holds it""" + if os.name == "nt": + import msvcrt + f.seek(0) + msvcrt.locking(f.fileno(), msvcrt.LK_NBLCK, 1) + else: + import fcntl + fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + + +def work_lock_holder(path): + """The process id the holder of a work folder's lock wrote after the locked byte, or None""" + try: + with open(path, "rb") as f: + f.seek(1) + return int(f.read(32).split()[0]) + except (OSError, ValueError, IndexError): + return None + + +def lock_work(work, wait=0.0): + """Takes the work folder's lock for this process, or stops the build (fail) when another build holds it. wait: the + seconds to keep trying first, for after stop_stray_compilers stopped a build (Windows frees a killed process's lock + a little late). Asked again for the same folder it does nothing; for another folder the old lock is let go first. + Only "held by another" refuses: any other error is said and the build goes on without the lock, as before""" + global _work_lock + if os.environ.get("HOOKSHOT_WORK_LOCK") == "0": + return False + work = os.path.abspath(work) + if _work_lock: + if os.path.normcase(_work_lock[1]) == os.path.normcase(work): + return True + release_work() + path = os.path.join(work, WORK_LOCK) + try: # (made when it is not there, never emptied: the holder's process id stays readable) + f = os.fdopen(os.open(path, os.O_RDWR | os.O_CREAT | getattr(os, "O_BINARY", 0), 0o666), "r+b") + except OSError as e: + print("could not open %s (%s): this build runs without the work folder's lock" % (path, e), flush=True) + return False + until = time.monotonic() + wait + while True: + try: + _lock_first_byte(f) + break + except OSError as e: + if e.errno not in WORK_LOCK_HELD: + f.close() + print("could not lock %s (%s): this build runs without the work folder's lock" % (path, e), flush=True) + return False + if time.monotonic() >= until: + f.close() + pid = work_lock_holder(path) + fail("Another build is already working in %s%s. Wait for it to finish, or stop it, then build again." + % (work, " (process %d)" % pid if pid else "")) + time.sleep(0.2) + try: # (after the locked byte, where another process may read on Windows) + f.seek(1) + f.truncate() + f.write(b"%d\n" % os.getpid()) + f.flush() + except OSError: + pass + _work_lock = (f, work) + return True + + +def release_work(): + """Lets this process's work folder lock go (the file stays): for a lock on another folder, and for the tests that run + main() in temporary folders. A build never calls it: its lock ends with the process""" + global _work_lock + if _work_lock: + f, _work_lock = _work_lock[0], None + with contextlib.suppress(OSError, ValueError): + f.close() # (closing the file frees its lock, on Windows and with flock alike) + + def makefile_words(text): """Decode clang -M words, preserving ordinary Windows backslashes.""" text = re.sub(r"\\\r?\n", " ", text) @@ -1176,9 +1267,117 @@ def copy_writable(src, dst): shutil.copyfile(src, dst) +# (Triforce, 11-22, the performance audit's IO-45, after Source A's) What the finish put into the game folder, noted in +# the folder: for each file it installed, the SHA-1 of the source it was copied from and the installed file's size, +# modification time, file number and mode right after the swap. The next finish passes over a file whose source hashes +# the same and whose installed copy still has that size, time, number and mode: every build copied and swapped every +# file, image.bin and the prebuilt DLLs included, and the installed file is never read to find out (a cold read of the +# exe is seconds). Anything not noted, not readable or not as noted is copied as before, and so is everything on a file +# system that gives no file number (FAT, exFAT). HOOKSHOT_FINISH_COPY_ALL=1: every file copied and swapped, no note. +INSTALLED_NOTE = ".triforce_installed.json" + + +def file_sha1(path): + """The SHA-1 of a file's bytes, as hex; None when it cannot be read""" + h = hashlib.sha1() + try: + with open(path, "rb") as f: + for piece in iter(lambda: f.read(1 << 20), b""): + h.update(piece) + except OSError: + return None + return h.hexdigest() + + +def installed_stat(path): + """What tells an installed file is still the one the finish put there, without reading it: [size, modification time + in ns, file number, mode]. None when it is not there, is no plain file, or the file system gives no file number""" + try: + st = os.stat(path) + except OSError: + return None + if not stat.S_ISREG(st.st_mode) or st.st_ino == 0: + return None + return [st.st_size, st.st_mtime_ns, st.st_ino, st.st_mode] + + +def read_installed(out): + """The game folder's note: {a file's path under the folder, with /: {"sha1": its source's, "stat": installed_stat's}}. + Empty when there is none or it is not what write_installed writes (then everything is copied)""" + try: + with open(os.path.join(out, INSTALLED_NOTE), encoding="utf-8") as f: + note = json.load(f) + files = note["files"] if note.get("format") == 1 else {} + if not isinstance(files, dict): + return {} + return {k: v for k, v in files.items() + if isinstance(v, dict) and isinstance(v.get("sha1"), str) and isinstance(v.get("stat"), list)} + except (OSError, ValueError, KeyError, AttributeError, TypeError): + return {} + + +def installed_name(out, dst): + """An installed file's name in the note: its path under the game folder, with / between the parts""" + return os.path.relpath(dst, out).replace(os.sep, "/") + + +def installed_same(note, out, dst, digest): + """Whether dst in the game folder is still the copy of a source with this SHA-1 that the note says the finish + installed: the note's hash is this one and the file's size, time, number and mode are the noted ones""" + was = note.get(installed_name(out, dst)) + now = installed_stat(dst) + return bool(digest and was and now and was["sha1"] == digest and was["stat"] == now) + + +def write_installed(out, files): + """The note, whole: written beside itself and moved in, so a build stopped here leaves the old note or the new one. + A note that cannot be written is said and costs the next build its copies, nothing more""" + path = os.path.join(out, INSTALLED_NOTE) + try: + with open(path + ".new", "w", encoding="utf-8", newline="\n") as f: + json.dump({"format": 1, "files": files}, f, indent=1, sort_keys=True) + os.replace(path + ".new", path) + except OSError as e: + print("could not note what was installed in %s (%s): the next build copies every file again" % (out, e), flush=True) + with contextlib.suppress(OSError): + os.remove(path + ".new") + + def refuse_dump_links(folder): """An extracted dump holds only files and folders: a link in it was followed, and hard-linked or copied the file it - names (anywhere on this PC) into the game folder. The folder itself may be one. (Walked as a long path: #1468)""" + names (anywhere on this PC) into the game folder. The folder itself may be one. (Walked as a long path: #1468) + + (Triforce, 11-21, the performance audit's IO-41, after Source A's) From the folders' listings: os.scandir gives each + entry's kind and, on Windows, its attributes with the listing itself (the reparse-point bit is in them), so no file is + opened or asked about one by one. An os.lstat per entry took 36 to 49 s over the game's 186,583 files where the + listings take 1.3 to 1.5 s (Source A's, Windows 11, NTFS). And a folder that cannot be listed stops the build: os.walk + passed over it silently, and what it held was never looked at. HOOKSHOT_DUMP_LINKS_LSTAT=1: an os.lstat each, as before""" + if os.environ.get("HOOKSHOT_DUMP_LINKS_LSTAT", "") not in ("", "0"): + return refuse_dump_links_lstat(folder) + reparse = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400) + stack = [long_path(folder)] + while stack: + d = stack.pop() + try: + listing = os.scandir(d) + except OSError as e: + fail("Could not read the game files at %s (%s: %s)." % (folder, d, e.strerror or e)) + with listing: + for entry in listing: + try: + link = entry.is_symlink() or getattr(entry.stat(follow_symlinks=False), "st_file_attributes", 0) & reparse + folder_too = not link and entry.is_dir(follow_symlinks=False) + except OSError as e: + fail("Could not read the game files at %s (%s: %s)." % (folder, entry.path, e.strerror or e)) + if link: + fail("The game files at %s contain a link (%s): copy them without links and choose that folder." + % (folder, entry.path)) + if folder_too: + stack.append(entry.path) + + +def refuse_dump_links_lstat(folder): + """(HOOKSHOT_DUMP_LINKS_LSTAT=1) refuse_dump_links as it was: os.walk, an os.lstat per entry""" for d, dirs, files in os.walk(long_path(folder)): for name in dirs + files: path = os.path.join(d, name) @@ -1214,31 +1413,43 @@ def long_path(p): def link_or_copy(src, dst): - """the romfs by hard links where the disk allows: 26 GB not written twice""" + """the romfs by hard links where the disk allows: 26 GB not written twice + + (Triforce, 11-21, the performance audit's IO-41, after Source A's) One walk; each folder made once, not asked for at + every file of it; and no os.path.exists before each link: os.link says FileExistsError itself for a file a retry left, + which is passed over. The staged copy, for a disk that cannot link, is as it was""" src, dst = long_path(src), long_path(dst) - files = [os.path.join(d, f) for d, _, fs in os.walk(src) for f in fs] - for i, s in enumerate(files): - t = os.path.join(dst, os.path.relpath(s, src)) - os.makedirs(os.path.dirname(t), exist_ok=True) - if not os.path.exists(t): + walked = [(d, fs) for d, _, fs in os.walk(src)] + total, i = sum(len(fs) for _, fs in walked), 0 + for d, fs in walked: + rel = os.path.relpath(d, src) + # (the top folder is dst itself: an extended-length path keeps "dst\\." as it is written, a folder named ".") + to = dst if rel == os.curdir else os.path.join(dst, rel) + os.makedirs(to, exist_ok=True) + for f in fs: + s, t = os.path.join(d, f), os.path.join(to, f) try: os.link(s, t) + except FileExistsError: + pass # (a retry: what an earlier try put there stays) except OSError: - # A retry skips existing targets, so publish the copy only after it is complete. - fd, staged = tempfile.mkstemp(prefix=".romfs-", dir=os.path.dirname(t)) - try: - os.close(fd) - shutil.copy2(s, staged) - os.replace(staged, t) - except BaseException: - with contextlib.suppress(FileNotFoundError): - if os.name == "nt": - # copy2 can carry the source's read-only attribute over. - os.chmod(staged, stat.S_IWRITE) - os.unlink(staged) - raise - if i % 500 == 0: - say("progress", "%.3f" % (i / max(1, len(files)))) + if not os.path.exists(t): + # A retry skips existing targets, so publish the copy only after it is complete. + fd, staged = tempfile.mkstemp(prefix=".romfs-", dir=to) + try: + os.close(fd) + shutil.copy2(s, staged) + os.replace(staged, t) + except BaseException: + with contextlib.suppress(FileNotFoundError): + if os.name == "nt": + # copy2 can carry the source's read-only attribute over. + os.chmod(staged, stat.S_IWRITE) + os.unlink(staged) + raise + if i % 500 == 0: + say("progress", "%.3f" % (i / max(1, total))) + i += 1 def shader_census_cmd(romfs, out): @@ -1266,24 +1477,111 @@ def compile_alone_failed(lines): elif "link FAILED" in line: return False return bool(failed) + + +LIFT_KEPT_ONCE = ".lift_kept_after_failure" # (in /sa, not in sa/gen, whose files gen_print vouches for: the lift was kept after a failed compile, once) + + +def failed_only_own_sources(lines, work): + """(Triforce, 11-22, the performance audit's LIFT-27, after Source A's) Whether a build that failed in its compile + (compile_alone_failed) failed only in the program's own sources, host/ and the renderers, and in nothing the lift + wrote: then the translated game is not what failed, and its key can stay, so that the next build does not translate + the game again (a compiler out of memory on a renderer's file cost the retry the whole lift, a minute to several). + lines: build.py's own, "FAILED (...)" for each file that failed alone too and "compiled in Ns, K failed". + False unless every one of the K files is named, none is a lifted file (code_*) and none is a file of the lift's + folder or of the build's generated one (/sa/gen, /sa/build/gen): a lifted file cut short or left by + another builder fails there or at the link (issues 1197, 1176), and the lift is then forgotten as before""" + names, count = [], None + for line in lines: + m = re.match(r"FAILED (\S+) \(", line) + if m: + names.append(m.group(1)) + m = re.match(r"compiled in \d+s, (\d+) failed", line) + if m: + count = int(m.group(1)) + if not names or count != len(names): + return False + made = (os.path.join(work, "sa", "gen"), os.path.join(work, "sa", "build", "gen")) + return not any(n.startswith("code_") or any(os.path.exists(os.path.join(d, n)) for d in made) for n in names) + + +def keep_lift_after_failure(lines, work, environ=None): + """Whether a failed build keeps the lift's key: HOOKSHOT_KEEP_LIFT is not 0, the compile alone failed, only the + program's own sources failed (failed_only_own_sources), and the build before this one did not keep it the same way. + That last: a generated header cut short fails the program's own files, which name nothing of the lift; kept twice, + such a lift would fail at every try, so the second failure in a row forgets it, as every failure did before. + Leaves LIFT_KEPT_ONCE in /sa when it says yes and removes it when it says no (main removes it after a build + that compiled)""" + environ = os.environ if environ is None else environ + mark = os.path.join(work, "sa", LIFT_KEPT_ONCE) + again = os.path.exists(mark) + keep = environ.get("HOOKSHOT_KEEP_LIFT", "1") != "0" and not again and compile_alone_failed(lines) and \ + failed_only_own_sources(lines, work) + try: + if keep: + open(mark, "w").close() + elif again: + os.remove(mark) + except OSError: + keep = False # (the mark cannot be kept: not twice in a row cannot be told, so the lift is forgotten) + return keep PROMPT_KEY = ".made_from" # (in /prompts: prompts_key of what that folder was made from) -def prompts_key(out): +def prompt_game_files(romfs): + """((Triforce, 11-21, the performance audit's IO-43) The game's files nx2/prompt_assets.py reads, as names under the + romfs, each once and in a fixed order: the zstd dictionaries' pack, the font, the textures, the layouts, the camera's + pack, the resource size table and every language's messages. None when the tool's own names cannot be had. + tests/test_prompts_background.py holds this list against what the tool opens""" + try: + import prompt_assets + except Exception: # (the tool is not beside this one, or does not load: the prompts are made at every build, as before) + return None + names = ["Pack/ZsDic.pack.zs", prompt_assets.FONT, prompt_assets.TEXTURES, prompt_assets.LAYOUTS, prompt_assets.CAMERA] + try: # (the first by name, as prompt_assets.read_size_table takes it) + names += ["System/Resource/" + n for n in sorted(os.listdir(os.path.join(romfs, "System", "Resource"))) + if n.startswith("ResourceSizeTable.") and n.endswith(".rsizetable.zs")][:1] + except OSError: + pass + try: + names += [prompt_assets.MALS + "/" + n for n in sorted(os.listdir(os.path.join(romfs, prompt_assets.MALS))) + if n.endswith(".sarc.zs")] + except OSError: + pass + return names + + +def prompts_key(out, romfs=None): """What the button prompts of other controllers are made from, as one hash: the tools that draw them (nx2/prompt_*.py, the faces' drawings and parts in nx2/prompt_faces, the icon fonts in third_party/kenney_input), the packages they are drawn with (nx2/prompt_requirements.txt) and the game's files, by the unpack's stamp (.triforce_romfs: the .nup's name - and size). None when the romfs has no such stamp (the player's own folders, linked in): its prompts are made at every - build, as before. (Triforce) The keyboard and mouse face's too: nx2/prompt_keys.py, which draws it, and the font its - labels are set in (third_party/fonts/libertinus: that one file, the menus' other fonts are not the prompts')""" + and size). (Triforce) The keyboard and mouse face's too: nx2/prompt_keys.py, which draws it, and the font its + labels are set in (third_party/fonts/libertinus: that one file, the menus' other fonts are not the prompts') + + (Triforce, 11-21, the performance audit's IO-43, after Source A's) A romfs with no such stamp (the player's own + folders, linked in) had no key, and its prompts were drawn at every build and every update, 60 to 87 s each time: + its key is now the content of the game's files the tool reads (prompt_game_files, about 100 MB hashed). And for + every game the key holds host/prompts.c, which reads what is drawn, and the release these tools are (version.txt or + VERSION; a checkout has neither). None, the prompts made at every build as before: with no stamp and no romfs given, + when the tool's names cannot be had, or with HOOKSHOT_PROMPTS_FOLDER_KEY=0""" stamp = os.path.join(out, ".triforce_romfs") + game = [] # (the game's files by content, where no stamp says which dump they are) if not os.path.isfile(stamp): - return None + if not romfs or os.environ.get("HOOKSHOT_PROMPTS_FOLDER_KEY", "") == "0": + return None + names = prompt_game_files(romfs) + if names is None: + return None + game = [("romfs/" + n, os.path.join(long_path(romfs), *n.split("/"))) for n in names] + stamp = None # (each file under a name that does not depend on where the tools are installed) files = [("nx2/" + n, os.path.join(HERE, n)) for n in ("prompt_assets.py", "prompt_draw.py", "prompt_faces.py", "prompt_requirements.txt", "prompt_keys.py")] labels = "third_party/fonts/libertinus/LibertinusSans-Bold.ttf" files.append((labels, os.path.join(ROOT, *labels.split("/")))) + # ((Triforce, 11-21, IO-43) what reads the drawn files in the game, and which release these tools are) + files.append(("host/prompts.c", os.path.join(ROOT, "host", "prompts.c"))) + files += [("release/" + n, os.path.join(ROOT, n)) for n in ("version.txt", "VERSION")] for name, root in (("nx2/prompt_faces", os.path.join(HERE, "prompt_faces")), ("third_party/kenney_input", os.path.join(ROOT, "third_party", "kenney_input"))): for folder, dirs, names in os.walk(root): @@ -1291,13 +1589,19 @@ def prompts_key(out): files += [(name + "/" + os.path.relpath(os.path.join(folder, n), root).replace(os.sep, "/"), os.path.join(folder, n)) for n in sorted(names)] h = hashlib.sha1() - with open(stamp, "rb") as f: - h.update(f.read()) - for name, path in files: + if stamp: + with open(stamp, "rb") as f: + h.update(f.read()) + else: + h.update(b"the game's files, by content\n") + for name, path in game + files: if os.path.isfile(path): h.update(name.encode()) with open(path, "rb") as f: - h.update(f.read()) + for piece in iter(lambda: f.read(1 << 20), b""): + h.update(piece) + elif (name, path) in game: + h.update(name.encode() + b": none\n") # (a file the tool would miss: not the key of a romfs that has it) return h.hexdigest() @@ -1355,7 +1659,7 @@ class Prompts: def __init__(self, romfs, out, work): self.out, self.folder, self.new = out, os.path.join(out, "prompts"), os.path.join(out, "prompts.new") - self.key = prompts_key(out) + self.key = prompts_key(out, romfs) # ((Triforce, 11-21, IO-43) the romfs: its files' content where it has no stamp) self.proc = self.log = None self.state = "no fontTools" if importlib.util.find_spec("fontTools") is None else None if self.state: @@ -1418,7 +1722,12 @@ class Prompts: # the lift can import and to what its files name. LIFT_KEY_NOT = ("make_game.py", "build.py", "nup.py", "version.py", "shader_census.py", "prompt_assets.py", "prompt_draw.py", "prompt_faces.py", "prompt_requirements.txt", "prompt_keys.py", - "code_mods.py") # (the mods' code patches: what it does to the game's code is in the key, the exefs's files) + "code_mods.py", # (the mods' code patches: what it does to the game's code is in the key, the exefs's files) + # ((Triforce, 11-22, the performance audit's LIFT-27) the last release's shader translation stamp, which + # build.py alone reads (RELEASED_STAMP): a release that changed the shader translator lifted the game again. + # HOOKSHOT_KEEP_LIFT=0 puts it back in the key: lift_key) + "released_translation_stamp.json", + "cold_parts.py") # ((Triforce, 11-22, LIFT-17) the compile's parts of a -O2 chunk: nx2/build.py's alone def lift_key_file(rel): @@ -1437,13 +1746,18 @@ def lift_key(exefs, work, env, chunks=True): import standalone h = hashlib.sha1(sys.version.encode(), usedforsecurity=False) h.update(" ".join(sorted(standalone.implemented())).encode()) + # ((Triforce, 11-22, the performance audit's LIFT-27, after Source A's) HOOKSHOT_KEEP_LIFT=0: the key as it was, with + # every A64_, OOT_ and RX_ variable and the paths in it, and the release's translation stamp among its files. Read + # here, inside the function, like COMPILE_ONLY_ENV below) + key_as_before = os.environ.get("HOOKSHOT_KEEP_LIFT", "1") == "0" # (each file named from its own base: the exefs and the work folder may be on another drive than the source, which relpath refuses) files = [(exefs, os.path.join(exefs, f)) for f in sorted(os.listdir(exefs))] for top in (HERE, os.path.join(ROOT, "third_party", "a64-kit")): for d, dirs, fs in os.walk(top): dirs[:] = sorted(x for x in dirs if x != "__pycache__") files += [(ROOT, os.path.join(d, f)) for f in sorted(fs) if f.endswith((".py", ".c", ".h", ".cpp", ".hpp", ".txt", ".json")) - and (top != HERE or lift_key_file(os.path.relpath(os.path.join(d, f), HERE).replace(os.sep, "/")))] + and (top != HERE or lift_key_file(os.path.relpath(os.path.join(d, f), HERE).replace(os.sep, "/")) + or (key_as_before and d == HERE and f == "released_translation_stamp.json"))] files.append((work, os.path.join(work, "hot_funcs.txt"))) if chunks: files.append((work, os.path.join(work, "image", "chunks.txt"))) @@ -1453,6 +1767,19 @@ def lift_key(exefs, work, env, chunks=True): with open(f, "rb") as src: h.update(hashlib.sha1(src.read(), usedforsecurity=False).digest()) lift_env = dict(env, **{k: v for k, v in os.environ.items() if k.startswith(("A64_", "OOT_", "RX_"))}) + # ((Triforce, 11-21, LIFT-10) what only the compile after the lift reads: the renderer's profile translates no game code. + # Named here, inside the function: tests/test_lift_key_files.py and test_spawn_emit.py run lift_key cut out by itself) + COMPILE_ONLY_ENV = ("RX_VK_PGO", "RX_VK_PGO_FILE") + for name in COMPILE_ONLY_ENV: + lift_env.pop(name, None) + # ((Triforce, 11-22, LIFT-27) only what shapes the C: the lifter's own switches (A64_*: nx2/fastvec.py and the kit's + # a64_lift.py read no others) and the target (RX_TARGET: the kit's toolchain.windows). The rest that the lift's files + # read are folders (OOT_WORK, OOT_DUMP, OOT_EXEFS, OOT_ROMFS: the files there are hashed above by their contents), + # job counts and where the compiler is, and what the compile after the lift reads (RX_RELEASE, the profile's path): + # a build from another folder, or one more variable in the environment, lifted the same C again. + # tests/test_lift_key_env.py holds the lift's files to that list) + if not key_as_before: + lift_env = {k: v for k, v in lift_env.items() if k.startswith("A64_") or k == "RX_TARGET"} h.update(repr(sorted(lift_env.items())).encode()) return h.hexdigest() @@ -1843,7 +2170,9 @@ def main(a=None): check_cpu() # (no game read yet: the AppImage on a CPU without AVX2 stops here) if not LINUX: check_windows_sdk() - stop_stray_compilers(work, a.default_work) + stopped = stop_stray_compilers(work, a.default_work) + # ((Triforce, 11-22, the performance audit's IO-47) after the stop, so that Windows still replaces a build left running) + lock_work(work, wait=10 if stopped else 0) drop_stale_objects(work) if getattr(a, "fetch_ngx_sdk", False): fetch_ngx_sdk(a) @@ -1853,9 +2182,9 @@ def main(a=None): os.environ["HOOKSHOT_FSR"] = "1" fetch_fsr_runtime(work) # (AMD's two libraries, for the game folder: place_fsr_runtime at the end) open(os.path.join(out, FSR_MARK), "w").close() - if os.environ.get("HOOKSHOT_VK_PREBUILT"): # (the AppImage's back end, built without FSR 4) - print("--fsr: this build uses the prebuilt Vulkan back end (HOOKSHOT_VK_PREBUILT), which has no AMD FSR 4: the " - "Upscaler row will not offer it", flush=True) + if os.environ.get("HOOKSHOT_VK_PREBUILT"): # (the AppImage's back end: built with FSR 4 since 11-20, appimage.sh) + print("--fsr: this build uses the prebuilt Vulkan back end (HOOKSHOT_VK_PREBUILT), which has AMD FSR 4 built in " + "already: the Upscaler row offers it once amd_fidelityfx_upscaler_vk.so is in the game folder", flush=True) else: os.environ.pop("HOOKSHOT_FSR", None) if os.path.isfile(os.path.join(out, FSR_MARK)): @@ -1966,7 +2295,8 @@ def main(a=None): code_patches = _code_mods.build_step(exefs, out, say=lambda line: print(line, flush=True)) # (made once on the development PC: lists of addresses and counts, no game code) - for name in ("hot_funcs.txt", "hot_order.txt", "hookshot_pieces.prof", "hookshot.res"): + # ((Triforce, 11-21, the performance audit's LIFT-10) and the renderer's play profile, triforce_vk.profdata) + for name in ("hot_funcs.txt", "hot_order.txt", "hookshot_pieces.prof", "triforce_vk.profdata", "hookshot.res"): src = os.path.join(ROOT, "work", name) dst = os.path.join(work, name) # (copied only when it differs: a compiler still ending from a cancelled build keeps the profile mapped, WinError 1224) @@ -1982,6 +2312,11 @@ def main(a=None): env["A64_HOT_FUNCS"] = os.path.join(work, "hot_funcs.txt") if os.path.exists(os.path.join(work, "hookshot_pieces.prof")) and profile_readable(os.path.join(work, "hookshot_pieces.prof")): env["RX_PGO"] = os.path.join(work, "hookshot_pieces.prof") + # ((Triforce, 11-21, LIFT-10) the renderer's own code with the shipped profile, unless RX_VK_PGO says what to do (gen, a + # file, 0) or this compiler cannot read it: nx2/build.py vk_pgo_flags) + vk_profile = os.path.join(work, "triforce_vk.profdata") + if "RX_VK_PGO" not in os.environ and os.path.exists(vk_profile) and profile_readable(vk_profile): + env["RX_VK_PGO"] = vk_profile prompts = Prompts(romfs, out, work) # (drawn beside the lift and the compile: finish() below) atexit.register(prompts.stop) # (a build that stops for any reason takes its drawing with it) @@ -2060,7 +2395,8 @@ def main(a=None): said = [] # (build.py's lines about how its compile and its links went: compile_alone_failed) def compiled(line): - if line.startswith("compiled in ") or "link FAILED" in line: + # ((Triforce, 11-22, LIFT-27) and which files failed, "FAILED (...": failed_only_own_sources) + if line.startswith("compiled in ") or "link FAILED" in line or line.startswith("FAILED "): said.append(line) m = re.match(r"(\d+) files, ", line) if m: @@ -2093,13 +2429,21 @@ def main(a=None): # (hookshot) a compile or link that failed forgets the lift: the next build translates the game again. A lifted file cut short # (an antivirus, a full disk, a stopped run: "g_module_b" for g_module_base, issue 1197) or lifted C another builder left (20 # undefined game functions at the link, issue 1176) otherwise failed the same way at every try, the key still matching + # ((Triforce, 11-22, the performance audit's LIFT-27, after Source A's) but for a compile that failed only in the + # program's own sources, once: the translated game is not what failed. keep_lift_after_failure; HOOKSHOT_KEEP_LIFT=0: + # forgotten after every failed build, as before) + lift_kept = keep_lift_after_failure(said, work) key_file = os.path.join(work, "sa", "gen", LIFT_KEY) - if os.path.exists(key_file): + if os.path.exists(key_file) and not lift_kept: os.remove(key_file) # (and its compiled objects go too: a cached runtime object from another version linked with every object "compiled in 0s" # and a64_dirty, a64_icall, a64_lookup, recomp_cntpct undefined, at every try (issue 1197, Linux, v2.0.10)) # (Triforce: but not when the compile alone failed, before any link: compile_alone_failed) - if compile_alone_failed(said): + if lift_kept: + print("the build failed in its compile, in the program's own files only: the translated game and the files that " + "did compile are kept, and the next build compiles only the rest (a second failure like it translates the " + "game again)", flush=True) + elif compile_alone_failed(said): print("the build failed in its compile: the files that did compile are kept, and the next build translates the game " "again and compiles only the rest", flush=True) else: @@ -2108,6 +2452,8 @@ def main(a=None): print("the build failed: the next one translates and compiles the game again (nothing of this one is reused)", flush=True) prompts.stop() raise + with contextlib.suppress(OSError): # ((Triforce, 11-22, LIFT-27) a build that compiled: no failure in a row any more) + os.remove(os.path.join(work, "sa", LIFT_KEPT_ONCE)) compile_seconds = time.time() - compile_started say("stage", "finish", "Putting the game folder together") @@ -2137,6 +2483,25 @@ def main(a=None): staged.append((os.path.join(work, "sa", "image", "image.bin"), os.path.join(out, "image.bin"))) if launcher: staged.append((launcher, os.path.join(out, BESIDE[0]))) + # ((Triforce, 11-22, the performance audit's IO-45) a file the folder already has from the same source is passed + # over: INSTALLED_NOTE above. What is left in staged is copied and swapped as before) + copy_all = os.environ.get("HOOKSHOT_FINISH_COPY_ALL", "0") not in ("", "0") + installed_note = {} if copy_all else read_installed(out) + installed_now, wanted, there = {}, staged, [] + staged = [] + for src, dst in wanted: + digest = None if copy_all else file_sha1(src) + if installed_same(installed_note, out, dst, digest): + installed_now[installed_name(out, dst)] = installed_note[installed_name(out, dst)] + there.append(installed_name(out, dst)) + with contextlib.suppress(OSError): # (a .new an interrupted build left: gone when it can go, never unlocked) + os.remove(dst + ".new") + else: + staged.append((src, dst)) + installed_now[installed_name(out, dst)] = {"sha1": digest, "stat": None} + if there: + print("the game folder has %d of this build's %d files already (%s): not copied again" + % (len(there), len(wanted), ", ".join(there)), flush=True) for src, dst in staged: try: if os.path.exists(dst + ".new"): @@ -2152,6 +2517,16 @@ def main(a=None): except PermissionError: fail("Could not replace %s. Close any running copy of Triforce and check that the file and folder are writable, then retry. " "Staged replacements remain beside the installed files; the compiled source remains at %s." % (dst, src)) + # ((Triforce, 11-22, IO-45) noted only now, with every swap done: each file just installed as it is in the folder) + if copy_all: + with contextlib.suppress(OSError): + os.remove(os.path.join(out, INSTALLED_NOTE)) + else: + for src, dst in staged: + installed_now[installed_name(out, dst)]["stat"] = installed_stat(dst) + installed_now = {k: v for k, v in installed_now.items() if v["sha1"] and v["stat"]} + if installed_now != installed_note: # (nothing installed and nothing gone: the note is the one there) + write_installed(out, installed_now) if launcher: # (a folder built before the launcher: its libraries left the folder for bin\; hookshot is the launcher now) stale = [] if LINUX else sorted(f for f in os.listdir(out) if f.endswith(".dll") and f.startswith(("av", "sw"))) for n in BESIDE[1:] + stale + ["Triforce.map", "hookshot.map", "hookshot-game.exe"]: diff --git a/nx2/nup.py b/nx2/nup.py index 623eb26..d221dce 100644 --- a/nx2/nup.py +++ b/nx2/nup.py @@ -395,13 +395,82 @@ def long_path(p): return "\\\\?\\" + p -def extraction_path(out, part, rel, temporary_suffix=".part"): - """A package path stays under its part, without following existing output links.""" +def package_parts(rel): + """A package path's parts ("/" between them), or the build stops: no empty part, no "." or "..", no backslash, drive + or stream, nothing Windows would rename or open as a device. No file is touched""" parts = rel.split("/") if (not rel or "\\" in rel or any(not p or p in (".", "..") or p[-1] in " ." or any(c in '<>:"|?*' or ord(c) < 32 for c in p) or p.split(".")[0].rstrip(" ").upper() in WINDOWS_DEVICES for p in parts)): raise SystemExit("unsafe path in the package: %r" % rel) + return parts + + +def is_link(info): + """Whether what os.lstat found is a symbolic link or, on Windows, any reparse point (a junction, a mount)""" + return bool(stat.S_ISLNK(info.st_mode) or + getattr(info, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + + +# (Triforce, 11-22, the performance audit's IO-51, after Source A's) HOOKSHOT_NUP_IN_PLACE=0: extract writes each member +# to .part and renames it, and resolves the output folder and looks at every folder above a member for every +# member, as before. Read once. +IN_PLACE = os.environ.get("HOOKSHOT_NUP_IN_PLACE", "1") != "0" + + +class Destination: + """(Triforce, 11-22, the performance audit's IO-51, after Source A's) Where one extraction writes: the output folder + resolved once, and each folder under it looked at once (one os.lstat: there and no link, or made). extraction_path + did both again for every member, 186,583 times over the game's package, and under a virus scanner on NTFS each look + costs. A member's own name is still looked at by extract, once. + + made: the folders this run made itself, which can hold nothing an older run left behind.""" + def __init__(self, out): + self.root = long_path(os.path.realpath(os.path.abspath(out))) + self.checked = set() + self.made = set() + + def folder(self, path, base): + """path (the root, or a folder under it) is a real folder from here on: its parent first, then itself""" + if path in self.checked: + return + if path != self.root: + if os.path.commonpath((os.path.normcase(base), os.path.normcase(os.path.abspath(path)))) != os.path.normcase(base): + raise SystemExit("path leaves the extraction folder: %r" % path) + self.folder(os.path.dirname(path), base) + try: + info = os.lstat(path) + except FileNotFoundError: + info = None + if info is None: + try: + if path == self.root: + os.makedirs(path) + else: + os.mkdir(path) + self.made.add(path) + except FileExistsError: # (made meanwhile by another unpack of the same package: looked at like any other) + info = os.lstat(path) + if info is not None: + if is_link(info): + raise SystemExit("extraction path is a link or reparse point: %s" % path) + if not stat.S_ISDIR(info.st_mode): + raise SystemExit("extraction path is not a folder: %s" % path) + self.checked.add(path) + + def member(self, part, rel): + """Where the package's rel of part goes, its folder there: (the path, whether this run made that folder)""" + parts = package_parts(rel) + base = os.path.join(self.root, part) + folder = os.path.join(base, *parts[:-1]) + if folder not in self.checked: + self.folder(folder, base) + return os.path.join(folder, parts[-1]), folder in self.made + + +def extraction_path(out, part, rel, temporary_suffix=".part"): + """A package path stays under its part, without following existing output links.""" + parts = package_parts(rel) root = long_path(os.path.realpath(os.path.abspath(out))) base = os.path.join(root, part) dst = os.path.join(base, *parts) @@ -419,7 +488,7 @@ def extraction_path(out, part, rel, temporary_suffix=".part"): info = os.lstat(path) except FileNotFoundError: continue - if stat.S_ISLNK(info.st_mode) or getattr(info, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0): + if is_link(info): raise SystemExit("extraction path is a link or reparse point: %s" % path) if path == temporary and info.st_nlink > 1: raise SystemExit("extraction temporary file has other hard links: %s" % path) @@ -502,8 +571,58 @@ def extract(nup, t, out, only=None, quiet=False, zs=None): print(" %.1f MiB written (%d files completed)" % (written / (1 << 20), n), flush=True) next_report = now + interval + # (Triforce, 11-22, the performance audit's IO-51, after Source A's) Each member is written straight to its name: + # one os.lstat, then a create that fails if anything is there (O_EXCL), no .part and no rename. Under a + # virus scanner on NTFS the temporary's create and rename and extraction_path's looks at every folder were most of + # a small member's cost. A member that is there whole is kept (a re-run picks up where it stopped); one of another + # size is unlinked, never truncated (another hard link to it keeps its data); one whose write fails is deleted, so + # the next run writes it again. HOOKSHOT_NUP_IN_PLACE=0: the loop below this one, as before. + where = Destination(out) if IN_PLACE else None for part in ("exefs", "romfs"): - if only and part != only: + if where is None or (only and part != only): + continue + for rel, r in nup.walk(t[part]): + dst, fresh = where.member(part, rel) + looked = zs is not None and part == "romfs" and rel.endswith(".zs") + try: + info = os.lstat(dst) + except FileNotFoundError: + info = None + if info is not None: + if is_link(info): + raise SystemExit("extraction path is a link or reparse point: %s" % dst) + if stat.S_ISREG(info.st_mode) and info.st_size == r[2]: + if looked: + with open(dst, "rb") as f: + zs["files"] += 1 + if not zstd_start(f.read(4)): + zs["damaged"].append(rel) + continue # (re-runs pick up where they stopped) + os.unlink(dst) + if not fresh: # (a .part an interrupted older run left: a folder this run made has none) + with suppress(FileNotFoundError): + os.unlink(dst + ".part") + g = os.fdopen(os.open(dst, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_BINARY", 0), 0o666), "wb") + try: + with g: + sink = HeadSink(g) if looked else g + nup.copy_to(r, sink if quiet else ProgressSink(sink, progress), rel) + if looked: + zs["files"] += 1 + if not zstd_start(sink.head): + zs["damaged"].append(rel) + except BaseException as error: + full = out_of_space(error, dst, part, rel, r[2]) # (measured before the member's space is freed) + # Free the failed member's space without hiding the extraction error. + with suppress(OSError): + os.unlink(dst) + if full: + raise full from error + raise + total += r[2] + n += 1 + for part in ("exefs", "romfs"): + if where is not None or (only and part != only): continue for rel, r in nup.walk(t[part]): dst = extraction_path(out, part, rel) diff --git a/nx2/oot_patches.py b/nx2/oot_patches.py index f2edf34..efde2c9 100644 --- a/nx2/oot_patches.py +++ b/nx2/oot_patches.py @@ -22,6 +22,13 @@ Each is a runtime switch where it can be, so the game's own path stays reachable - ((Triforce) 11-19) The texture data compaction thread's step called through the host (host/oot_tex_compact.c): f_807da3f0's call of f_807da480. The host counts the cycles it starts and, with HOS_OOT_TEX_COMPACT_MS set, holds them to a pace; unset, the step runs as shipped. The supported build only. +- ((Triforce) 11-22) The texture data manager's ask for a compaction round (f_80797240) also asks the host, which lets one through + at most once a second (host/oot_tex_compact.c hos_oot_tex_compact_due, HOS_OOT_TEX_ASK_MS; 0: as shipped). The supported + build only. +- ((Triforce) 11-21, the performance audit's GAME-05) A character's or an object's shadow map's side through the host + (host/oot_shadow_step.h): f_802640a0 reads it once before it asks for the texture; hos_oot_shadow_side rounds it up to a + multiple of 64 (HOS_OOT_SHADOW_STEP), so the sides come round again and the render targets' pool serves them. The supported + build only. """ import glob import os @@ -805,6 +812,67 @@ def patch_tex_compact(s): return s, 1 +# ((Triforce) f_80797240, the game's texture data manager's step, runs every frame: with no compaction round asked for (+776) +# and bit 1 of +960 set, it asks for one and sends the compaction thread the message, so a round could start every frame. The +# ask's test also asks the host (host/oot_tex_compact.c hos_oot_tex_compact_due: at most once a second, HOS_OOT_TEX_ASK_MS); +# a round held back is asked for again at the next step. +TEX_ASK_HEAD = "void f_80797240(Ctx *__restrict A64_NOESC c) {\n" +TEX_ASK_OLD = " c->x[8]=recomp_load8(c,c->x[19] + 960);\nif (!((c->x[8] >> 1) & 1)) { goto L_80797288; }" +TEX_ASK_NEW = (" c->x[8]=recomp_load8(c,c->x[19] + 960);\nif (!((c->x[8] >> 1) & 1) || !hos_oot_tex_compact_due()) { goto L_80797288; }" + " /* (nx2/oot_patches.py: host/oot_tex_compact.c) */") +TEX_ASK_DECL = "int hos_oot_tex_compact_due(void);\n" + + +def patch_tex_ask(s): + """The ask's test above in one file's text: (text, 1 when made now or found made). apply() wants 1 over the whole lift""" + at = _function(s, TEX_ASK_HEAD) + if at is None: + return s, 0 + head, end = at + body = s[head:end] + if body.count(TEX_ASK_NEW) == 1: + return s, 1 + if body.count(TEX_ASK_OLD) != 1 or s.count('#include "a64.h"\n') != 1: + return s, 0 + s = s[:head] + body.replace(TEX_ASK_OLD, TEX_ASK_NEW, 1) + s[end:] + if TEX_ASK_DECL not in s: + s = s.replace('#include "a64.h"\n', '#include "a64.h"\n' + TEX_ASK_DECL, 1) + return s, 1 + + +# ((Triforce, 11-21, the performance audit's GAME-05, after Source B's) f_802640a0 (the game's 0x80264100 is in it) sizes a +# character's or an object's shadow map: it reads the side once (ldr w10, [x21, #0x4e8]) before it asks for the texture, and +# its later uses read the same register. That one read goes through the host, which rounds it up to a multiple of 64 +# (host/oot_shadow_step.h; HOS_OOT_SHADOW_STEP). The lifter writes a register as c->x[n] or, in a function whose registers +# it keeps in locals, as xn_: both forms are looked for. +SHADOW_STEP_HEAD = "void f_802640a0(Ctx *__restrict A64_NOESC c) {\n" +SHADOW_STEP_FORMS = (("c->x[10]=recomp_load32(c,c->x[21] + 1256);", + "c->x[10]=hos_oot_shadow_side(recomp_load32(c,c->x[21] + 1256));"), + ("x10_=recomp_load32(c,x21_ + 1256);", + "x10_=hos_oot_shadow_side(recomp_load32(c,x21_ + 1256));")) +SHADOW_STEP_MARK = " /* (nx2/oot_patches.py: a shadow map's side in steps, host/oot_shadow_step.h) */" +SHADOW_STEP_DECL = "extern uint32_t hos_oot_shadow_side(uint32_t side);\n" + + +def patch_shadow_step(s): + """The shadow map's side through the host: (text, 1) when the function is in s with it (made now or before), (s, 0) when + the function is not here or its code differs (another game version: left as it is)""" + at = _function(s, SHADOW_STEP_HEAD) + if at is None: + return s, 0 + head, end = at + body = s[head:end] + for old, new in SHADOW_STEP_FORMS: + if body.count(new + SHADOW_STEP_MARK) == 1: + return s, 1 + if body.count(old) == 1 and s.count('#include "a64.h"\n') == 1: + s = s[:head] + body.replace(old, new + SHADOW_STEP_MARK, 1) + s[end:] + if SHADOW_STEP_DECL not in s: + s = s.replace('#include "a64.h"\n', '#include "a64.h"\n' + SHADOW_STEP_DECL, 1) + return s, 1 + return s, 0 + + def apply(gen, *, protect_hearts=False, required=False): """required: the lift of the supported build (LIFE_BUILD_ID), whole: a missing output-size edit stops it (apply_output_size). The other patches only warn when their place is not found once.""" @@ -830,6 +898,8 @@ def apply(gen, *, protect_hearts=False, required=False): mod_size_sites = 0 # ((Triforce) patch_mod_sizes: its one call, the supported build's alone) heap_sites = [0, 0] # (the GPU memory's sites, the CPU buffers': patch_heap_retry, the supported build only) tex_compact_sites = 0 # ((Triforce) patch_tex_compact: its one call, the supported build's alone) + tex_ask_sites = 0 # ((Triforce) patch_tex_ask: its one test, the supported build's alone) + shadow_step_sites = 0 # ((Triforce, 11-21, GAME-05) patch_shadow_step: the one read of a shadow map's side) for path in sorted(glob.glob(os.path.join(glob.escape(gen), "code_*.c"))): with open(path, encoding="utf-8", newline="") as f: s = f.read() @@ -962,6 +1032,16 @@ def apply(gen, *, protect_hearts=False, required=False): if compacted != out: print("oot_patches: the texture compaction's step through the host in", os.path.basename(path)) out = compacted + asked, found = patch_tex_ask(out) + tex_ask_sites += found + if asked != out: + print("oot_patches: the texture compaction's ask at most once a second in", os.path.basename(path)) + out = asked + stepped, found = patch_shadow_step(out) + shadow_step_sites += found + if stepped != out: + print("oot_patches: the shadow maps' side through the host in", os.path.basename(path)) + out = stepped if out != s: whole_write.write_whole(path, out, newline="") if sound_jobs != 1: # (without it, above 60 fps sounds and then the music can stop after a few loading zones) @@ -983,6 +1063,10 @@ def apply(gen, *, protect_hearts=False, required=False): print("oot_patches: WARNING: the resource size table's lookup was found at %d of 1 place" % mod_size_sites) if protect_hearts and tex_compact_sites != 1: # (without it the compaction is neither counted nor paced: host/oot_tex_compact.c) print("oot_patches: WARNING: the texture compaction's step was found at %d of 1 place" % tex_compact_sites) + if protect_hearts and shadow_step_sites != 1: # (without it every shadow map has a side of its own, as shipped: host/oot_shadow_step.h) + print("oot_patches: WARNING: the shadow maps' side was found at %d of 1 place" % shadow_step_sites) + if protect_hearts and tex_ask_sites != 1: # (without it the compaction may be asked for every frame: host/oot_tex_compact.c) + print("oot_patches: WARNING: the texture compaction's ask was found at %d of 1 place" % tex_ask_sites) if protect_hearts and zstd_sites != len(ZSTD_SITES): # (without them HOS_HOST_ZSTD does nothing, or half: host/hos_zstd.c) print("oot_patches: WARNING: the host's zstd decoder's calls were found at %d of %d places" % (zstd_sites, len(ZSTD_SITES))) if protect_hearts and heap_sites != [len(GPU_ALLOC_SITES), len(HEAP_ALLOC_SITES)]: # (the supported build's functions) diff --git a/nx2/pipeline_states.py b/nx2/pipeline_states.py index 470f83a..77eddf3 100644 --- a/nx2/pipeline_states.py +++ b/nx2/pipeline_states.py @@ -4,6 +4,19 @@ triforce_pipeline_states_shipped.bin, whose pipelines the first start's shader p python -I nx2/pipeline_states.py ... what they hold python -I nx2/pipeline_states.py --out work/pipeline_states.bin ... merged, each pipeline once + python -I nx2/pipeline_states.py --check ... each list checked (exit 1 when one is not sound) + python -I nx2/pipeline_states.py --out work/pipeline_states.bin --contrib contrib ... + and the players' lists in contrib/ (*.bin.xz, a note each), checked one by one, + with what each source added + python -I nx2/pipeline_states.py --export a player's list checked and packed with a report + +(Triforce, 11-22, the performance audit's PIPE-11, after Source A's.) The shipped list grows from more than one PC's play: +--check says whether a list is sound (its head, its size, each record's counts and topology); --contrib merges the lists +players handed in, each checked first (a list that is not sound is left out whole, with why), in a fixed order (the +positional lists, then contrib/'s by name), and says for each source how many pipelines it added that the ones before +lacked; --export makes what a player hands in (the list as .bin.xz and a report of what it holds, nothing of the PC's). +Point-list records are counted as suspect: --check refuses them and --contrib leaves them out (Source A found 728 of its +734 wrong) unless --keep-points; the plain merge keeps every record, as before. A list holds no game code: each record is a pipeline's stage code keys (hashes of the game's shaders, which the player's own triforce_shaders.bin turns back into the shaders), the program variant's hash, and the formats and draw state baked into the pipeline. @@ -17,8 +30,12 @@ triforce_shaders.bin turns back into the shaders), the program variant's hash, a """ import argparse import collections +import io +import lzma +import os import struct import sys +import zipfile MAGIC = b"HKSTATE1" SIZE = 528 @@ -33,6 +50,131 @@ def read(path): return [body[i:i + SIZE] for i in range(0, len(body) - SIZE + 1, SIZE)] +# ---- (Triforce, 11-22, PIPE-11) checking a list + +TOPOLOGY_MOST = 10 # (VK_PRIMITIVE_TOPOLOGY_PATCH_LIST) +POINT_LIST = 0 # (VK_PRIMITIVE_TOPOLOGY_POINT_LIST) + + +def record_faults(r): + """What is wrong with one record, as words (none: sound). Its counts within their arrays, a first stage, flags 0 or 1""" + faults = [] + code0 = struct.unpack_from(" 1 or has_depth > 1: + faults.append("a flag other than 0 or 1") + if topo > TOPOLOGY_MOST: + faults.append("topology %d" % topo) + if n > 8: + faults.append("%d colour targets" % n) + if samples not in (1, 2, 4, 8, 16): + faults.append("%d samples" % samples) + if nvattr > 16: + faults.append("%d vertex attributes" % nvattr) + if nanswers > 24 or nquestions > 32: + faults.append("%d answers, %d questions" % (nanswers, nquestions)) + return faults + + +def is_point(r): + return struct.unpack_from("= 12 else "?", SIZE)], 0 + body = data[12:] + faults = [] + if len(body) % SIZE: + faults.append("%s: %d bytes past its last whole record (cut short)" % (name, len(body) % SIZE)) + recs = [body[i:i + SIZE] for i in range(0, len(body) - SIZE + 1, SIZE)] + bad = 0 + for k, r in enumerate(recs): + f = record_faults(r) + if f: + bad += 1 + if bad <= 8: + faults.append("%s: record %d: %s" % (name, k, ", ".join(f))) + if bad > 8: + faults.append("%s: %d records not sound in all" % (name, bad)) + return recs, faults, sum(1 for r in recs if is_point(r)) + + +def read_any(path): + """A list's bytes, plain or xz-packed (.xz)""" + with open(path, "rb") as f: + data = f.read() + return lzma.decompress(data) if path.endswith(".xz") else data + + +def contrib_sources(folder): + """The players' lists in a folder: (name, path, note), by name; a list's note is .note beside it ("" without)""" + out = [] + for name in sorted(os.listdir(folder)): + if not name.endswith(".bin.xz"): + continue + note_path = os.path.join(folder, name[:-len(".bin.xz")] + ".note") + note = "" + if os.path.exists(note_path): + with open(note_path, encoding="utf-8", errors="replace") as f: + note = f.read().strip() + out.append((name, os.path.join(folder, name), note)) + return out + + +def merge_sources(sources, keep_points=False): + """sources: (name, bytes, note) in order. The merged records (each once, in the order first met) and a report line a + source: its records, what it added, its repeats, the point-list records left out, or why it was left out whole""" + merged, seen, report = [], set(), [] + for name, data, note in sources: + recs, faults, points = check_bytes(data, name) + if faults: + report.append("%s: left out: %s" % (name, "; ".join(faults))) + continue + added = repeats = dropped = 0 + for r in recs: + if is_point(r) and not keep_points: + dropped += 1 + continue + if r in seen: + repeats += 1 + continue + seen.add(r) + merged.append(r) + added += 1 + report.append("%s: %d pipelines, %d added, %d already in, %d point-list records left out%s" % + (name, len(recs), added, repeats, dropped, (" (" + note + ")") if note else "")) + return merged, report + + +def export_zip(list_path, zip_path): + """A player's list checked and packed: the list as triforce_pipeline_states.bin.xz and report.txt (what it holds, and + the check's words). Nothing of the PC's. False, with the faults printed, when the list is not sound""" + data = read_any(list_path) + recs, faults, points = check_bytes(data, os.path.basename(list_path)) + if faults: + for f in faults: + print(f) + return False + report = io.StringIO() + report.write("Triforce pipeline state list, for the shipped list (hashes and draw state only, no game code)\n") + report.write("%s\n" % describe(recs)) + report.write("point-list records: %d\n" % points) + report.write("record size %d, %s\n" % (SIZE, MAGIC.decode())) + with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_STORED) as z: + z.writestr("triforce_pipeline_states.bin.xz", lzma.compress(data)) + z.writestr("report.txt", report.getvalue()) + return True + + def describe(recs): programs = collections.Counter() kinds = collections.Counter() @@ -46,9 +188,37 @@ def describe(recs): def main(): p = argparse.ArgumentParser(description=__doc__.split("\n")[0]) - p.add_argument("lists", nargs="+") + p.add_argument("lists", nargs="*") p.add_argument("--out") + p.add_argument("--check", action="store_true", help="check each list: exit 1 when one is not sound") + p.add_argument("--contrib", help="a folder of players' lists (*.bin.xz, a .note each), checked and merged after the lists") + p.add_argument("--keep-points", action="store_true", help="keep point-list records (refused by --check, left out by --contrib)") + p.add_argument("--export", nargs=2, metavar=("LIST", "ZIP"), help="a list checked and packed with its report") a = p.parse_args() + if a.export: + return 0 if export_zip(*a.export) else 1 + if a.check: + bad = False + for path in a.lists: + recs, faults, points = check_bytes(read_any(path), path) + if points and not a.keep_points: + faults.append("%s: %d point-list records (suspect: --keep-points to allow)" % (path, points)) + print("%s: %s" % (path, "; ".join(faults) if faults else "sound, " + describe(recs))) + bad |= bool(faults) + return 1 if bad else 0 + if a.contrib: + sources = [(os.path.basename(x), read_any(x), "") for x in a.lists] + sources += [(name, read_any(path), note) for name, path, note in contrib_sources(a.contrib)] + merged, report = merge_sources(sources, a.keep_points) + for line in report: + print(line) + if a.out: + with open(a.out, "wb") as f: + f.write(MAGIC + struct.pack(" 0, GUIDE_CROSS, 0) dirs = ((1, 0), (-1, 0), (0, 1), (0, -1)) goals = {} # ((i, j, dir): (the end, extra cost)): the cell two before the end, coming the way it ends diff --git a/nx2/prompt_faces/README.md b/nx2/prompt_faces/README.md index cfc441a..9e1eeef 100644 --- a/nx2/prompt_faces/README.md +++ b/nx2/prompt_faces/README.md @@ -12,8 +12,8 @@ what it is made from, or draw it by hand and put it here. | `KeyTexFont_00..png` | the white letters on black (the Talk ring's small buttons) | the icon's letter (its holes), or its shape when it has none | | `Controller_Pro.png` | the controller in the pause menu's controls page (System) | the face's `drawing` in `nx2/prompt_faces.py`, drawn by `nx2/prompt_draw.py` | | `Controller_Pro1.png`, `ControllerSh_Pro1.png` | the page's tinted copy of the controller, and its shadow | `Controller_Pro.png` (yours if you made one) | -| `ControllerGuide_Pro_0<0 1>.png` | the page's leader lines that stay, whatever the controls | the face's `parts` in `nx2/prompt_faces.py` (where each control is in the picture) and the router in `prompt_assets.py` (`GUIDE_*`) | -| `ControllerGuide_Pro_0<0 1>...png` | the line of that Switch button's label when it is on that pad input, drawn over the lines that stay | the same | +| `ControllerGuide_Pro_0<0-4>.png` | the page's leader lines that stay, whatever the controls | the face's `parts` in `nx2/prompt_faces.py` (where each control is in the picture) and the router in `prompt_assets.py` (`GUIDE_*`) | +| `ControllerGuide_Pro_0<0-4>...png` | the line of that Switch button's label when it is on that pad input, drawn over the lines that stay (`_02`: ZR's, `_03`: the D-pad's frame, by Switch Up's input, `_04`: L's and R's) | the same | The text icons (dialogue, the control hints, the menus) are not images: they are the face's font, the game's `External.bfotf` with the face's Kenney glyphs, picked by `icons` in `nx2/prompt_faces.py`. diff --git a/nx2/rtgen.py b/nx2/rtgen.py index aee0ed8..d045bed 100644 --- a/nx2/rtgen.py +++ b/nx2/rtgen.py @@ -35,6 +35,17 @@ void a64_poll_cur(void); #define A64_POLL(c) ((void)0) #endif""" +# (Triforce, 11-22, the performance audit's LIFT-47, after Source B's) what a DMB of the load or the store kind is in code +# lifted with A64_DMB_LIGHT=1 (nx2/insn.py dmb_light). x86-64 orders loads with loads, loads with later stores and stores +# with stores itself; the empty asm with a "memory" clobber keeps the compiler from moving or merging guest accesses across +# it (a C11 fence alone would not order the plain accesses), and no instruction is emitted. Any other host: the full fence. +DMB_LDST = r""" +#if defined(__x86_64__) +#define A64_DMB_LDST() __asm__ __volatile__("" ::: "memory") +#else +#define A64_DMB_LDST() recomp_barrier() +#endif""" + ADDR64 = r"""#ifndef A64_ADDR64 #define A64_ADDR64 1 /* (nx2/rtgen.py: 64-bit guest window) */ #endif @@ -73,7 +84,7 @@ SUBST = { "a64.h": [ (THREAD, THREAD.replace("\n} GuestContext;", "\n uint32_t poll; " "/* (nx2/rtgen.py) interrupt request: A64_POLL clears it and calls a64_poll_cur when set */\n} GuestContext;")), - (RET_MISMATCH, RET_MISMATCH + POLL_DECL), + (RET_MISMATCH, RET_MISMATCH + POLL_DECL + DMB_LDST), ("#define A64_PTR(a) (A64_MEM + (uint32_t)(a))", ADDR64), (MARK_NATIVE, MARK_NATIVE_64), ], diff --git a/nx2/split_hot.py b/nx2/split_hot.py index 199fd52..a938cf6 100644 --- a/nx2/split_hot.py +++ b/nx2/split_hot.py @@ -5,6 +5,22 @@ Functions that call each other stay together, heaviest calls first, so what used to inline still can. The pieces cost about the same to compile (chunks.fn_cost). code_hot.c stays as it was, renamed code_hot.c.whole, and build.py compiles code_hot_*.c at the hot flags. A profile from the whole file fits the pieces: its counters go by function name and shape. + +(Triforce, 11-22, the performance audit's LIFT-30, after Source A's) The pieces are sticky, and each declares only what it +uses. Every split dealt the functions out afresh by their sizes, so a few more lines in one hot function could move +functions between most of the pieces, and every piece began with code_hot.c's whole preamble, so one new declaration +changed all of them: an update that touched a few hot functions compiled up to eight pieces of about ten MB again, the +longest files of the build. Now: + - /image/hot_pieces.txt keeps each hot function's piece. The next split leaves the functions it knows where they + are and puts a new one in the piece it shares the most calls with, else in the cheapest; when that leaves the costliest + piece more than STICKY_SLACK times a fresh split's costliest, or the file is not there, or it names more pieces than + this split makes, the split is the fresh one, as before, and the file is written again from it; + - a piece keeps a declaration of code_hot.c's preamble only if it uses the name: the lift's `void f_
(Ctx + *A64_NOESC);` lines and the patches' one-line `hos_` declarations (a function's, or a variable's with extern), each a + whole statement at the top level (declared()). Every other line of the preamble stays in every piece. +A64_HOT_STICKY=0 (at lift time; an A64_ name, in the lift key): the fresh split and the whole preamble, byte for byte as +before, and hot_pieces.txt is neither read nor written. The file is not in the lift key: any map gives the same functions. +tests/test_sticky_hot_pieces.py. """ import collections import glob @@ -21,21 +37,39 @@ HEAD = lifted_c.HEAD CALL = re.compile(r"\b(f_[0-9a-f]{8})\(c\)") -def split(gen, pieces): - path = os.path.join(gen, "code_hot.c") - whole = path + ".whole" - if os.path.exists(path): - os.replace(path, whole) - for old in glob.glob(os.path.join(glob.escape(gen), "code_hot_*.c")): - os.remove(old) - with open(whole, encoding="utf-8", newline="") as f: - text = f.read() - marks = list(HEAD.finditer(text)) - if not marks: - raise SystemExit("split_hot: no functions in %s" % whole) - preamble = text[:marks[0].start()] - bodies = {m.group(1): text[m.start():(marks[i + 1].start() if i + 1 < len(marks) else len(text))] for i, m in enumerate(marks)} +STICKY_SLACK = 1.15 # (how much dearer than a fresh split's the costliest sticky piece may be) +DECL = re.compile(r"void (f_[0-9a-f]+)\(Ctx \*A64_NOESC\);") # (the lift's declaration of a function a hot one names) +# (a one-line declaration of a hos_ name, nx2/oot_patches.py's: a function's, extern or not, or a variable's with extern. A +# variable's without extern is a definition, and stays in every piece as before) +EXTERN = re.compile(r"(?Pextern\s+)?(?:[A-Za-z_]\w*[\s*]+)+(?Phos_\w+)\s*(?P\([^;{}=#]*\))?\s*;") +NAME = re.compile(r"\b(?:f_[0-9a-f]+|hos_\w+)\b") +TOKEN = re.compile(r'/\*|\*/|//|"(?:\\.|[^"\\])*"|\'(?:\\.|[^\'\\])*\'|[{}()]') + +def sticky(): + """Whether the pieces are sticky with their own declarations: unless A64_HOT_STICKY=0""" + return os.environ.get("A64_HOT_STICKY", "1").strip() != "0" + + +def pieces_file(gen): + """/image/hot_pieces.txt for a gen folder (/gen or /sa/gen): beside chunks.txt, which a player's build keeps""" + up = os.path.dirname(os.path.abspath(gen)) + work = os.path.dirname(up) if os.path.basename(up) == "sa" else up + return os.path.join(work, "image", "hot_pieces.txt") + + +def call_edges(bodies): + """How often each hot function calls each other one: {(caller, callee): calls}""" + edges = collections.Counter() + for f, b in bodies.items(): + for callee in CALL.findall(b): + if callee in bodies and callee != f: + edges[(f, callee)] += 1 + return edges + + +def fresh_split(bodies, pieces, edges): + """The split from the sizes alone, as every split was: (the pieces' function names, the calls that stay in one group)""" # (groups joined along the calls: small callees first, those are what the optimiser inlines) limit = sum(len(b) for b in bodies.values()) * 1.1 / pieces parent = {f: f for f in bodies} @@ -46,11 +80,6 @@ def split(gen, pieces): parent[f] = parent[parent[f]] f = parent[f] return f - edges = collections.Counter() - for f, b in bodies.items(): - for callee in CALL.findall(b): - if callee in bodies and callee != f: - edges[(f, callee)] += 1 for (f, callee), n in sorted(edges.items(), key=lambda e: (len(bodies[e[0][1]]) / e[1], e[0])): a, b = root(f), root(callee) if a != b and size[a] + size[b] <= limit: @@ -68,17 +97,158 @@ def split(gen, pieces): b = min(bins, key=lambda x: x[0]) b[0] += sum(cost[f] for f in g) b[1].extend(g) + kept = sum(n for (f, c), n in edges.items() if root(f) == root(c)) + return [names for _, names in bins], kept + + +def read_pieces(path): + """{function: piece} from hot_pieces.txt, or {} when it is not there or not what save_pieces writes""" + try: + with open(path, encoding="utf-8") as f: + rows = [line.split() for line in f if line.strip()] + except OSError: + return {} + if not all(len(r) == 2 and r[1].isdigit() for r in rows): + return {} + return {name: int(piece) for name, piece in rows} + + +def save_pieces(path, bins): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8", newline="\n") as f: + f.write("".join("%s %d\n" % (name, i) for i, names in enumerate(bins) for name in sorted(names))) + + +def sticky_split(bodies, pieces, saved, edges, fresh): + """The pieces with the functions `saved` knows left where they are: (the pieces' names, why), or (fresh, why) when + there is nothing to keep or keeping it costs more than STICKY_SLACK times the fresh split's costliest piece""" + if not saved: + return fresh, "fresh (no pieces file)" + if max(saved.values()) >= pieces: + return fresh, "fresh (the pieces file is for another number of pieces)" # (fewer: the slack below decides) + cost = {f: chunks.fn_cost(len(b)) for f, b in bodies.items()} + bins = [[] for _ in range(pieces)] + where = {} + for f in sorted(bodies): + if f in saved: + bins[saved[f]].append(f) + where[f] = saved[f] + if not where: + return fresh, "fresh (none of the pieces file's functions is hot now)" + known = len(where) + load = [sum(cost[f] for f in names) for names in bins] + calls = collections.defaultdict(collections.Counter) # (each function's calls with every other, either way) + for (f, callee), n in edges.items(): + calls[f][callee] += n + calls[callee][f] += n + for f in sorted(bodies): # (a new one: beside the function it shares the most calls with, else in the cheapest piece) + if f in where: + continue + shared = collections.Counter() + for other, n in calls[f].items(): + if other in where: + shared[where[other]] += n + i = min(range(pieces), key=lambda i: (-shared[i], load[i], i)) + bins[i].append(f) + where[f] = i + load[i] += cost[f] + fresh_top = max(sum(cost[f] for f in names) for names in fresh) + if max(load) > STICKY_SLACK * fresh_top: + return fresh, "fresh (kept where they were, the costliest piece is %.2f times a fresh split's)" % (max(load) / max(1, fresh_top)) + return bins, "sticky (%d functions where they were, %d new)" % (known, len(where) - known) + + +def declared(lines): + """For each line of code_hot.c's preamble the name it declares when a piece may leave it out, else None. Only a whole + statement at the top level is taken: not a line inside braces or parentheses, a comment, a macro's continued lines, or + after an unfinished statement (an attribute on the line before), where a line that reads like a declaration is code""" + out = [] + depth = paren = 0 + comment = cont = False + done = True + for line in lines: + s = line.strip() + name = None + if depth == 0 and paren == 0 and not comment and not cont and done and not s.startswith(("typedef", "static")): + m = DECL.fullmatch(s) + e = None if m else EXTERN.fullmatch(s) + name = m.group(1) if m else e.group("name") if e and (e.group("ext") or e.group("args")) else None + out.append(name) + directive = cont or s.startswith("#") + cont = line.rstrip("\r\n").endswith("\\") + code, pos = [], 0 # (the line without its comments, and the braces and parentheses it opens and closes) + for t in TOKEN.finditer(line): + g = t.group() + if comment: + if g == "*/": + comment, pos = False, t.end() + elif g in ("//", "/*"): + code.append(line[pos:t.start()]) + pos = len(line) + if g == "//": + break + comment = True + elif g in ("{", "}", "(", ")") and not directive: + depth += (g == "{") - (g == "}") + paren += (g == "(") - (g == ")") + if not comment: + code.append(line[pos:]) + code = " ".join(code).strip() + if not directive and code: + done = code.endswith((";", "{", "}")) + return out + + +def piece_preamble(preamble, names_used): + """code_hot.c's preamble for one piece: a declaration of a function or of a hos_ name stays only if the piece uses the + name (names_used: every f_ and hos_ word of its functions); every other line stays""" + lines = preamble.splitlines(keepends=True) + names = declared(lines) + # (a name a line that always stays uses is used too: a macro or an inline helper the preamble may carry) + used = set(names_used) + for line, name in zip(lines, names): + if name is None: + used.update(NAME.findall(line)) + return "".join(line for line, name in zip(lines, names) if name is None or name in used) + + +def split(gen, pieces): + path = os.path.join(gen, "code_hot.c") + whole = path + ".whole" + if os.path.exists(path): + os.replace(path, whole) + for old in glob.glob(os.path.join(glob.escape(gen), "code_hot_*.c")): + os.remove(old) + with open(whole, encoding="utf-8", newline="") as f: + text = f.read() + marks = list(lifted_c.definitions(text)) # ((Triforce, 11-22, the performance audit's IO-40) HEAD's matches, found by jumps) + if not marks: + raise SystemExit("split_hot: no functions in %s" % whole) + preamble = text[:marks[0].start()] + bodies = {m.group(1): text[m.start():(marks[i + 1].start() if i + 1 < len(marks) else len(text))] for i, m in enumerate(marks)} + + edges = call_edges(bodies) + bins, kept = fresh_split(bodies, pieces, edges) + keep = sticky() + if keep: # ((Triforce, LIFT-30) the functions the last split placed stay where they are) + bins, how = sticky_split(bodies, pieces, read_pieces(pieces_file(gen)), edges, bins) + where = {f: i for i, names in enumerate(bins) for f in names} + kept = sum(n for (f, c), n in edges.items() if where[f] == where[c]) + save_pieces(pieces_file(gen), bins) + print("split_hot: %s" % how) + cost = {f: chunks.fn_cost(len(b)) for f, b in bodies.items()} costs = {} - for i, (c, names) in enumerate(bins): + for i, names in enumerate(bins): + c = sum(cost[f] for f in names) + head = piece_preamble(preamble, {n for name in names for n in NAME.findall(bodies[name])}) if keep else preamble with open(os.path.join(gen, "code_hot_%d.c" % i), "w", encoding="utf-8", newline="") as f: - f.write(preamble) + f.write(head) for name in sorted(names): f.write(bodies[name]) - costs["code_hot_%d.c" % i] = len(preamble) + c + costs["code_hot_%d.c" % i] = len(head) + c print("split_hot: piece %d: %d functions, %d MB, cost %d MB" % (i, len(names), sum(len(bodies[f]) for f in names) >> 20, c >> 20)) chunks.write_costs(gen, costs) - kept = sum(n for (f, c), n in edges.items() if root(f) == root(c)) - print("split_hot: %d of %d calls stay in one group" % (kept, sum(edges.values()))) + print("split_hot: %d of %d calls stay in one %s" % (kept, sum(edges.values()), "piece" if keep else "group")) if __name__ == "__main__": diff --git a/nx2/test_nup_in_place.py b/nx2/test_nup_in_place.py new file mode 100644 index 0000000..bf38bf8 --- /dev/null +++ b/nx2/test_nup_in_place.py @@ -0,0 +1,286 @@ +"""Game-free: a package's unpack writes each member in place and looks at each folder once (nx2/nup.py extract, +Destination). + +(Triforce, 11-22, the performance audit's IO-51, after Source A's.) extract wrote every member to .part and +renamed it, and extraction_path resolved the output folder and looked at every folder above the member for every one of +the game's 186,583 members. Under a virus scanner on NTFS those creates, renames and looks were most of a small member's +cost. + +On made-up packages in a temporary folder: + - a fresh unpack calls no os.replace, makes no .part, unlinks nothing, and asks os.lstat once for each folder + and once for each member; the old loop asks more than that and renames every member; + - the files and the .zs notes are the same as the old loop's, byte for byte, nested and non-ASCII names included; + - a member that is there whole is kept and not opened for writing (its .zs head is read back); + - a member whose write fails is removed, and the next run writes it; + - a short member is written again; + - a hard-linked member of another size is replaced and its twin keeps its data; + - a file that appears between the look and the create is not written over (the create is exclusive); + - a .part an older run left in a folder that was there is removed; a link is refused as before; + - HOOKSHOT_NUP_IN_PLACE=0 brings the old loop back, and the switches table says so. + + python -I -B nx2/test_nup_in_place.py +""" +import errno +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +import nup + +ZSTD = b"\x28\xb5\x2f\xfd" + + +class Package: + """Made-up members: name -> bytes.""" + def __init__(self, members): + self.members = dict(members) + + def walk(self, directory): + for name, data in self.members.items(): + yield name, (name, 0, len(data), 0, len(data)) + + def copy_to(self, record, output, name): + output.write(self.members[name][:5]) + output.write(self.members[name][5:]) + + +MEMBERS = { + "Model/Forest/tree.bfres.zs": ZSTD + b"a tree" * 9, + "Model/Forest/rock.bfres.zs": b"\0\0\0\0 a damaged file", + "Model/Forest/Deep/leaf.bin": b"leaf", + "Model/empty.bin": b"", + "Text/Français.msbt.zs": b"\x50\x2a\x4d\x18 a skippable frame", + "top.bin": b"top level", + ".hidden": b"h", +} +FOLDERS = {"", "Model", "Model/Forest", "Model/Forest/Deep", "Text"} # (under romfs; "" is romfs itself) + + +def tree(root): + out = {} + for folder, dirs, files in os.walk(root): + for name in files: + path = os.path.join(folder, name) + with open(path, "rb") as f: + out[os.path.relpath(path, root).replace(os.sep, "/")] = f.read() + return out + + +def plain(path): + """A path without Windows' extended-length prefix, in one case, to compare what os.lstat was asked""" + path = os.path.normcase(os.path.abspath(path)) + return path[4:] if path.startswith("\\\\?\\") else path + + +class InPlace(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(os.path.realpath(self.temporary.name)) + self.out = self.root / "out" + self.out.mkdir() + self.assertTrue(nup.IN_PLACE) + + def extract(self, members=MEMBERS, out=None, zs=None): + return nup.extract(Package(members), {"romfs": 0}, str(out or self.out), only="romfs", quiet=True, zs=zs) + + def test_fresh_unpack_one_look_each_and_no_rename(self): + asked = [] + real_lstat = os.lstat + + def lstat(path, *args, **kwargs): + asked.append(plain(path)) + return real_lstat(path, *args, **kwargs) + + with patch.object(nup.os, "lstat", side_effect=lstat), \ + patch.object(nup.os, "replace", side_effect=AssertionError("a rename")) as replace, \ + patch.object(nup.os, "unlink", side_effect=AssertionError("an unlink")) as unlink: + self.assertEqual(self.extract(), (len(MEMBERS), sum(len(d) for d in MEMBERS.values()))) + replace.assert_not_called() + unlink.assert_not_called() # (romfs was made by this run: no stale .part can be in it) + base = plain(str(self.out / "romfs")) + under = [p for p in asked if p == base or p.startswith(base + os.sep)] + want = sorted(plain(str(self.out / "romfs" / f)) for f in FOLDERS) + \ + sorted(plain(str(self.out / "romfs" / m)) for m in MEMBERS) + self.assertEqual(sorted(under), sorted(want)) # (each folder once, each member once, nothing twice) + self.assertEqual(tree(self.out / "romfs"), MEMBERS) + self.assertEqual([p for p in tree(self.out) if p.endswith(".part")], []) + + def test_the_old_loop_looks_more_and_renames(self): + # (what the count above is compared with, and the switch back) + asked = [] + real_lstat = os.lstat + + def lstat(path, *args, **kwargs): + asked.append(plain(path)) + return real_lstat(path, *args, **kwargs) + + with patch.object(nup, "IN_PLACE", False), patch.object(nup.os, "lstat", side_effect=lstat), \ + patch.object(nup.os, "replace", wraps=os.replace) as replace: + self.extract() + self.assertEqual(replace.call_count, len(MEMBERS)) + base = plain(str(self.out / "romfs")) + under = [p for p in asked if p == base or p.startswith(base + os.sep)] + self.assertGreater(len(under), 2 * (len(FOLDERS) + len(MEMBERS))) + self.assertEqual(tree(self.out / "romfs"), MEMBERS) + + def test_same_files_and_notes_as_the_old_loop(self): + new, old = self.root / "new", self.root / "old" + zs_new, zs_old = {}, {} + counts_new = self.extract(out=new, zs=zs_new) + with patch.object(nup, "IN_PLACE", False): + counts_old = self.extract(out=old, zs=zs_old) + self.assertEqual(counts_new, counts_old) + self.assertEqual(tree(new), tree(old)) + self.assertEqual(tree(new), {"romfs/" + k: v for k, v in MEMBERS.items()}) + self.assertEqual(zs_new, zs_old) + self.assertEqual(zs_new, dict(files=3, damaged=["Model/Forest/rock.bfres.zs"])) + # and a second run over each: nothing written, the notes read back from the files + again_new, again_old = {}, {} + self.assertEqual(self.extract(out=new, zs=again_new), (0, 0)) + with patch.object(nup, "IN_PLACE", False): + self.assertEqual(self.extract(out=old, zs=again_old), (0, 0)) + self.assertEqual(again_new, zs_new) + self.assertEqual(again_old, zs_new) + + def test_whole_member_is_kept_and_not_opened_for_writing(self): + self.extract() + before = {p: os.stat(self.out / "romfs" / p).st_mtime_ns for p in MEMBERS} + with patch.object(nup.os, "open", side_effect=AssertionError("a create")), \ + patch.object(nup.os, "unlink", wraps=os.unlink) as unlink: + self.assertEqual(self.extract(), (0, 0)) + self.assertEqual(before, {p: os.stat(self.out / "romfs" / p).st_mtime_ns for p in MEMBERS}) + # (romfs was there this time: nothing was written, so no .part was looked for either) + unlink.assert_not_called() + + def test_failed_member_is_removed_then_written_on_retry(self): + package = Package(MEMBERS) + failing = "Model/Forest/rock.bfres.zs" + real = package.copy_to + + def copy_to(record, output, name): + if name == failing: + output.write(b"half") + raise OSError(errno.EIO, "a made-up read error") + real(record, output, name) + + with patch.object(package, "copy_to", side_effect=copy_to), self.assertRaises(OSError) as error: + nup.extract(package, {"romfs": 0}, str(self.out), only="romfs", quiet=True) + self.assertEqual(error.exception.errno, errno.EIO) + self.assertFalse((self.out / "romfs" / failing).exists()) + done = tree(self.out / "romfs") + self.assertEqual(done, {k: v for k, v in MEMBERS.items() if k in done}) # (what was written is whole) + n, total = nup.extract(package, {"romfs": 0}, str(self.out), only="romfs", quiet=True) + self.assertEqual(n, len(MEMBERS) - len(done)) + self.assertEqual(tree(self.out / "romfs"), MEMBERS) + + def test_short_member_is_written_again(self): + self.extract() + short = self.out / "romfs" / "top.bin" + short.write_bytes(b"top") + long = self.out / "romfs" / "Model" / "Forest" / "Deep" / "leaf.bin" + long.write_bytes(b"leaf and more than the package has") + self.assertEqual(self.extract(), (2, len(b"top level") + len(b"leaf"))) + self.assertEqual(tree(self.out / "romfs"), MEMBERS) + + def test_hard_linked_member_is_replaced_and_its_twin_keeps_its_data(self): + (self.out / "romfs").mkdir() + twin = self.root / "twin.bin" + twin.write_bytes(b"another file's data, of another size") + os.link(twin, self.out / "romfs" / "top.bin") + self.extract({"top.bin": b"top level"}) + self.assertEqual((self.out / "romfs" / "top.bin").read_bytes(), b"top level") + self.assertEqual(twin.read_bytes(), b"another file's data, of another size") + self.assertEqual(os.stat(twin).st_nlink, 1) + + def test_member_link_is_refused(self): + (self.out / "romfs").mkdir() + outside = self.root / "outside.bin" + outside.write_bytes(b"keep outside data") + try: + os.symlink(outside, self.out / "romfs" / "top.bin") + except OSError as error: + self.skipTest("no symbolic links here: %s" % error) + with self.assertRaisesRegex(SystemExit, "^extraction path is a link or reparse point: "): + self.extract({"top.bin": b"top level"}) + self.assertEqual(outside.read_bytes(), b"keep outside data") + self.assertTrue((self.out / "romfs" / "top.bin").is_symlink()) + + def test_file_that_appears_after_the_look_is_not_written_over(self): + (self.out / "romfs").mkdir() + planted = self.out / "romfs" / "top.bin" + real_lstat = os.lstat + + def lstat(path, *args, **kwargs): + if plain(path) == plain(str(planted)): # (the look sees nothing; then the file is there) + planted.write_bytes(b"planted") + raise FileNotFoundError(errno.ENOENT, "not there yet", str(path)) + return real_lstat(path, *args, **kwargs) + + with patch.object(nup.os, "lstat", side_effect=lstat), self.assertRaises(FileExistsError): + self.extract({"top.bin": b"top level"}) + self.assertEqual(planted.read_bytes(), b"planted") + + def test_stale_part_in_a_folder_that_was_there_is_removed(self): + (self.out / "romfs" / "Model").mkdir(parents=True) + stale = self.out / "romfs" / "Model" / "empty.bin.part" + stale.write_bytes(b"an interrupted older run") + self.extract() + self.assertFalse(stale.exists()) + self.assertEqual(tree(self.out / "romfs"), MEMBERS) + + def test_folder_that_is_a_file_stops_the_unpack(self): + (self.out / "romfs").mkdir() + (self.out / "romfs" / "Model").write_bytes(b"not a folder") + with self.assertRaisesRegex(SystemExit, "^extraction path is not a folder: "): + self.extract() + self.assertEqual((self.out / "romfs" / "Model").read_bytes(), b"not a folder") + + def test_destination_checks_each_folder_once(self): + where = nup.Destination(str(self.out / "new" / "deeper")) + with patch.object(nup.os, "lstat", wraps=os.lstat) as lstat: + first, fresh = where.member("romfs", "A/B/one.bin") + calls = lstat.call_count + second, fresh_again = where.member("romfs", "A/B/two.bin") + self.assertEqual(lstat.call_count, calls) # (the same folder: no further look) + where.member("romfs", "A/C/three.bin") + self.assertEqual(lstat.call_count, calls + 1) # (one new folder: one look) + self.assertTrue(fresh and fresh_again) + self.assertEqual(os.path.dirname(first), os.path.dirname(second)) + self.assertTrue(os.path.isdir(self.out / "new" / "deeper" / "romfs" / "A" / "C")) + with self.assertRaisesRegex(SystemExit, "^unsafe path in the package: "): + where.member("romfs", "A/../../outside.bin") + + def test_switch_is_read_from_the_environment(self): + for value, want in ((None, "True"), ("1", "True"), ("0", "False")): + env = {k: v for k, v in os.environ.items() if k != "HOOKSHOT_NUP_IN_PLACE"} + if value is not None: + env["HOOKSHOT_NUP_IN_PLACE"] = value + said = subprocess.run([sys.executable, "-I", "-B", "-c", + "import sys; sys.path.insert(0, sys.argv[1]); import nup; print(nup.IN_PLACE)", HERE], + env=env, capture_output=True, text=True) + self.assertEqual(said.stdout.strip(), want, said.stderr) + + def test_source_and_docs(self): + source = Path(HERE, "nup.py").read_text(encoding="utf-8") + extract = source[source.index("def extract("):source.index("def heal(")] + self.assertIn("where = Destination(out) if IN_PLACE else None", extract) + self.assertIn("os.O_WRONLY | os.O_CREAT | os.O_EXCL", extract) + self.assertIn('IN_PLACE = os.environ.get("HOOKSHOT_NUP_IN_PLACE", "1") != "0"', source) + # (heal and the .plain path keep the per-member check) + self.assertIn('p = extraction_path(out, "exefs", m)', source) + self.assertIn('extraction_path(a.extract, "exefs", m, temporary_suffix=".plain")', source) + docs = Path(HERE).parent.joinpath("docs", "development.md").read_text(encoding="utf-8") + row = [line for line in docs.splitlines() if line.startswith("| `HOOKSHOT_NUP_IN_PLACE=0`")] + self.assertEqual(len(row), 1) + self.assertIn("IO-51", row[0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/nx2/test_nup_out_of_space.py b/nx2/test_nup_out_of_space.py index a3316df..d2cc545 100644 --- a/nx2/test_nup_out_of_space.py +++ b/nx2/test_nup_out_of_space.py @@ -34,11 +34,11 @@ class OutOfSpace(unittest.TestCase): def test_full_drive_names_the_member_and_the_drive(self): with tempfile.TemporaryDirectory() as out: member = Path(os.path.realpath(out)) / "romfs" / "Model" / "authored.zs" - partial = member.with_name("authored.zs.part") seen = [] def usage(path): - seen.append((path, partial.exists())) + # (the member itself: it is written straight to its name since the performance audit's IO-51) + seen.append((path, member.exists())) return SimpleNamespace(free=196 * 10 ** 9) original = OSError(errno.ENOSPC, "authored disk-full failure") @@ -51,8 +51,9 @@ class OutOfSpace(unittest.TestCase): "No space left on device writing romfs/Model/authored.zs (12345 bytes): 196.0 GB free there, NTFS") self.assertIs(error.exception.__cause__, original) # (measured while the failed member still holds its space) - self.assertEqual(seen, [(str(member.parent), True)]) - self.assertFalse(partial.exists()) + self.assertEqual(seen, [(nup.long_path(str(member.parent)), True)]) + self.assertFalse(member.exists()) + self.assertFalse(member.with_name("authored.zs.part").exists()) def test_other_errors_pass_through(self): with tempfile.TemporaryDirectory() as out: diff --git a/nx2/test_nup_paths.py b/nx2/test_nup_paths.py index b74bc5d..b715657 100644 --- a/nx2/test_nup_paths.py +++ b/nx2/test_nup_paths.py @@ -207,10 +207,11 @@ class ExtractionPaths(unittest.TestCase): with self.assertRaisesRegex(SystemExit, "^" + re.escape("unsafe path in the package: %r" % name) + "$"): self.extract([name]) self.assertEqual(attempts, []) - # In this exact context, a valid name must pass validation and reach a write. - with self.assertRaisesRegex(WriteAttempt, "^os\\.makedirs$"): + # In this exact context, a valid name must pass validation and reach a write (the part's folder, + # made once by nup.Destination since the performance audit's IO-51). + with self.assertRaisesRegex(WriteAttempt, "^os\\.mkdir$"): self.extract(["valid.bin"]) - self.assertEqual(attempts, ["os.makedirs"]) + self.assertEqual(attempts, ["os.mkdir"]) self.assertEqual(before, snapshot(self.root)) def test_write_guards_block_alternative_io(self): @@ -255,7 +256,9 @@ class ExtractionPaths(unittest.TestCase): self.assertEqual((self.out / "romfs" / "file.bin").read_bytes(), b"authored fixture") self.assertFalse(partial.exists()) - def test_failed_member_keeps_previous_file_and_can_be_retried(self): + def test_failed_member_is_removed_and_can_be_retried(self): + # (the performance audit's IO-51: a member is written straight to its name, so a file of another size that was + # there is unlinked first and a write that fails leaves nothing; before, the previous file stayed) target = self.out / "romfs" / "file.bin" target.parent.mkdir() partial = target.with_name("file.bin.part") @@ -266,18 +269,16 @@ class ExtractionPaths(unittest.TestCase): output.write(b"some bytes") raise OSError(errno.ENOSPC, "authored disk-full failure") - failures = ( - (patch.object(package, "copy_to", side_effect=out_of_space), errno.ENOSPC), - (patch.object(nup.os, "replace", side_effect=PermissionError(errno.EACCES, "authored replacement failure")), - errno.EACCES), - ) - for failure, code in failures: + def refused(record, output, name): + raise PermissionError(errno.EACCES, "authored write failure") + + for failure, code in ((out_of_space, errno.ENOSPC), (refused, errno.EACCES)): with self.subTest(error=code): target.write_bytes(b"previous file") - with failure, self.assertRaises(OSError) as error: + with patch.object(package, "copy_to", side_effect=failure), self.assertRaises(OSError) as error: nup.extract(package, title, str(self.out), only="romfs", quiet=True) self.assertEqual(error.exception.errno, code) - self.assertEqual(target.read_bytes(), b"previous file") + self.assertFalse(target.exists()) self.assertFalse(partial.exists()) self.assertEqual(nup.extract(package, title, str(self.out), only="romfs", quiet=True), (1, len(package.data))) @@ -317,18 +318,23 @@ class ExtractionPaths(unittest.TestCase): self.fail("Cannot create the file-link fixture: %s" % error) self.reject(["file.bin"], "extraction path is a link or reparse point: ") - def test_temporary_link_cannot_truncate_an_outside_file(self): - (self.out / "romfs").mkdir() - try: - os.symlink(self.sentinel, self.out / "romfs" / "file.bin.part") - except OSError as error: - self.fail("Cannot create the file-link fixture: %s" % error) - self.reject(["file.bin"], "extraction path is a link or reparse point: ") - - def test_temporary_hard_link_cannot_truncate_an_outside_file(self): - (self.out / "romfs").mkdir() - os.link(self.sentinel, self.out / "romfs" / "file.bin.part") - self.reject(["file.bin"], "extraction temporary file has other hard links: ") + def test_stale_temporary_link_is_removed_without_touching_an_outside_file(self): + # (the performance audit's IO-51: no .part is written any more, so one an older run left, a link or a + # hard link included, is only unlinked, which never reaches what it points at; before, both were refused) + for kind, make in (("link", os.symlink), ("hard link", os.link)): + with self.subTest(kind=kind): + folder = self.out / "romfs" / kind.replace(" ", "_") + folder.mkdir(parents=True) + partial = folder / "file.bin.part" + try: + make(self.sentinel, partial) + except OSError as error: + self.fail("Cannot create the file-link fixture: %s" % error) + name = folder.name + "/file.bin" + self.assertEqual(self.extract([name]), (1, len(b"authored fixture"))) + self.assertEqual((folder / "file.bin").read_bytes(), b"authored fixture") + self.assertFalse(os.path.lexists(partial)) + self.assertEqual(self.sentinel.read_bytes(), b"keep outside data") def test_heal_missing_module_and_resume(self): (self.out / "exefs").mkdir() diff --git a/nx2/vecfp.py b/nx2/vecfp.py new file mode 100644 index 0000000..2d24d57 --- /dev/null +++ b/nx2/vecfp.py @@ -0,0 +1,308 @@ +"""(Triforce, 11-22, the performance audit's LIFT-03, after Source A's) SSE fast paths for the float forms the lifter +still writes lane by lane, behind A64_VECFP (off by default). + +The kit turns straight FADD / FSUB, FMUL .4S and FMUL / FMLA / FMLS .4S by element into one SSE op with the scalar fast +path's tests on every lane (a64_lift.py's vectorize_fpx and vectorize_elem). These stay loops of one helper call a lane, +or exact integer code a lane: + + elem FMUL, FMLA, FMLS .2S by element + vec FMUL .2S; FMLA and FMLS .2S and .4S; FDIV .2S and .4S + minmax FMAX, FMIN, FMAXNM, FMINNM: scalar S, .2S and .4S + fcvt FCVT S to D and D to S (scalar); FCVTL and FCVTL2 (.2S to .2D); FCVTN (.2D to .2S) + +With A64_VECFP=1 (or a list of the groups above) each such instruction is lifted as + + if (!a64_vfpx_q_(c, ...)) { } + +The helper (HEADER, appended to a64_fpx.h) does the op in SSE and writes the register only when the result is provably +Arm's with no new FPSR bit, by the rules the kit's own helpers use (RECOMP_FPX_OPEN, RECOMP_FPX_KEEP32 / KEEPDIV32, no +subnormal operand under FZ); otherwise it writes nothing, returns 0, and the C as before runs: + - multiplies, multiply-adds and divides: under the open gate (default FPCR but FZ, IXC already raised); a lane is kept + when its result is a normal number above the smallest, or a finite result with a zero factor (dividend, for a divide); + the multiply-adds are one fused op and need FMA3, without it they always decline; + - min and max: under the default FPCR but FZ; kept unless a lane is a NaN or, under FZ, a subnormal. Zeros of both signs + come out as Arm's (max of +0 and -0 is +0) by taking the op both ways round; + - S to D: exact; kept unless the input is a NaN or, under FZ, a subnormal; + - D to S: under the open gate; kept when the result is a normal number above the smallest, or the input is a zero. + +The kit's files stay as they are (third_party/ is not edited): install(), called from nx2/lift.py after insn's, wraps +a64emit.translate (the word decides the form; the text behind the test is whatever the translator wrote, so no pattern +of its text is assumed) and a64_lift.fpx_header. A word whose text the kit's own passes already turn into a vector +helper is left alone (main_handles), so nothing is wrapped twice. The closing brace goes on the text's last line, never +at a line's start: nx2/oot_patches.py ends a function at the first `}` that starts a line. + +A64_VECFP is read at lift time and is in the lift key (an A64_ name). Unset, empty or 0: the kit's text, the default +until it is measured in the game. tests/test_vecfp.py runs every form both ways and against the exact body.""" +import os +import re + +import a64emit +import a64_lift + +GROUPS = ("elem", "vec", "minmax", "fcvt") + + +def groups(): + """The groups A64_VECFP turns on: none when unset, empty or 0; all for 1; else the listed ones""" + v = os.environ.get("A64_VECFP", "0").strip() + if v in ("", "0"): + return frozenset() + if v == "1": + return frozenset(GROUPS) + got = frozenset(p.strip() for p in v.split(",") if p.strip()) + if got - set(GROUPS): + raise SystemExit("vecfp: A64_VECFP=%s: not 0, 1 or a list of %s" % (v, ", ".join(GROUPS))) + return got + + +def fast_call(w, on): + """The helper call for guest word w (C, nonzero when it did the op), or None: not one of the forms, or its group is off""" + top = w & 0x1F000000 + if top not in (0x0E000000, 0x0F000000, 0x1E000000) or not on: + return None + rd, rn, rm = w & 31, (w >> 5) & 31, (w >> 16) & 31 + d, n, m = "c->vreg[%d]" % rd, "c->vreg[%d]" % rn, "c->vreg[%d]" % rm + k = 4 if w & 0x40000000 else 2 + v = w & 0xBFE0FC00 # (the three-register vector forms of single precision, Q left out) + if "vec" in on: + if v == 0x2E20DC00 and k == 2: # FMUL .2S (.4S is the kit's a64_vfpx_mul32) + return "a64_vfpx_q_mul(c,%s,%s,%s,2)" % (d, n, m) + if v in (0x0E20CC00, 0x0EA0CC00): # FMLA, FMLS .2S / .4S + return "a64_vfpx_q_fma(c,%s,%s,%s,%d,%d)" % (d, n, m, v == 0x0EA0CC00, k) + if v == 0x2E20FC00: # FDIV .2S / .4S + return "a64_vfpx_q_div(c,%s,%s,%s,%d)" % (d, n, m, k) + if "minmax" in on: + if v in (0x0E20F400, 0x0EA0F400, 0x0E20C400, 0x0EA0C400): # FMAX, FMIN, FMAXNM, FMINNM .2S / .4S + return "a64_vfpx_q_minmax(c,%s,%s,%s,%d,%d)" % (d, n, m, bool(v & 0x00800000), k) + s = w & 0xFFE0FC00 + if s in (0x1E204800, 0x1E205800, 0x1E206800, 0x1E207800): # FMAX, FMIN, FMAXNM, FMINNM S + return "a64_vfpx_q_minmax(c,%s,%s,%s,%d,1)" % (d, n, m, bool(s & 0x00001000)) + if "elem" in on: + e = w & 0xFFC0F400 # (.2S by element: Q = 0, size = 10; L, M:Rm and H left out) + if e in (0x0F809000, 0x0F801000, 0x0F805000): # FMUL, FMLA, FMLS .2S by element + index = ((w >> 11) & 1) << 1 | ((w >> 21) & 1) + if e == 0x0F809000: + return "a64_vfpx_q_mule(c,%s,%s,%s,%d,2)" % (d, n, m, index) + return "a64_vfpx_q_fmae(c,%s,%s,%s,%d,%d,2)" % (d, n, m, index, e == 0x0F805000) + if "fcvt" in on: + u = w & 0xFFFFFC00 + if u == 0x1E22C000: # FCVT Dd, Sn + return "a64_vfpx_q_cvtsd(c,%s,%s)" % (d, n) + if u == 0x1E624000: # FCVT Sd, Dn + return "a64_vfpx_q_cvtds(c,%s,%s)" % (d, n) + if (w & 0xBFFFFC00) == 0x0E617800: # FCVTL / FCVTL2 .2D from .2S + return "a64_vfpx_q_cvtl(c,%s,%s,%d)" % (d, n, k == 4) + if u == 0x0E616800: # FCVTN .2S from .2D (FCVTN2 keeps the low half: not one) + return "a64_vfpx_q_cvtn(c,%s,%s)" % (d, n) + return None + + +KIT_VECTOR = re.compile(r"\ba64_vfpx_(?!q_)\w+\(|\ba64_vfcmp\(") + + +def main_handles(text): + """Whether the kit's own passes already turn this instruction's text into a vector helper (then it is not wrapped)""" + return KIT_VECTOR.search(a64_lift.vectorize_elem(a64_lift.widen_vstores(a64_lift.vectorize_fpx( + a64_lift.outline_fpx(text, set()))))) is not None + + +def wrap(text, call): + """The instruction's text behind the helper's test. The closing brace stays on the text's last line""" + return "if (!%s) { %s }\n" % (call, text.strip()) + + +HEADER = r""" +/* ---- (Triforce, 11-22, the performance audit's LIFT-03, after Source A's) SSE fast paths for float forms lifted lane by + lane (nx2/vecfp.py, A64_VECFP): each returns 1 when it did the op and wrote Vd, 0 with nothing written when the C behind + it must run. k: the lanes (4, 2, or 1 for a scalar). A lane is kept by the scalar fast paths' rules. ---- */ +#ifndef A64_VFPX_Q_PROBE +#define A64_VFPX_Q_PROBE() ((void)0) /* (tests/test_vecfp.py counts the ops done here) */ +#endif +#if RECOMP_FPX_HOST == 2 +#if defined(__FMA__) +#include +#endif +static inline __m128i a64_vq_ult(__m128i x, uint32_t k) { /* unsigned x < k, per lane */ + return _mm_cmplt_epi32(_mm_xor_si128(x, _mm_set1_epi32((int)0x80000000u)), _mm_set1_epi32((int)(k ^ 0x80000000u))); +} +static inline __m128i a64_vq_abs(__m128i x) { return _mm_and_si128(x, _mm_set1_epi32(0x7fffffff)); } +static inline __m128i a64_vq_big(__m128i t) { return a64_vq_ult(_mm_sub_epi32(a64_vq_abs(t), _mm_set1_epi32(0x00800001)), 0x7effffffu); } +static inline __m128i a64_vq_zero(__m128i x) { return _mm_cmpeq_epi32(a64_vq_abs(x), _mm_setzero_si128()); } +static inline __m128i a64_vq_nonfin(__m128i t) { const __m128i e = _mm_set1_epi32(0x7f800000); return _mm_cmpeq_epi32(_mm_and_si128(t, e), e); } +static inline __m128i a64_vq_nan(__m128i x) { return _mm_cmpgt_epi32(a64_vq_abs(x), _mm_set1_epi32(0x7f800000)); } +static inline __m128i a64_vq_subn(__m128i x) { return a64_vq_ult(_mm_sub_epi32(_mm_slli_epi32(x, 1), _mm_set1_epi32(1)), 0x00ffffffu); } +/* an operand's low k lanes, repeated to fill the register: the unused lanes then hold what the used ones do, so every + test is made on all four and no lane computes on bytes the instruction does not read */ +static inline __attribute__((always_inline)) __m128i a64_vq_load(const uint64_t *p, int k) { + if (k == 4) return _mm_loadu_si128((const __m128i *)p); + if (k == 2) { const __m128i x = _mm_loadl_epi64((const __m128i *)p); return _mm_unpacklo_epi64(x, x); } + return _mm_set1_epi32((int)(uint32_t)p[0]); +} +static inline __attribute__((always_inline)) __m128i a64_vq_lane(const uint64_t *p, int index) { /* one lane in all four */ + uint32_t e; + memcpy(&e, (const uint8_t *)p + 4 * index, 4); + return _mm_set1_epi32((int)e); +} +/* the result's low k lanes into Vd, the rest of the register zero */ +static inline __attribute__((always_inline)) void a64_vq_store(uint64_t *d, __m128i r, int k) { + if (k == 4) _mm_storeu_si128((__m128i *)d, r); + else if (k == 2) a64_vst_lo64(d, r); + else a64_vset64(d, (uint32_t)_mm_cvtsi128_si32(r)); +} +static inline __attribute__((always_inline)) int a64_vq_mul(const Ctx *c, uint64_t *d, __m128i a, __m128i b, int k) { + if (!RECOMP_FPX_OPEN(c)) return 0; + const __m128i t = _mm_castps_si128(_mm_mul_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b))); + /* kept: big, or a finite product of a zero (RECOMP_FPX_KEEP32); under FZ no subnormal operand */ + const __m128i keep = _mm_or_si128(a64_vq_big(t), _mm_andnot_si128(a64_vq_nonfin(t), _mm_or_si128(a64_vq_zero(a), a64_vq_zero(b)))); + __m128i bad = _mm_andnot_si128(keep, _mm_set1_epi32(-1)); + if (RECOMP_FPX_FZ(c)) bad = _mm_or_si128(bad, _mm_or_si128(a64_vq_subn(a), a64_vq_subn(b))); + if (_mm_movemask_epi8(bad)) return 0; + a64_vq_store(d, t, k); + A64_VFPX_Q_PROBE(); + return 1; +} +static inline __attribute__((always_inline)) int a64_vq_fma(const Ctx *c, uint64_t *d, __m128i a, __m128i b, int neg, int k) { +#if defined(__FMA__) + if (!RECOMP_FPX_OPEN(c)) return 0; + const __m128i z = a64_vq_load(d, k); + if (neg) a = _mm_xor_si128(a, _mm_set1_epi32((int)0x80000000u)); + const __m128i t = _mm_castps_si128(_mm_fmadd_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), _mm_castsi128_ps(z))); + /* kept: big, or a finite result of a zero factor (RECOMP_FPX_KEEP32); under FZ no subnormal operand */ + const __m128i keep = _mm_or_si128(a64_vq_big(t), _mm_andnot_si128(a64_vq_nonfin(t), _mm_or_si128(a64_vq_zero(a), a64_vq_zero(b)))); + __m128i bad = _mm_andnot_si128(keep, _mm_set1_epi32(-1)); + if (RECOMP_FPX_FZ(c)) bad = _mm_or_si128(bad, _mm_or_si128(_mm_or_si128(a64_vq_subn(a), a64_vq_subn(b)), a64_vq_subn(z))); + if (_mm_movemask_epi8(bad)) return 0; + a64_vq_store(d, t, k); + A64_VFPX_Q_PROBE(); + return 1; +#else + (void)c; (void)d; (void)a; (void)b; (void)neg; (void)k; return 0; /* (no FMA3: a fused op is not one instruction here) */ +#endif +} +static inline __attribute__((always_inline)) int a64_vfpx_q_mul(const Ctx *c, uint64_t *d, const uint64_t *n, const uint64_t *m, int k) { + return a64_vq_mul(c, d, a64_vq_load(n, k), a64_vq_load(m, k), k); +} +static inline __attribute__((always_inline)) int a64_vfpx_q_mule(const Ctx *c, uint64_t *d, const uint64_t *n, const uint64_t *m, int index, int k) { + return a64_vq_mul(c, d, a64_vq_load(n, k), a64_vq_lane(m, index), k); +} +static inline __attribute__((always_inline)) int a64_vfpx_q_fma(const Ctx *c, uint64_t *d, const uint64_t *n, const uint64_t *m, int neg, int k) { + return a64_vq_fma(c, d, a64_vq_load(n, k), a64_vq_load(m, k), neg, k); +} +static inline __attribute__((always_inline)) int a64_vfpx_q_fmae(const Ctx *c, uint64_t *d, const uint64_t *n, const uint64_t *m, int index, int neg, int k) { + return a64_vq_fma(c, d, a64_vq_load(n, k), a64_vq_lane(m, index), neg, k); +} +static inline __attribute__((always_inline)) int a64_vfpx_q_div(const Ctx *c, uint64_t *d, const uint64_t *n, const uint64_t *m, int k) { + if (!RECOMP_FPX_OPEN(c)) return 0; + const __m128i a = a64_vq_load(n, k), b = a64_vq_load(m, k); + const __m128i t = _mm_castps_si128(_mm_div_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b))); + /* kept: big, or a finite quotient of a zero (RECOMP_FPX_KEEPDIV32); under FZ no subnormal operand */ + const __m128i keep = _mm_or_si128(a64_vq_big(t), _mm_andnot_si128(a64_vq_nonfin(t), a64_vq_zero(a))); + __m128i bad = _mm_andnot_si128(keep, _mm_set1_epi32(-1)); + if (RECOMP_FPX_FZ(c)) bad = _mm_or_si128(bad, _mm_or_si128(a64_vq_subn(a), a64_vq_subn(b))); + if (_mm_movemask_epi8(bad)) return 0; + a64_vq_store(d, t, k); + A64_VFPX_Q_PROBE(); + return 1; +} +/* FMAX / FMIN / FMAXNM / FMINNM (the NM forms differ only for a NaN, which is never kept). x86's MAXPS and MINPS give + their second operand for two zeros; taking the op both ways round and combining (and for max, or for min) gives Arm's + zero, and changes nothing where the operands are not both zeros (equal numbers have equal bits). No FPSR bit. */ +static inline __attribute__((always_inline)) int a64_vfpx_q_minmax(const Ctx *c, uint64_t *d, const uint64_t *n, const uint64_t *m, int min, int k) { + if (c->fpcr & RECOMP_FPX_FPCR_MASK) return 0; + const __m128i a = a64_vq_load(n, k), b = a64_vq_load(m, k); + __m128i bad = _mm_or_si128(a64_vq_nan(a), a64_vq_nan(b)); + if (RECOMP_FPX_FZ(c)) bad = _mm_or_si128(bad, _mm_or_si128(a64_vq_subn(a), a64_vq_subn(b))); + if (_mm_movemask_epi8(bad)) return 0; + const __m128 x = _mm_castsi128_ps(a), y = _mm_castsi128_ps(b); + const __m128 r = min ? _mm_or_ps(_mm_min_ps(x, y), _mm_min_ps(y, x)) : _mm_and_ps(_mm_max_ps(x, y), _mm_max_ps(y, x)); + a64_vq_store(d, _mm_castps_si128(r), k); + A64_VFPX_Q_PROBE(); + return 1; +} +/* FCVT Dd, Sn: every binary32 number is a binary64 number. A NaN (quieted, IOC) and, under FZ, a subnormal (flushed, IDC) + are left to the C behind it */ +static inline __attribute__((always_inline)) int a64_vq_s_to_d(const Ctx *c, uint32_t x, uint64_t *r) { + if ((x & 0x7fffffffu) > 0x7f800000u || (RECOMP_FPX_FZ(c) && RECOMP_FPX_SUBN32(x))) return 0; + *r = recomp_fpx_db((double)recomp_fpx_f(x)); + return 1; +} +/* FCVT Sd, Dn: one rounding to nearest. Kept when the result is a normal number above the smallest (no overflow, not + tiny; an inexact result raises IXC, already raised under the open gate), or the input is a zero */ +static inline __attribute__((always_inline)) int a64_vq_d_to_s(uint64_t x, uint32_t *r) { + const uint32_t t = (uint32_t)recomp_fpx_fb((float)recomp_fpx_d(x)); + if (!(RECOMP_FPX_BIG32(t) || RECOMP_FPX_ZERO64(x))) return 0; + *r = t; + return 1; +} +static inline __attribute__((always_inline)) int a64_vfpx_q_cvtsd(const Ctx *c, uint64_t *d, const uint64_t *n) { + uint64_t r; + if ((c->fpcr & RECOMP_FPX_FPCR_MASK) || !a64_vq_s_to_d(c, (uint32_t)n[0], &r)) return 0; + a64_vset64(d, r); + A64_VFPX_Q_PROBE(); + return 1; +} +static inline __attribute__((always_inline)) int a64_vfpx_q_cvtds(const Ctx *c, uint64_t *d, const uint64_t *n) { + uint32_t r; + if (!RECOMP_FPX_OPEN(c) || !a64_vq_d_to_s(n[0], &r)) return 0; + a64_vset64(d, r); + A64_VFPX_Q_PROBE(); + return 1; +} +static inline __attribute__((always_inline)) int a64_vfpx_q_cvtl(const Ctx *c, uint64_t *d, const uint64_t *n, int hi) { + const uint64_t x = n[hi ? 1 : 0]; + uint64_t r0, r1; + if ((c->fpcr & RECOMP_FPX_FPCR_MASK) || !a64_vq_s_to_d(c, (uint32_t)x, &r0) || !a64_vq_s_to_d(c, (uint32_t)(x >> 32), &r1)) return 0; + a64_vset128(d, r0, r1); + A64_VFPX_Q_PROBE(); + return 1; +} +static inline __attribute__((always_inline)) int a64_vfpx_q_cvtn(const Ctx *c, uint64_t *d, const uint64_t *n) { + const uint64_t x0 = n[0], x1 = n[1]; + uint32_t r0, r1; + if (!RECOMP_FPX_OPEN(c) || !a64_vq_d_to_s(x0, &r0) || !a64_vq_d_to_s(x1, &r1)) return 0; + a64_vset64(d, (uint64_t)r1 << 32 | r0); + A64_VFPX_Q_PROBE(); + return 1; +} +#else /* (no SSE2 host path: the C behind each test always runs) */ +#define a64_vfpx_q_mul(c, d, n, m, k) 0 +#define a64_vfpx_q_mule(c, d, n, m, index, k) 0 +#define a64_vfpx_q_fma(c, d, n, m, neg, k) 0 +#define a64_vfpx_q_fmae(c, d, n, m, index, neg, k) 0 +#define a64_vfpx_q_div(c, d, n, m, k) 0 +#define a64_vfpx_q_minmax(c, d, n, m, min, k) 0 +#define a64_vfpx_q_cvtsd(c, d, n) 0 +#define a64_vfpx_q_cvtds(c, d, n) 0 +#define a64_vfpx_q_cvtl(c, d, n, hi) 0 +#define a64_vfpx_q_cvtn(c, d, n) 0 +#endif +""" + + +_installed = False + + +def install(): + """The wrap around a64emit.translate (insn's, installed before) and HEADER at the end of the kit's fpx_header. Once""" + global _installed + if _installed: + return + _installed = True + base_translate, base_header = a64emit.translate, a64_lift.fpx_header + + def translate(items, batch=65536): + texts, flags = base_translate(items, batch) + on = groups() + if not on: + return texts, flags + texts = list(texts) + for k, (pc, w) in enumerate(items): + call = fast_call(w, on) + if call and not flags[k] and not main_handles(texts[k]): + texts[k] = wrap(texts[k], call) + return texts, flags + + def fpx_header(used): + return base_header(used) + HEADER # (after the kit's VSET_HEADER, whose a64_vst_lo64, a64_vset64 and a64_vset128 it uses) + + a64emit.translate, a64_lift.fpx_header = translate, fpx_header diff --git a/nx2/win_manifest.py b/nx2/win_manifest.py index 3b86a85..bfae1c0 100644 --- a/nx2/win_manifest.py +++ b/nx2/win_manifest.py @@ -12,6 +12,14 @@ W functions and the C runtime's file functions take paths of MAX_PATH (260) and folder (the game's own nn::fs paths add the long-path prefix themselves, host/nn_fs.c fs_wide; host/hos_paths.h's and gpu/vk/vk_paths.h's probe too). Without that setting it changes nothing. +(Triforce) HOOKSHOT_SEGMENT_HEAP=1 at the build: heapType SegmentHeap (Windows 10 2004 and newer) puts the whole process on +Windows' segment heap in place of the NT heap, its DLLs and the graphics driver too. Measured on an RX 6700 XT (AMD's Windows +driver, a cold shader preparation): 76% of the driver's compile threads' samples waited on the NT heap's lock, 12 workers ran +slower than 6, and the first part of the preparation took 1,478 s against 982 s on the segment heap; play, four pairs of +runs, within the noise. Not measured on NVIDIA, Intel or Windows 10, and the heap is fixed when the process starts, so the +arms are two exes. On by default since the lab's play A/B on an RTX card (11-22: no fps lost); HOOKSHOT_SEGMENT_HEAP=0: off. The log's "the shaders' preparation runs on" line says which heap the process has +(gpu/vk/prepare_threads.h segment_heap). + python nx2/win_manifest.py [out.res] """ import os @@ -33,6 +41,27 @@ MANIFEST = b""" """ +# ((Triforce) HOOKSHOT_SEGMENT_HEAP=1: the element manifest() puts after longPathAware) +SEGMENT_HEAP = b""" SegmentHeap +""" +_SEGMENT_HEAP_AFTER = b"""true +""" + + +def segment_heap_asked(environ=None): + """(Triforce) The exe on Windows' segment heap unless HOOKSHOT_SEGMENT_HEAP=0 (the NT heap, as before). On by default since the + lab A/B of 11-22 (the Market, 3+3, one build each way on an RTX card: 81.4 against 80.6 fps, 1% low 59.9 against 58.0)""" + return (os.environ if environ is None else environ).get("HOOKSHOT_SEGMENT_HEAP", "").strip() != "0" + + +def manifest(segment_heap=False): + """MANIFEST, with SEGMENT_HEAP in its windowsSettings when segment_heap""" + if not segment_heap: + return MANIFEST + at = MANIFEST.index(_SEGMENT_HEAP_AFTER) + len(_SEGMENT_HEAP_AFTER) + return MANIFEST[:at] + SEGMENT_HEAP + MANIFEST[at:] + + RT_MANIFEST = 24 CREATEPROCESS_MANIFEST_RESOURCE_ID = 1 # (the one an exe's loader reads) @@ -48,9 +77,10 @@ def res_bytes(manifest=MANIFEST): return res_entry(0, 0, b"", 0, 0) + res_entry(RT_MANIFEST, CREATEPROCESS_MANIFEST_RESOURCE_ID, manifest) -def write(path): - """hookshot_manifest.res at path, rewritten only when its bytes change (the link reads it each time)""" - data = res_bytes() +def write(path, segment_heap=None): + """hookshot_manifest.res at path, rewritten only when its bytes change (the link reads it each time). segment_heap: None, + as HOOKSHOT_SEGMENT_HEAP says (segment_heap_asked)""" + data = res_bytes(manifest(segment_heap_asked() if segment_heap is None else segment_heap)) try: with open(path, "rb") as f: if f.read() == data: diff --git a/readme.md b/readme.md index 26824d4..3df320c 100644 --- a/readme.md +++ b/readme.md @@ -253,6 +253,40 @@ F8 to one of the controls yourself, the key stays the game's and the row takes t

+## Achievements + +*On by default.* `HOS_ACHIEVEMENTS=0` in `triforce_env.txt` beside the game turns them off. Things you do in the game are marked as earned, as a console's or a store's achievements are: a notice at the +top of the screen shows the one you earned, with its icon and a sound at the **Menu sounds** row's volume. + +- **The list:** **Achievements** on the System tab of the Options opens a page with every achievement, its icon, the + line about it, and whether and when you earned it. Left, Right, Up and Down move; **L** and **R** turn the page. + +- **25 come with Triforce:** the Spiritual Stones, the Master Sword, each medallion, the songs, Gold Skulltula tokens + and a few items. They are earned by what Link carries, read from the save data once a second while a save is loaded. + Nothing in the game or the save is changed. +- **Earned once, for the game folder:** `triforce_achievements.txt` beside the game keeps them, with the time. Loading + an older save takes none away; delete the file to start over. `triforce_log.txt` has a line for each one earned. +- **Your own:** a file `achievements.txt` beside the game adds more, read at the start: + + ``` + [Name of the achievement] + about: A line about it. + icon: gem blue + item Item_Token_Gold 30 + flag
+ ``` + + A name of up to 40 characters, and 1 to 16 conditions that must all hold: `item []` (an item's + name as `gpu/vk/oot_cheat_items.h` lists it) or `flag ` (a name from the Debug tab's **Game flags** page). The icon is + drawn, not a picture file: a shape (`star`, `sword`, `gem`, `medal`, `note`, `token`, `shield`) and, if you like, a + colour (`gold`, `green`, `red`, `blue`, `purple`, `orange`, `yellow`, `silver`); without the line it is a gold star. + One that does not read is left out, and the log says how many. +- The Debug tab's inventory and flag tools earn them too: they change the same save data. + +They are Triforce's own: Steam and Xbox do not see them. + +

+ ## If something goes wrong | What happens | What to try | diff --git a/scripts/linux/appimage.sh b/scripts/linux/appimage.sh old mode 100644 new mode 100755 index 7644edd..b0da48f --- a/scripts/linux/appimage.sh +++ b/scripts/linux/appimage.sh @@ -78,7 +78,7 @@ cp -a third_party/a64-kit/* "$T/third_party/a64-kit/" rm -f "$T/third_party/a64-kit/tools/lld-link.exe" if [ -d work ]; then mkdir -p "$T/work/image" - for f in hot_funcs.txt hot_order.txt hookshot_pieces.prof pipeline_states.bin; do [ -f "work/$f" ] && cp "work/$f" "$T/work/"; done + for f in hot_funcs.txt hot_order.txt hookshot_pieces.prof triforce_vk.profdata pipeline_states.bin; do [ -f "work/$f" ] && cp "work/$f" "$T/work/"; done [ -f work/image/chunks.txt ] && cp work/image/chunks.txt "$T/work/image/" fi find "$T" -name __pycache__ -prune -exec rm -rf {} + diff --git a/scripts/linux/check_player_build.sh b/scripts/linux/check_player_build.sh old mode 100644 new mode 100755 diff --git a/setup/Setup.cs b/setup/Setup.cs index c50415f..83cbfd6 100644 --- a/setup/Setup.cs +++ b/setup/Setup.cs @@ -352,7 +352,6 @@ static class Install /// Removes Setup, its tools and the shortcuts. Game folders and saves are the player's and stay. public static bool Uninstall(bool quiet) { - var dir = InstalledDir ?? SelfDir; if (!quiet && MessageBox.Show("Remove Triforce Setup and its tools from this PC?\n\nYour game folders and saves are not touched.", "Uninstall Triforce", MessageBoxButton.YesNo, MessageBoxImage.Question) != MessageBoxResult.Yes) return false; foreach (var f in new[] { DesktopLink }) @@ -363,10 +362,22 @@ static class Install try { if (Directory.Exists(work)) Directory.Delete(work, true); } catch (Exception) { } Remove(UninstallKey); Remove(SettingsKey); - // (this exe is running from the folder: a short-lived cmd removes it once we are gone) - if (File.Exists(Path.Combine(dir, Program.SetupAsset))) - Process.Start(new ProcessStartInfo("cmd.exe", "/c timeout /t 2 /nobreak >nul & rmdir /s /q \"" + dir + "\"") + // (this exe is running from the folder: a short-lived cmd removes it once we are gone. (Triforce, 11-21, ticket + // "Downloads folder got deleted while uninstalling the Triforce launcher") Only the installed Setup's folder, never the + // folder a Setup that is not installed runs from (Downloads), and only Setup's own entries there, the folder itself + // only when nothing else is left (rmdir without /s): the whole folder went before, with whatever the player kept there) + if (InstalledDir is { } inst && File.Exists(Path.Combine(inst, Program.SetupAsset))) + { + var d = Path.GetFullPath(inst).TrimEnd('\\', '/'); + var own = string.Equals(Path.GetFileName(d), "Triforce", StringComparison.OrdinalIgnoreCase); + var line = "/c timeout /t 2 /nobreak >nul"; + foreach (var f in new[] { "TriforceSetup*.exe", "TriforceSetup*.exe.*", "setup.log", "setup.log.*" }) + line += " & del /f /q \"" + Path.Combine(d, f) + "\" 2>nul"; + if (own) + foreach (var f in new[] { "tools", "downloads", "ngx-sdk" }) line += " & rmdir /s /q \"" + Path.Combine(d, f) + "\" 2>nul"; + Process.Start(new ProcessStartInfo("cmd.exe", line + " & rmdir \"" + d + "\" 2>nul") { CreateNoWindow = true, UseShellExecute = false, WorkingDirectory = Path.GetTempPath() }); + } if (!quiet) MessageBox.Show("Triforce has been removed.", "Uninstall Triforce", MessageBoxButton.OK, MessageBoxImage.Information); return true; } @@ -774,7 +785,12 @@ sealed class Runner process = p; p.ErrorDataReceived += (_, e) => { if (e.Data != null) Line?.Invoke(e.Data); }; p.BeginErrorReadLine(); - try { p.PriorityClass = ProcessPriorityClass.BelowNormal; } catch (Exception) { } + // (Triforce, 11-22, the performance audit's OS-57, after Source A's) The build tools run at normal priority; + // below normal, as every build and update was before, only with HOOKSHOT_LOW_PRIORITY=1. Source A's reason, + // a report it did not measure: below normal, Windows kept the steps that are one process (the read, the + // lift) on a hybrid Intel CPU's efficiency cores while the performance cores idled. + if (Environment.GetEnvironmentVariable("HOOKSHOT_LOW_PRIORITY") == "1") + try { p.PriorityClass = ProcessPriorityClass.BelowNormal; } catch (Exception) { } for (string? line; (line = p.StandardOutput.ReadLine()) != null;) { if (line.StartsWith("@@error ")) error = line.Substring(8); diff --git a/setup/pack_tools.ps1 b/setup/pack_tools.ps1 index f7dc52c..ca3f436 100644 --- a/setup/pack_tools.ps1 +++ b/setup/pack_tools.ps1 @@ -56,7 +56,7 @@ foreach ($d in 'python', 'cpm', 'wgpu-native', 'llvm-mingw', 'third_party') { # 3. the speed profile (hot functions, link order, play profile, chunk list) and the Triforce icon for the game's exe $work = Join-Path $tools 'work' New-Item -ItemType Directory -Force $work | Out-Null -foreach ($n in 'hot_funcs.txt', 'hot_order.txt', 'hookshot_pieces.prof') { +foreach ($n in 'hot_funcs.txt', 'hot_order.txt', 'hookshot_pieces.prof', 'triforce_vk.profdata') { if (Test-Path (Join-Path $repo "work\$n")) { Copy-Item (Join-Path $repo "work\$n") $work } } # 3b. the pipeline state list (nx2/pipeline_states.py --out work\pipeline_states.bin ): nx2/make_game.py puts it diff --git a/tests/achievement_rows_test.cpp b/tests/achievement_rows_test.cpp new file mode 100644 index 0000000..56377a0 --- /dev/null +++ b/tests/achievement_rows_test.cpp @@ -0,0 +1,99 @@ +// gpu/vk/achievement_rows.h alone (tests/test_achievements.py): the list as the host names it, one earned and its notice, +// the page's choice and pages, and how a notice comes in and goes. +#include +#include +#include "achievement_rows.h" + +static int fails; +#define CHECK(c) do { if (!(c)) { printf("FAIL line %d: %s\n", __LINE__, #c); fails++; } } while (0) + +namespace ar = achievement_rows; +static ar::List list; + +static void say(const char* text) { + for (const char* p = text; *p; p++) list.text((unsigned char)*p); +} + +int main() { + // nothing shown until the list is whole; then as named, in order + say("First\nThe line about it.\n2026-11-18"); + list.add(1 | 3 << 4 | ar::kEarned); + say("Second\nAnother line.\n"); + list.add(5); + say("Third"); // (no line about it and no day) + list.add(2 | 9 << 4); + list.add(0); // (no name at all) + CHECK(list.shown() == 0 && list.earned_count() == 0 && list.take_fresh() == -1); + CHECK(!list.earn(0) && list.take_fresh() == -1); // (earned before the list is whole: no such place yet) + list.done(4); + CHECK(list.shown() == 4 && list.earned_count() == 1); + CHECK(!strcmp(list.names[0], "First") && !strcmp(list.abouts[0], "The line about it.") && !strcmp(list.days[0], "2026-11-18")); + CHECK(list.is_earned(0) && ar::shape_of(list.icons[0]) == ar::kSword && ar::tone_of(list.icons[0]) == ar::kBlue); + CHECK(!strcmp(list.names[1], "Second") && !strcmp(list.abouts[1], "Another line.") && !list.days[1][0] && !list.is_earned(1)); + CHECK(ar::shape_of(list.icons[1]) == ar::kToken && ar::tone_of(list.icons[1]) == ar::kGold); + CHECK(!strcmp(list.names[2], "Third") && !list.abouts[2][0] && !list.days[2][0]); + CHECK(ar::shape_of(list.icons[2]) == ar::kGem && ar::tone_of(list.icons[2]) == ar::kGold); // (a tone that is none: gold) + CHECK(!strcmp(list.names[3], "(no name)") && ar::shape_of(15) == ar::kStar && ar::shape_of(ar::kShield) == ar::kShield); + // a list sent again changes nothing + say("Again\nx\n"); + list.add(0); + list.done(1); + CHECK(list.shown() == 4 && !strcmp(list.names[0], "First")); + + // one earned: its day, and the notice's once + say("2026-11-19"); + CHECK(list.earn(1) && list.is_earned(1) && !strcmp(list.days[1], "2026-11-19") && list.earned_count() == 2); + CHECK(list.take_fresh() == 1 && list.take_fresh() == -1); + say("2030-01-01"); + CHECK(list.earn(0) && !strcmp(list.days[0], "2026-11-18") && list.take_fresh() == 0); // (earned before: the day it was) + say("junk"); + CHECK(!list.earn(4) && !list.earn(-1) && list.take_fresh() == -1 && list.earned_count() == 2); + say("2026-11-20"); + CHECK(list.earn(2) && !strcmp(list.days[2], "2026-11-20")); // (the bytes of the refused ones are not this one's day) + CHECK(list.take_fresh() == 2); + + // a name, a line and a day longer than their room: cut, a name never inside a letter + static ar::List wide; + for (int i = 0; i < 39; i++) wide.text('n'); + wide.text(0xC3), wide.text(0xA9), wide.text(0xC3), wide.text(0xA9); // (two letters of two bytes: the second does not fit) + wide.text('\n'); + for (int i = 0; i < 400; i++) wide.text('a'); + wide.add(0); + wide.done(1); + CHECK(strlen(wide.names[0]) == 39 && strlen(wide.abouts[0]) <= (size_t)ar::kAbout - 1 && wide.shown() == 1); + // more than there is room for: the first kSlots + static ar::List many; + for (int i = 0; i < ar::kSlots + 5; i++) many.text('a' + i % 26), many.add(0); + many.done(ar::kSlots + 5); + CHECK(many.shown() == ar::kSlots); + many.done(-3); + CHECK(many.shown() == ar::kSlots); + + // the page: 2 cards to a row, 6 to a page, 14 of them + ar::View v; + CHECK(ar::View::pages(14, 6) == 3 && ar::View::pages(12, 6) == 2 && ar::View::pages(0, 6) == 1 && ar::View::pages(5, 0) == 5); + CHECK(!v.move(-1, 0, 0, false, false, 14, 2, 6) && v.choice == 0); // (Left at the first: stays) + CHECK(v.move(1, 0, 0, false, false, 14, 2, 6) && v.choice == 1 && v.page == 0); + CHECK(v.move(0, 1, 0, false, false, 14, 2, 6) && v.choice == 3); // (Down: a row) + CHECK(v.move(0, 2, 0, false, false, 14, 2, 6) && v.choice == 7 && v.page == 1); // (onto the next page: shown) + CHECK(v.move(0, 0, 1, false, false, 14, 2, 6) && v.choice == 13 && v.page == 2); // (a page on, kept to the last card) + CHECK(!v.move(0, 0, 1, false, false, 14, 2, 6) && v.choice == 13); + CHECK(v.move(0, 0, 0, true, false, 14, 2, 6) && v.choice == 0 && v.page == 0); // (Home) + CHECK(v.move(0, 0, 0, false, true, 14, 2, 6) && v.choice == 13 && v.page == 2); // (End) + CHECK(v.turn(-1, 14, 6) && v.page == 1 && v.choice == 13); // (the wheel: the page, not the choice) + CHECK(v.turn(-5, 14, 6) && v.page == 0 && !v.turn(-1, 14, 6) && v.turn(9, 14, 6) && v.page == 2); + CHECK(v.choose(4, 14) && v.choice == 4 && !v.choose(4, 14) && !v.choose(14, 14) && !v.choose(-1, 14)); + v.keep(3, 6); // (a shorter list under the choice) + CHECK(v.choice == 2 && v.page == 0); + CHECK(!v.move(1, 0, 0, false, false, 0, 2, 6)); + v.keep(0, 6); + CHECK(v.choice == 0 && v.page == 0); + + // a notice: in over a quarter of a second, there, out over the last 300 ms, gone + CHECK(ar::notice_in(-1, 4000) == 0 && ar::notice_in(0, 4000) == 0 && ar::notice_in(4000, 4000) == 0 && ar::notice_in(9000, 4000) == 0); + CHECK(ar::notice_in(125, 4000) == 0.5f && ar::notice_in(250, 4000) == 1 && ar::notice_in(2000, 4000) == 1 && ar::notice_in(3700, 4000) == 1); + CHECK(ar::notice_in(3850, 4000) == 0.5f); + + if (!fails) printf("PASS\n"); + return fails != 0; +} diff --git a/tests/achievements_host_test.c b/tests/achievements_host_test.c new file mode 100644 index 0000000..a53c5f9 --- /dev/null +++ b/tests/achievements_host_test.c @@ -0,0 +1,174 @@ +/* Achievements on the host's side: host/achievements.c's own text (achievements_actual.inc, cut by + tests/test_achievements.py) against a stand-in save data, a stand-in menu and a game folder of the test's. + Run three times: "off" with HOS_ACHIEVEMENTS=0, then "first" and "second" with it on the same folder, the second the + next start. */ +#include +#include +#include +#include +#include +#include + +static int fails; +#define CHECK(c) do { if (!(c)) { printf("FAIL line %d: %s\n", __LINE__, #c); fails++; } } while (0) + +/* ---- what host/achievements.c takes from hos.h, hos_paths.h, windows.h and menu_sound.c ---- */ +#define MAX_PATH 260 +typedef unsigned long DWORD; +#define MOVEFILE_REPLACE_EXISTING 1 +static char game_dir[400]; +static DWORD hos_game_path_a(char *buf, DWORD size) { return (DWORD)snprintf(buf, size, "%s/Triforce.exe", game_dir); } +static int MoveFileExA(const char *from, const char *to, int flags) { + (void)flags; + remove(to); + return rename(from, to) == 0; +} +static char logged[1 << 14]; +static void hos_log(const char *fmt, ...) { + va_list ap; + va_start(ap, fmt); + const size_t at = strlen(logged); + vsnprintf(logged + at, sizeof logged - at - 2, fmt, ap); + va_end(ap); + strcat(logged, "\n"); +} +static int said(const char *text) { /* how often the log has it */ + int n = 0; + for (const char *p = logged; (p = strstr(p, text)) != NULL; p++) n++; + return n; +} +#include "achievements_constants.inc" +static int sounds; +static int64_t sound_at; +static void hos_menu_sound_play(int id, int64_t now_ns) { + CHECK(id == HOS_MENU_SOUND_SYS_DECIDE); + sounds++, sound_at = now_ns; +} + +/* ---- the menu's side of HOS_OPT_ACH_* (gpu/vk/achievement_rows.h is tested by itself) ---- */ +static int renderer; /* 0: no renderer yet (rx_option answers what it is given) */ +static char incoming[600], row_text[160][400], notice_day[16][32]; +static int incoming_n, row_bits[160], rows_added, rows_shown = -1, notices[16], nnotices; +int rx_option(int id, int otherwise) { + if (!renderer) return otherwise; + CHECK(id == HOS_OPT_ACH_ASK); + return 1; +} +void rx_option_set(int id, int value) { + CHECK(renderer); + if (id == HOS_OPT_ACH_TEXT) { + CHECK(value > 0 && value < 256 && incoming_n + 1 < (int)sizeof incoming); + incoming[incoming_n++] = (char)value; + } else if (id == HOS_OPT_ACH_ADD) { + incoming[incoming_n] = 0, incoming_n = 0; + CHECK(rows_shown < 0 && rows_added < 160); + snprintf(row_text[rows_added], sizeof row_text[0], "%s", incoming); + row_bits[rows_added++] = value; + } else if (id == HOS_OPT_ACH_COUNT) { + CHECK(rows_shown < 0 && !incoming_n); + rows_shown = value; + } else if (id == HOS_OPT_ACH_EARNED) { + incoming[incoming_n] = 0, incoming_n = 0; + CHECK(rows_shown >= 0 && value >= 0 && value < rows_shown && nnotices < 16); /* (never before the list is whole) */ + snprintf(notice_day[nnotices], sizeof notice_day[0], "%s", incoming); + notices[nnotices++] = value; + } else CHECK(!"an option of another feature set"); +} + +#include "achievements.h" +#include "achievements_actual.inc" + +/* ---- a stand-in for the save data ---- */ +static struct { int kind; const char *what; int value; } have[16]; +static int nhave, asked; +static void set(int kind, const char *what, int value) { have[nhave].kind = kind, have[nhave].what = what, have[nhave++].value = value; } +static int probe(void *ctx, int kind, const char *what) { + (void)ctx; + asked++; + for (int i = 0; i < nhave; i++) + if (have[i].kind == kind && !strcmp(have[i].what, what)) return have[i].value; + return -1; +} +#define S 1000000000LL +static void at(int64_t seconds_x10) { hos_achievements_check(probe, NULL, seconds_x10 * (S / 10)); } +static int row_of(const char *name) { /* the place the menu was given an achievement at, -1 none */ + const size_t n = strlen(name); + for (int i = 0; i < rows_added; i++) + if (!strncmp(row_text[i], name, n) && row_text[i][n] == '\n') return i; + return -1; +} +static int is_day(const char *text) { /* "2026-11-18" */ + return strlen(text) == 10 && text[4] == '-' && text[7] == '-' && text[0] == '2'; +} + +static char file_text[8192]; +static const char *file(const char *name) { /* a file of the game folder as it is now ("" when there is none) */ + char path[600]; + snprintf(path, sizeof path, "%s/%s", game_dir, name); + FILE *f = fopen(path, "rb"); + file_text[f ? fread(file_text, 1, sizeof file_text - 1, f) : 0] = 0; + if (f) fclose(f); + return file_text; +} +static int lines(const char *text) { + int n = 0; + for (; *text; text++) n += *text == '\n'; + return n; +} + +int main(int argc, char **argv) { + if (argc != 3) return 2; + snprintf(game_dir, sizeof game_dir, "%s", argv[1]); + set(ACH_ITEM, "Weapon_Sword_Kokiri", 1), set(ACH_ITEM, "Item_Stone_Forest", 1), set(ACH_ITEM, "Item_Stone_Fire", 1); + set(ACH_ITEM, "Item_Stone_Water", 1), set(ACH_FLAG, "My_Flag", 1), set(ACH_ITEM, "Item_Token_Gold", 9); + if (!strcmp(argv[2], "off")) { /* turned off: nothing asked of the save data or the menu, nothing read, written or said */ + CHECK(!hos_achievements_on()); + renderer = 1; + hos_achievements_frame(), at(10), hos_achievements_frame(), at(100); + CHECK(!asked && !logged[0] && !rows_added && rows_shown < 0 && !nnotices && !sounds && !ready && !file("triforce_achievements.txt")[0]); + } else if (!strcmp(argv[2], "first")) { + CHECK(hos_achievements_on()); + hos_achievements_frame(); /* the first present, before the renderer answers: the lists are read, nothing is named */ + CHECK(said("achievements: 26 (1 of achievements.txt), 0 earned so far") == 1 && said("achievements: 2 left out") == 1); + CHECK(!rows_added && rows_shown < 0); + at(20); /* six earned before the menu has the list: written and said at once, shown only once it has */ + CHECK(said("achievements: earned \"") == 6 && said("earned \"A Sword of Your Own\" (1 of 26)") == 1 && said("earned \"Mine\" (6 of 26)") == 1); + CHECK(said("earned \"Three Stones\"") == 1 && !said("Spider Hunter") && !said("could not be written")); + CHECK(lines(file("triforce_achievements.txt")) == 7 && strstr(file_text, " A Sword of Your Own\n") && strstr(file_text, " Mine\n")); + CHECK(!nnotices && !sounds); + renderer = 1; + hos_achievements_frame(); /* the list named, whole, in its order: name, the line about it, the day; the icon and earned */ + CHECK(rows_added == 26 && rows_shown == 26 && !nnotices); + CHECK(!strncmp(row_text[0], "A Sword of Your Own\nFind a blade in the forest.\n", 48) && is_day(row_text[0] + 48)); + CHECK(row_bits[0] == (1 | 1 << 4 | HOS_ACH_EARNED)); /* (sword green, earned) */ + CHECK(!strcmp(row_text[1], "First Melody\nLearn the royal family's song.\n") && row_bits[1] == 4); /* (a note, gold; not earned: no day) */ + CHECK(row_of("Mine") == 25 && row_bits[25] == (2 | 3 << 4 | HOS_ACH_EARNED) && row_bits[row_of("Twice as Tough")] == (6 | 7 << 4)); + hos_achievements_frame(), hos_achievements_frame(); + CHECK(rows_added == 26); /* (named once) */ + /* shown one at a time, each for its 4.5 s, in the list's order, with the sound and the day */ + at(30); + CHECK(nnotices == 1 && sounds == 1 && sound_at == 3 * S && notices[0] == 0 && is_day(notice_day[0])); + at(40), at(74); + CHECK(nnotices == 1); + at(75); + CHECK(nnotices == 2 && sounds == 2 && notices[1] == row_of("Forest's Thanks")); + for (int t = 120; t <= 400; t += 10) at(t); + CHECK(nnotices == 6 && sounds == 6 && notices[5] == 25); + CHECK(said("achievements: earned \"") == 6 && said("achievements: 26 (") == 1 && lines(file("triforce_achievements.txt")) == 7); + } else { + renderer = 1; + hos_achievements_frame(); /* the next start: what was earned is, and is neither said nor shown again */ + CHECK(said("achievements: 26 (1 of achievements.txt), 6 earned so far") == 1 && rows_shown == 26); + at(10); + CHECK(!said("earned \"") && !nnotices && !sounds); + CHECK((row_bits[0] & HOS_ACH_EARNED) && !(row_bits[row_of("Spider Hunter")] & HOS_ACH_EARNED)); + have[5].value = 10; /* (the tenth token) */ + at(20); + CHECK(said("earned \"Spider Hunter\" (7 of 26)") == 1 && nnotices == 1 && notices[0] == row_of("Spider Hunter") && is_day(notice_day[0])); + /* the file whole again: the seven, and the line of an achievement no list has now, as it was */ + CHECK(lines(file("triforce_achievements.txt")) == 9 && strstr(file_text, " Spider Hunter\n") && strstr(file_text, "\ndeadbeef 5 Gone\n")); + CHECK(!file("triforce_achievements.txt.new")[0]); + } + if (!fails) printf("PASS\n"); + return fails != 0; +} diff --git a/tests/achievements_test.c b/tests/achievements_test.c new file mode 100644 index 0000000..ae5e155 --- /dev/null +++ b/tests/achievements_test.c @@ -0,0 +1,190 @@ +/* host/achievements.h alone (tests/test_achievements.py): the list's form and what is refused, when an achievement is + earned, and the file that keeps the earned ones. With "names" as its argument it prints the built-in list's + conditions instead, a line each, for the test to hold against gpu/vk/oot_cheat_items.h. */ +#include +#include +#include +#include "achievements.h" + +static int fails; +#define CHECK(c) do { if (!(c)) { printf("FAIL line %d: %s\n", __LINE__, #c); fails++; } } while (0) + +static AchList list, again; + +/* ---- a stand-in for the save data ---- */ +static struct { int kind; const char *what; int value; } have[16]; +static int nhave, asked; +static void set(int kind, const char *what, int value) { + for (int i = 0; i < nhave; i++) + if (have[i].kind == kind && !strcmp(have[i].what, what)) { have[i].value = value; return; } + have[nhave].kind = kind, have[nhave].what = what, have[nhave++].value = value; +} +static int probe(void *ctx, int kind, const char *what) { + CHECK(ctx == (void *)&asked); + asked++; + for (int i = 0; i < nhave; i++) + if (have[i].kind == kind && !strcmp(have[i].what, what)) return have[i].value; + return -1; +} + +static const char FORM[] = + "stray line\n# a comment\n[One]\r\nabout: First. \r\nitem A_b 3\r\n\tflag F.x-1\r\n\r\n" + "[No conditions]\nabout: x\n" + "[Bad line]\nitem A\nwhat is this\n" + "[One]\nitem Dup\n" + "[]\nitem A\n" + "[A name that is too long to be an achievement's]\nitem A\n" + "[Bad count]\nitem A 0\n" + "[Bad count 2]\nitem A 10000\n" + "[Bad count 3]\nitem A 3x\n" + "[Flag count]\nflag F 2\n" + "[Bad name]\nitem A$b\n" + "[No name]\nitem\n" + "[No bracket\nitem A\n" + "[ Last ]\n# its line about it is left out\nitem Z 2"; + +static void built_in(int print) { + int refused = 0; + memset(&list, 0, sizeof list); + const int n = ach_parse(&list, ACH_BUILTIN, &refused); + if (print) { + for (int i = 0; i < list.n; i++) + for (int k = 0; k < list.a[i].nconds; k++) printf("%c %s %d\n", list.a[i].conds[k].kind, list.a[i].conds[k].what, list.a[i].conds[k].least); + return; + } + CHECK(n == 25 && list.n == 25 && refused == 0); + for (int i = 0; i < list.n; i++) { + const Ach *a = &list.a[i]; + CHECK(a->name[0] && a->about[0] && a->nconds >= 1 && !a->earned && a->key == ach_key(a->name)); + CHECK(ach_find(&list, a->key) == i); + for (int k = 0; k < a->nconds; k++) CHECK(a->conds[k].kind == ACH_ITEM && a->conds[k].least >= 1); + } + CHECK(ach_key("") == 2166136261u && ach_key("a") == 0xe40c292cu); + CHECK(list.a[0].icon == (1 | 1 << 4) && list.a[1].icon == 4 && list.a[24].icon == (6 | 7 << 4)); /* (sword green, a note, shield silver) */ +} + +static void form(void) { + int refused = 0; + memset(&list, 0, sizeof list); + CHECK(ach_parse(&list, FORM, &refused) == 2 && list.n == 2); + CHECK(refused == 12); + const Ach *one = &list.a[0], *last = &list.a[1]; + CHECK(!strcmp(one->name, "One") && !strcmp(one->about, "First.") && one->nconds == 2); + CHECK(one->conds[0].kind == ACH_ITEM && !strcmp(one->conds[0].what, "A_b") && one->conds[0].least == 3); + CHECK(one->conds[1].kind == ACH_FLAG && !strcmp(one->conds[1].what, "F.x-1") && one->conds[1].least == 1); + CHECK(!strcmp(last->name, "Last") && !last->about[0] && last->nconds == 1 && last->conds[0].least == 2 && !strcmp(last->conds[0].what, "Z")); + CHECK(ach_parse(&list, "", &refused) == 0 && ach_parse(&list, "\n\n# nothing\n", &refused) == 0 && refused == 12 && list.n == 2); + CHECK(ach_parse(&list, "[Whole]\nitem W\n", NULL) == 1 && list.n == 3); + + /* an icon: a shape, and a tone or none (gold); without the line a gold star; anything else is no achievement */ + CHECK(ach_icon_parse("star", 4) == 0 && ach_icon_parse("gem blue", 8) == (2 | 3 << 4) && ach_icon_parse("shield \t silver", 15) == (6 | 7 << 4)); + CHECK(ach_icon_parse("", 0) < 0 && ach_icon_parse("circle", 6) < 0 && ach_icon_parse("gem pink", 8) < 0 && ach_icon_parse("gem blue x", 10) < 0); + CHECK(ach_icon_parse("gemblue", 7) < 0 && ach_icon_parse("Gem", 3) < 0 && ach_icon_parse("blue", 4) < 0); + CHECK(one->icon == 0 && list.a[2].icon == 0); + refused = 0; + CHECK(ach_parse(&list, "[Pictured]\nicon: medal purple \nitem P\n[Unpictured]\nicon: medal mauve\nitem P\n[Bare]\nicon:\nitem P\n", &refused) == 1); + CHECK(refused == 2 && list.a[list.n - 1].icon == (3 | 4 << 4)); + refused = 12; + + /* ACH_CONDS conditions are an achievement, one more is none; a name of ACH_NAME bytes is one; a name's 95 bytes */ + static char text[1 << 16]; + for (int over = 0; over < 2; over++) { + int at = snprintf(text, sizeof text, "[%s]\n", over ? "Over" : "Full"); + for (int i = 0; i < ACH_CONDS + over; i++) at += snprintf(text + at, sizeof text - (size_t)at, "item I%d\n", i); + refused = 0; + CHECK(ach_parse(&list, text, &refused) == !over && refused == over); + } + CHECK(ach_parse(&list, "[0123456789012345678901234567890123456789]\nitem A\n", NULL) == 1 && strlen(list.a[list.n - 1].name) == ACH_NAME); + memset(text, 'n', sizeof text); + memcpy(text, "[Long]\nflag ", 12); + text[12 + ACH_WHAT - 1] = 0; + CHECK(ach_parse(&list, text, NULL) == 1 && strlen(list.a[list.n - 1].conds[0].what) == ACH_WHAT - 1); + text[12 + ACH_WHAT - 1] = 'n', text[12 + ACH_WHAT] = 0; + memcpy(text, "[Lon2]", 6); + refused = 0; + CHECK(ach_parse(&list, text, &refused) == 0 && refused == 1); + + /* no more than ACH_MAX */ + memset(&list, 0, sizeof list); + int at = 0; + for (int i = 0; i < ACH_MAX + 2; i++) at += snprintf(text + at, sizeof text - (size_t)at, "[N%d]\nitem A\n", i); + refused = 0; + CHECK(ach_parse(&list, text, &refused) == ACH_MAX && list.n == ACH_MAX && refused == 2); +} + +static void earning(void) { + int fresh[4]; + memset(&list, 0, sizeof list); + CHECK(ach_parse(&list, FORM, NULL) == 2); + CHECK(ach_check(&list, probe, &asked, 100, fresh, 4) == 0 && !list.a[0].earned && !list.a[1].earned); + set(ACH_ITEM, "A_b", 3); + CHECK(ach_check(&list, probe, &asked, 100, fresh, 4) == 0); /* (the flag is not in the save data) */ + set(ACH_FLAG, "F.x-1", 0); + CHECK(ach_check(&list, probe, &asked, 100, fresh, 4) == 0); + set(ACH_FLAG, "A_b", 1), set(ACH_ITEM, "F.x-1", 9); /* (an item of a flag's name is not that flag) */ + CHECK(ach_check(&list, probe, &asked, 100, fresh, 4) == 0); + set(ACH_FLAG, "F.x-1", 1), set(ACH_ITEM, "A_b", 2); + CHECK(ach_check(&list, probe, &asked, 100, fresh, 4) == 0); /* (one too few) */ + set(ACH_ITEM, "A_b", 3); + CHECK(ach_check(&list, probe, &asked, 101, fresh, 4) == 1 && fresh[0] == 0 && list.a[0].earned && list.a[0].when == 101 && !list.a[1].earned); + CHECK(ach_earned_count(&list) == 1); + /* earned stays earned, and is not asked about again */ + set(ACH_ITEM, "A_b", 0), set(ACH_FLAG, "F.x-1", 0); + asked = 0; + CHECK(ach_check(&list, probe, &asked, 102, fresh, 4) == 0 && asked == 1 && list.a[0].earned && list.a[0].when == 101); + set(ACH_ITEM, "Z", 1); + CHECK(ach_check(&list, probe, &asked, 102, fresh, 4) == 0); + set(ACH_ITEM, "Z", 7); + CHECK(ach_check(&list, probe, &asked, 103, fresh, 4) == 1 && fresh[0] == 1 && list.a[1].when == 103 && ach_earned_count(&list) == 2); + asked = 0; + CHECK(ach_check(&list, probe, &asked, 104, fresh, 4) == 0 && asked == 0); + + /* no more than the caller has room for: the rest at the next call */ + memset(&list, 0, sizeof list); + CHECK(ach_parse(&list, "[P]\nitem Z\n[Q]\nitem Z\n[R]\nitem Z\n", NULL) == 3); + CHECK(ach_check(&list, probe, &asked, 1, fresh, 2) == 2 && fresh[0] == 0 && fresh[1] == 1 && !list.a[2].earned); + CHECK(ach_check(&list, probe, &asked, 2, fresh, 2) == 1 && fresh[0] == 2 && list.a[2].when == 2); +} + +static void kept(void) { + static char text[4096], more[4096]; + int fresh[4]; + memset(&list, 0, sizeof list); + CHECK(ach_parse(&list, FORM, NULL) == 2); + size_t n = ach_earned_write(&list, text, sizeof text); + CHECK(n == strlen(text) && text[0] == '#' && strchr(text, '\n') == text + n - 1); /* (none earned: the first line alone) */ + set(ACH_ITEM, "Z", 2); + CHECK(ach_check(&list, probe, &asked, 1700000000, fresh, 4) == 1); + n = ach_earned_write(&list, text, sizeof text); + char line[80]; + snprintf(line, sizeof line, "\n%08x 1700000000 Last\n", (unsigned)ach_key("Last")); + CHECK(n && strstr(text, line) && !strstr(text, " One\n")); + CHECK(ach_earned_write(&list, more, 40) == 0 && ach_earned_write(&list, more, n) == 0 && ach_earned_write(&list, more, n + 1) == n); + + /* the next start: the same lists, the file read */ + memset(&again, 0, sizeof again); + CHECK(ach_parse(&again, FORM, NULL) == 2); + CHECK(ach_earned_read(&again, text) == 1 && !again.a[0].earned && again.a[1].earned && again.a[1].when == 1700000000 && !again.norphans); + CHECK(ach_earned_read(&again, text) == 0); /* (read twice: marked once) */ + + /* a line of an achievement no list has is kept as it is, once; what is not such a line is dropped */ + snprintf(more, sizeof more, "%sdeadbeef 5 Gone from the lists\r\nhello\n1234 5 short key\ndeadbeef 6 Gone again\n0000000g 1 x\n%08x -3 One\n", text, + (unsigned)ach_key("One")); + memset(&again, 0, sizeof again); + CHECK(ach_parse(&again, FORM, NULL) == 2); + CHECK(ach_earned_read(&again, more) == 2 && again.a[0].earned && again.a[0].when == 0 && again.norphans == 1); + CHECK(!strcmp(again.orphans[0], "deadbeef 5 Gone from the lists")); + n = ach_earned_write(&again, text, sizeof text); + CHECK(n && strstr(text, "\ndeadbeef 5 Gone from the lists\n") && !strstr(text, "hello") && !strstr(text, "short key") && !strstr(text, "Gone again")); + CHECK(strstr(text, " 0 One\n") && strstr(text, " 1700000000 Last\n")); +} + +int main(int argc, char **argv) { + if (argc > 1 && !strcmp(argv[1], "names")) return built_in(1), 0; + built_in(0); + form(); + earning(); + kept(); + if (!fails) printf("PASS\n"); + return fails != 0; +} diff --git a/tests/audio_mix_effects_test.c b/tests/audio_mix_effects_test.c new file mode 100644 index 0000000..b328a16 --- /dev/null +++ b/tests/audio_mix_effects_test.c @@ -0,0 +1,297 @@ +/* Authored mixes, packets and samples around the actual effect bodies and setters. No audio device or game is opened. + Run with HOS_AUDIO_EFFECTS=0 it checks that every effect leaves the samples as they were. */ +#include +#include +#include +#include +#include + +typedef struct { uint64_t x[32]; uint8_t vreg[32][16]; } Ctx; +static _Alignas(8) uint8_t guest[65536]; +#define CHECK(x) do { if (!(x)) { fprintf(stderr, "FAIL:%d %s\n", __LINE__, #x); exit(1); } } while (0) +static void *guest_ptr(uint64_t address) { CHECK(address < sizeof guest); return guest + address; } +#define GP(a) guest_ptr(a) +#define G64(a) (*(uint64_t *)GP(a)) +#define G32(a) (*(uint32_t *)GP(a)) +#define ARG(n) (c->x[n]) +#define RET(v) (c->x[0] = (uint64_t)(v)) +#define HLE(sym) void imp_##sym(Ctx *c) +#define AUT(...) ((void)0) + +/* Single-threaded test boundaries: the lock is checked for balance only. */ +static int lock_depth; +static void au_lock(void) { CHECK(lock_depth++ == 0); } +static void au_unlock(void) { CHECK(--lock_depth == 0); } + +#include "actual_audio_effects.inc" + +/* the voice's source: one authored sample a frame (render_voice's resampler at 1.0 hands them through a frame late) */ +static float tone[4096]; +static int tone_pos; +static int next_source_frame(Voice *v) { + v->cur[0] = tone[tone_pos++ % 4096]; + return 1; +} + +#define PI 3.14159265358979f +enum { CONFIG = 0x10, MIX0 = 0x40, MIX1 = 0x50, BQ = 0x100, DL = 0x110, RV = 0x120, SPLIT = 0x130, PACKET = 0x200, + IN_CH = 0x300, OUT_CH = 0x310, BUFFER = 0x1000 }; +static int effects; /* (HOS_AUDIO_EFFECTS: what this run expects) */ + +static Ctx call(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5, uint64_t a6) { + Ctx c; + memset(&c, 0, sizeof c); + c.x[0] = a0; c.x[1] = a1; c.x[2] = a2; c.x[3] = a3; c.x[4] = a4; c.x[5] = a5; c.x[6] = a6; + return c; +} +static Ctx with_float(uint64_t obj, float f) { + Ctx c = call(obj, 0, 0, 0, 0, 0, 0); + memcpy(&c.vreg[0][0], &f, 4); + return c; +} +static void reset(void) { + memset(guest, 0, sizeof guest); + memset(mixes, 0, sizeof mixes); + memset(splitters, 0, sizeof splitters); + memset(voices, 0, sizeof voices); + memset(biquads, 0, sizeof biquads); + for (int e = 0; e < MAX_FX; e++) free(delays[e].line); + memset(delays, 0, sizeof delays); + memset(reverbs, 0, sizeof reverbs); + ncmds = 0; + for (int m = 0; m < 2; m++) mixes[m].used = 1, mixes[m].buffers = 2, mixes[m].volume = 1.0f; + put_handle(MIX0, K_MIX, 0); + put_handle(MIX1, K_MIX, 1); +} +static void channels(const int8_t *in, const int8_t *out, int n) { + memcpy(GP(IN_CH), in, (size_t)n); + memcpy(GP(OUT_CH), out, (size_t)n); +} +static void float_packet(int on, const float coefs[5]) { + uint8_t packet[24]; + memset(packet, 0xa5, sizeof packet); + packet[0] = (uint8_t)on; + memcpy(packet + 4, coefs, 5 * sizeof *coefs); + memcpy(GP(PACKET), packet, sizeof packet); +} +static float rms(const float *x, int n) { + double e = 0; + for (int i = 0; i < n; i++) e += (double)x[i] * x[i]; + return (float)sqrt(e / n); +} + +/* ---- biquad filter: a Butterworth low pass at 1 kHz (the cookbook's, its denominator added and over a0) ---------- */ +static void lowpass(float f0, float out[5]) { + const float w = 2 * PI * f0 / RATE, alpha = sinf(w) / (2 * 0.7071068f), a0 = 1 + alpha; + out[0] = (1 - cosf(w)) / 2 / a0; + out[1] = (1 - cosf(w)) / a0; + out[2] = out[0]; + out[3] = 2 * cosf(w) / a0; /* (-a1 / a0) */ + out[4] = -(1 - alpha) / a0; /* (-a2 / a0) */ +} +static float biquad_tone(float hz) { /* the filtered tone's level against its own, after it has settled */ + reset(); + Ctx c = call(CONFIG, BQ, MIX0, 0, 0, 0, 0); + imp__ZN2nn5audio15AddBiquadFilterEPNS0_19AudioRendererConfigEPNS0_16BiquadFilterTypeEPNS0_10SubMixTypeE(&c); + CHECK(c.x[0] == 0); + channels((const int8_t[]){0, 1}, (const int8_t[]){0, 1}, 2); + c = call(BQ, IN_CH, OUT_CH, 2, 0, 0, 0); + imp__ZN2nn5audio26SetBiquadFilterInputOutputEPNS0_16BiquadFilterTypeEPKaS4_i(&c); + float coefs[5]; + lowpass(1000, coefs); + float_packet(1, coefs); + c = call(BQ, PACKET, 0, 0, 0, 0, 0); + imp__ZN2nn5audio24SetBiquadFilterParameterEPNS0_16BiquadFilterTypeERKNS0_22TBiquadFilterParameterIfEE(&c); + memset(GP(PACKET), 0xa5, 24); /* (the game's packet is its own again at once) */ + c = call(BQ, 1, 0, 0, 0, 0, 0); + imp__ZN2nn5audio22SetBiquadFilterEnabledEPNS0_16BiquadFilterTypeEb(&c); + c = call(BQ, 0, 0, 0, 0, 0, 0); + imp__ZN2nn5audio21IsBiquadFilterEnabledEPKNS0_16BiquadFilterTypeE(&c); + CHECK(c.x[0] == 1); + float in = 0, out = 0; + for (int frame = 0; frame < 20; frame++) { + for (int i = 0; i < FRAME; i++) { + const float s = sinf(2 * PI * hz * (float)(frame * FRAME + i) / RATE); + mixes[0].buf[0][i] = s; + mixes[0].buf[1][i] = 0.5f * s; + } + if (frame == 19) in = rms(mixes[0].buf[0], FRAME); + effects_frame(0); + if (frame == 19) { + out = rms(mixes[0].buf[0], FRAME); + CHECK(fabsf(rms(mixes[0].buf[1], FRAME) - 0.5f * out) < 0.001f); /* (each channel its own filter) */ + } + } + return out / in; +} +static void biquad(void) { + const float low = biquad_tone(200), high = biquad_tone(10000); + if (!effects) { CHECK(low == 1.0f && high == 1.0f); return; } + CHECK(low > 0.95f && low < 1.05f); + CHECK(high < 0.02f); /* (two poles: 40 dB down a decade above) */ + /* disabled: the buffers as they came */ + Ctx c = call(BQ, 0, 0, 0, 0, 0, 0); + imp__ZN2nn5audio22SetBiquadFilterEnabledEPNS0_16BiquadFilterTypeEb(&c); + for (int i = 0; i < FRAME; i++) mixes[0].buf[0][i] = (float)(i % 7) - 3.0f; + effects_frame(0); + for (int i = 0; i < FRAME; i++) CHECK(mixes[0].buf[0][i] == (float)(i % 7) - 3.0f); +} + +/* ---- delay: 3 ms (144 frames), in gain 0.5, feedback 0.5 ---------------------------------------------------------- */ +static void add_delay_on_mix0(float spread, float lowpass_amount) { + reset(); + Ctx c = call(CONFIG, DL, BUFFER, 0x10000, MIX0, 10000000, 2); /* (10 ms at most, 2 channels) */ + imp__ZN2nn5audio8AddDelayEPNS0_19AudioRendererConfigEPNS0_9DelayTypeEPvmPNS0_10SubMixTypeENS_8TimeSpanEi(&c); + CHECK(c.x[0] == 0); + channels((const int8_t[]){0, 1}, (const int8_t[]){0, 1}, 2); + c = call(DL, IN_CH, OUT_CH, 2, 0, 0, 0); + imp__ZN2nn5audio19SetDelayInputOutputEPNS0_9DelayTypeEPKaS4_i(&c); + c = call(DL, 3000000, 0, 0, 0, 0, 0); + imp__ZN2nn5audio12SetDelayTimeEPNS0_9DelayTypeENS_8TimeSpanE(&c); + c = call(DL, 0, 0, 0, 0, 0, 0); + imp__ZN2nn5audio12GetDelayTimeEPKNS0_9DelayTypeE(&c); + CHECK(c.x[0] == 3000000); + c = with_float(DL, 0.5f); imp__ZN2nn5audio14SetDelayInGainEPNS0_9DelayTypeEf(&c); + c = with_float(DL, 0.5f); imp__ZN2nn5audio20SetDelayFeedbackGainEPNS0_9DelayTypeEf(&c); + c = with_float(DL, 1.0f); imp__ZN2nn5audio15SetDelayDryGainEPNS0_9DelayTypeEf(&c); + c = with_float(DL, spread); imp__ZN2nn5audio21SetDelayChannelSpreadEPNS0_9DelayTypeEf(&c); + c = with_float(DL, lowpass_amount); imp__ZN2nn5audio21SetDelayLowPassAmountEPNS0_9DelayTypeEf(&c); + c = call(DL, 1, 0, 0, 0, 0, 0); + imp__ZN2nn5audio15SetDelayEnabledEPNS0_9DelayTypeEb(&c); + c = call(DL, 0, 0, 0, 0, 0, 0); + imp__ZN2nn5audio14IsDelayEnabledEPKNS0_9DelayTypeE(&c); + CHECK(c.x[0] == 1); +} +static void run_impulse(float out[2][4 * FRAME]) { /* an impulse in channel 0 at frame 0, four frames out */ + for (int frame = 0; frame < 4; frame++) { + memset(mixes[0].buf, 0, sizeof mixes[0].buf); + if (frame == 0) mixes[0].buf[0][0] = 1.0f; + effects_frame(0); + for (int ch = 0; ch < 2; ch++) memcpy(out[ch] + frame * FRAME, mixes[0].buf[ch], FRAME * sizeof(float)); + } +} +static void close_to(float actual, float expected) { CHECK(fabsf(actual - expected) < 0.00001f); } +static void delay(void) { + static float out[2][4 * FRAME]; + add_delay_on_mix0(0, 0); + run_impulse(out); + if (!effects) { + for (int i = 0; i < 4 * FRAME; i++) CHECK(out[0][i] == (i == 0) && out[1][i] == 0); + return; + } + /* the dry impulse, then echoes every 144 frames (across the frames' edges), each the feedback gain x 0.98 of the + one before; nothing in between and nothing in the other channel */ + const float fb = 0.5f * 0.98f; + for (int i = 0; i < 4 * FRAME; i++) { + const float want = i == 0 ? 1.0f : i % 144 == 0 ? 0.5f * powf(fb, (float)(i / 144 - 1)) : 0.0f; + close_to(out[0][i], want); + CHECK(out[1][i] == 0); + } + /* the whole spread: each echo in the other channel than the one before */ + add_delay_on_mix0(1, 0); + run_impulse(out); + close_to(out[0][144], 0.5f); close_to(out[1][144], 0); + close_to(out[1][288], 0.5f * fb); close_to(out[0][288], 0); + close_to(out[0][432], 0.5f * fb * fb); close_to(out[1][432], 0); + /* the low pass: the echo's first frame 1 - 0.95 x amount of it, the rest after it */ + add_delay_on_mix0(0, 0.5f); + run_impulse(out); + close_to(out[0][144], 0.5f * (1 - 0.475f)); + CHECK(out[0][145] > 0 && out[0][145] < out[0][144]); + /* off: the buffers as they came */ + Ctx c = call(DL, 0, 0, 0, 0, 0, 0); + imp__ZN2nn5audio15SetDelayEnabledEPNS0_9DelayTypeEb(&c); + run_impulse(out); + for (int i = 0; i < 4 * FRAME; i++) CHECK(out[0][i] == (i == 0) && out[1][i] == 0); + /* removed: the game's buffer back, the handle gone */ + c = call(CONFIG, DL, MIX0, 0, 0, 0, 0); + imp__ZN2nn5audio11RemoveDelayEPNS0_19AudioRendererConfigEPNS0_9DelayTypeEPNS0_10SubMixTypeE(&c); + CHECK(c.x[0] == BUFFER && G64(DL) == 0 && !delays[0].used); +} + +/* ---- plain reverb: a tail after the dry impulse, and the dry signal alone at out gain 0 ------------------------- */ +static void reverb(void) { + reset(); + Ctx c = call(CONFIG, RV, BUFFER, 0x10000, MIX0, 2, 0); + imp__ZN2nn5audio9AddReverbEPNS0_19AudioRendererConfigEPNS0_10ReverbTypeEPvmPNS0_10SubMixTypeEi(&c); + CHECK(c.x[0] == 0); + c = call(RV, 2, 0, 0, 0, 0, 0); imp__ZN2nn5audio18SetReverbEarlyModeEPNS0_10ReverbTypeENS1_9EarlyModeE(&c); + c = call(RV, 1, 0, 0, 0, 0, 0); imp__ZN2nn5audio17SetReverbLateModeEPNS0_10ReverbTypeENS1_8LateModeE(&c); + c = call(RV, 1500000000, 0, 0, 0, 0, 0); imp__ZN2nn5audio18SetReverbDecayTimeEPNS0_10ReverbTypeENS_8TimeSpanE(&c); + c = call(RV, 10000000, 0, 0, 0, 0, 0); imp__ZN2nn5audio21SetReverbPredelayTimeEPNS0_10ReverbTypeENS_8TimeSpanE(&c); + c = with_float(RV, 0.6f); imp__ZN2nn5audio18SetReverbEarlyGainEPNS0_10ReverbTypeEf(&c); + c = with_float(RV, 0.6f); imp__ZN2nn5audio17SetReverbLateGainEPNS0_10ReverbTypeEf(&c); + c = with_float(RV, 0.8f); imp__ZN2nn5audio19SetReverbReverbGainEPNS0_10ReverbTypeEf(&c); + c = with_float(RV, 1.0f); imp__ZN2nn5audio16SetReverbOutGainEPNS0_10ReverbTypeEf(&c); + c = with_float(RV, 0.5f); imp__ZN2nn5audio16SetReverbDryGainEPNS0_10ReverbTypeEf(&c); + c = with_float(RV, 0.5f); imp__ZN2nn5audio32SetReverbHighFrequencyDecayRatioEPNS0_10ReverbTypeEf(&c); + c = with_float(RV, 0.5f); imp__ZN2nn5audio19SetReverbColorationEPNS0_10ReverbTypeEf(&c); + c = call(RV, 1, 0, 0, 0, 0, 0); imp__ZN2nn5audio16SetReverbEnabledEPNS0_10ReverbTypeEb(&c); + c = call(RV, 0, 0, 0, 0, 0, 0); imp__ZN2nn5audio15IsReverbEnabledEPKNS0_10ReverbTypeE(&c); + CHECK(c.x[0] == 1); + static float out[2][4 * FRAME]; + run_impulse(out); + if (!effects) { + for (int i = 0; i < 4 * FRAME; i++) CHECK(out[0][i] == (i == 0) && out[1][i] == 0); + return; + } + close_to(out[0][0], 0.5f); /* (the dry gain; the wet part starts after the pre-delay) */ + float tail = 0; + for (int i = 1; i < 4 * FRAME; i++) { + CHECK(isfinite(out[0][i]) && isfinite(out[1][i]) && fabsf(out[0][i]) < 1 && fabsf(out[1][i]) < 1); + tail += fabsf(out[0][i]) + fabsf(out[1][i]); + } + CHECK(tail > 0.05f); + c = with_float(RV, 0.0f); imp__ZN2nn5audio16SetReverbOutGainEPNS0_10ReverbTypeEf(&c); + run_impulse(out); + for (int i = 0; i < 4 * FRAME; i++) { close_to(out[0][i], i == 0 ? 0.5f : 0.0f); CHECK(out[1][i] == 0); } +} + +/* ---- a splitter destination's filter, through render_voice to the destination's mix ---------------------------- */ +static float splitter_level(int filter_on) { + reset(); + Splitter *s = &splitters[0]; + s->used = 1; s->channels = 1; s->dests = 1; s->dest_mix[0] = 1; + s->vol[0][0][0] = s->ramp[0][0][0] = 1.0f; + put_handle(SPLIT, K_SPLITTER, 0); + /* a one-pole low pass, y = 0.1 x + 0.9 y': a tone at half the rate comes out at 0.1 / 1.9 */ + float_packet(filter_on, (const float[]){0.1f, 0, 0, 0.9f, 0}); + Ctx c = call(SPLIT, 0, MIX1, 1, PACKET, 0, 0); + imp__ZN2nn5audio32SetSplitterBiquadFilterParameterEPNS0_12SplitterTypeEiPNS0_10SubMixTypeEiRKNS0_22TBiquadFilterParameterIfEE(&c); + CHECK(ncmds == 1 && !s->bq_on[0][1]); /* (queued, applied at the update) */ + commit(); + CHECK(s->bq_on[0][1] == filter_on && !s->bq_on[0][0]); + Voice *v = &voices[0]; + v->used = 1; v->channels = 1; v->rate = RATE; v->pitch = 1; v->volume = v->ramp_volume = 1; + v->state = PLAY_STARTED; v->dest_kind = DEST_SPLITTER; v->dest = 0; + for (int i = 0; i < 4096; i++) tone[i] = i & 1 ? -1.0f : 1.0f; + tone_pos = 0; + float level = 0; + for (int frame = 0; frame < 10; frame++) { + memset(mixes[1].buf, 0, sizeof mixes[1].buf); + render_voice(v); + level = rms(mixes[1].buf[0] + 16, FRAME - 16); + } + return level; +} +static void splitter(void) { + const float plain = splitter_level(0), filtered = splitter_level(1); + CHECK(fabsf(plain - 1.0f) < 0.0001f); + if (!effects) CHECK(filtered == plain); + else CHECK(fabsf(filtered - 0.1f / 1.9f) < 0.001f); +} + +int main(void) { + const char *e = getenv("HOS_AUDIO_EFFECTS"); + effects = !(e && atoi(e) == 0); + CHECK(effects_on() == effects); + biquad(); + delay(); + reverb(); + splitter(); + reset(); + free(cmds); + printf("PASS (%s): biquad low pass, delay echo/feedback/spread/low pass, plain reverb tail, splitter filter\n", + effects ? "effects on" : "HOS_AUDIO_EFFECTS=0: samples unchanged"); + return 0; +} diff --git a/tests/audio_wave_buffer_release_test.c b/tests/audio_wave_buffer_release_test.c index 3a1d647..6c0144b 100644 --- a/tests/audio_wave_buffer_release_test.c +++ b/tests/audio_wave_buffer_release_test.c @@ -20,6 +20,8 @@ static void hos_log(const char *format, ...) { (void)format; } /* (host/rx_backend.c's: the lazy write-back's calls around the wave buffer dump, which no switch turns on here) */ static int hos_guest_io_begin(const void *p, uint64_t n) { (void)p; (void)n; return 0; } static void hos_guest_io_end(int held) { CHECK(!held); } +/* ((Triforce, 11-21, MEM-09) host/mkw_glue.c's: the dump's buffer read once before the kernel's write; not what is tested) */ +static void hos_guest_touch(uint64_t addr, uint64_t size) { (void)addr; (void)size; } /* Single-threaded test boundaries: the mixer's frames run between the game's calls, as its thread can. */ static int lock_depth; diff --git a/tests/desc_sets_test.cpp b/tests/desc_sets_test.cpp index ea8916e..18cb1d3 100644 --- a/tests/desc_sets_test.cpp +++ b/tests/desc_sets_test.cpp @@ -60,6 +60,7 @@ uint64_t gpu_bind_stamp() { return bind_stamp; } struct RecordingRes { VkDescriptorPool pool{}; std::vector extra; } recording; RecordingRes* cur = &recording; VkDescriptorPool make_pool() { return handle(0x77); } +void pool_took(const VkWriteDescriptorSet*, size_t) {} // ((Triforce, 11-22, DESC-33) vk_draw.cpp: the sets a recording took, counted) void vkDestroyDescriptorPool(VkDevice, VkDescriptorPool, const VkAllocationCallbacks*) {} // ---- the fake driver: each set's descriptors, (binding, element) -> (type, the handles and numbers that make it) ---- diff --git a/tests/dmb_light_test.c b/tests/dmb_light_test.c new file mode 100644 index 0000000..958d374 --- /dev/null +++ b/tests/dmb_light_test.c @@ -0,0 +1,96 @@ +/* tests/test_dmb_light.py's driver ((Triforce, 11-22, the performance audit's LIFT-47)): each shape (a barrier alone, or + between a store and a load of the same guest address) as lifted with A64_DMB_LIGHT=1 against the same words as the + kit's translator writes them. Both start from one random Ctx and guest memory; the whole Ctx and the bytes the shape + stores must match after, and the calls of recomp_barrier (counted here) must be the kit's count without the light + barriers' on x86-64. + + dmb_light_test n random states for each shape; the flags go through all sixteen combinations + Exit 0: no mismatch. 1: a mismatch. 2: no guest memory where the shapes store. */ +#include "a64.h" +#include +#include +#ifdef _WIN32 +#define WIN32_LEAN_AND_MEAN +#include +#else +#include +#endif + +static long fences; /* (the runtime's full fence, a64_rt.c's, counted) */ +void recomp_barrier(void) { + fences++; + __atomic_thread_fence(__ATOMIC_SEQ_CST); +} +#include "dmb_lifted.h" + +enum { kGuestLo = 0x10000, kGuestLen = 1 << 20 }; /* (the guest memory the shapes use, at A64_PTR(kGuestLo)) */ +static int map_at(void *p, size_t n) { +#ifdef _WIN32 + return VirtualAlloc(p, n, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE) == p; +#else + return mmap(p, n, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0) == p; +#endif +} + +static uint64_t rs; +static uint64_t rnd(void) { rs ^= rs << 13; rs ^= rs >> 7; rs ^= rs << 17; return rs; } +static uint64_t edge64(void) { + static const uint64_t e[] = {0, 1, 0x7fffffffffffffffULL, 0x8000000000000000ULL, ~0ULL, 0xffffffffULL, 0x100000000ULL}; + return (rnd() & 3) ? rnd() : e[rnd() % (sizeof e / sizeof e[0])]; +} +static void fill(Ctx *c, long i) { + memset(c, 0, sizeof *c); + for (int r = 0; r < 31; r++) c->x[r] = edge64(); + c->x[2] = kGuestLo + 0x1000 + (rnd() % (kGuestLen - 0x2000)); /* (the shapes' base register: unaligned too) */ + c->x[31] = kGuestLo + 0x800; + c->n = (uint8_t)(i & 1), c->z = (uint8_t)(i >> 1 & 1), c->c = (uint8_t)(i >> 2 & 1), c->v = (uint8_t)(i >> 3 & 1); + for (int r = 0; r < 32; r++) c->vreg[r][0] = edge64(), c->vreg[r][1] = edge64(); + c->fpcr = (rnd() & 1) ? 0x01000000u : 0, c->fpsr = rnd() & 0x9f; +} + +int main(int argc, char **argv) { + if (!map_at(A64_PTR(kGuestLo), kGuestLen)) { + printf("no guest memory at %p\n", (void *)A64_PTR(kGuestLo)); + return 2; + } + const long n = argc > 1 ? atol(argv[1]) : 20000; + rs = argc > 2 ? strtoull(argv[2], 0, 10) : 1; + if (!rs) rs = 1; + const int ncases = (int)(sizeof CASES / sizeof CASES[0]); +#if defined(__x86_64__) + const int host_light = 1; +#else + const int host_light = 0; /* (any other host: the light barrier is the full fence too) */ +#endif + long bad = 0, checks = 0; + for (int k = 0; k < ncases; k++) + for (long i = 0; i < n; i++) { + static Ctx a, b; + uint8_t before[16], ma[16], mb[16]; + fill(&a, i); + b = a; + uint8_t *mem = A64_PTR(a.x[2]); + for (int j = 0; j < 16; j++) mem[j] = (uint8_t)rnd(); + memcpy(before, mem, 16); + fences = 0; + CASES[k].now(&a); + const long fa = fences; + memcpy(ma, mem, 16); + memcpy(mem, before, 16); + fences = 0; + CASES[k].old(&b); + const long fb = fences; + memcpy(mb, mem, 16); + checks++; + const int state = memcmp(&a, &b, sizeof a) || memcmp(ma, mb, 16); + const int count = fb != CASES[k].barriers || fa != CASES[k].barriers - (host_light ? CASES[k].light : 0); + if (state || count) { + if (bad++ < 5) + printf("MISMATCH %s: %s%sfences now %ld, kit %ld (of %d, %d light)\n", CASES[k].name, + state ? "registers, flags or memory differ; " : "", count ? "fence count; " : "", fa, fb, + CASES[k].barriers, CASES[k].light); + } + } + printf("%d shapes, %ld checks, %ld mismatches\n", ncases, checks, bad); + return bad ? 1 : 0; +} diff --git a/tests/draw_stream_visibility_test.cpp b/tests/draw_stream_visibility_test.cpp index aa6e3d5..094833c 100644 --- a/tests/draw_stream_visibility_test.cpp +++ b/tests/draw_stream_visibility_test.cpp @@ -19,6 +19,8 @@ #include #include #include "barrier_rules.h" +#include "desc_pool_grow.h" // ((Triforce, 11-22, DESC-33) a slot's pool at half, whole once a recording ran out) +namespace desc_pool_grow = mkwvk::desc_pool_grow; static void require(bool ok,const char* why) { if(!ok) { std::fprintf(stderr,"FAIL: %s\n",why); std::exit(1); } } struct Allocation { std::vector cpu,visible; }; using VmaAllocation=Allocation*; @@ -55,7 +57,10 @@ void draw_flush_recording(int slot); VkCommandBuffer rec_cmd{}; uint64_t rec_value=0,targets_serial=0; VkBuffer zero_buffer{}; VmaAllocation zero_alloc{}; -VkDescriptorPool make_pool() { return handle(); } +VkDescriptorPool make_pool(uint32_t = 0) { return handle(); } +bool pool_grow_on() { return true; } // ((Triforce, 11-22, DESC-33) vk_draw.cpp: RX_VK_DESC_POOL_GROW and its counts) +desc_pool_grow::Took pool_most; +uint64_t pools_added = 0, pools_whole = 0; VKAPI_ATTR VkResult VKAPI_CALL vkResetDescriptorPool(VkDevice,VkDescriptorPool,VkDescriptorPoolResetFlags) { return VK_SUCCESS; } VKAPI_ATTR void VKAPI_CALL vkDestroyDescriptorPool(VkDevice,VkDescriptorPool,const VkAllocationCallbacks*) {} struct Pass {}; Pass& pass() { static Pass p;return p; } @@ -90,7 +95,9 @@ bool batch=false;bool batch_on() { return batch; } namespace barrier_rules = mkwvk::barrier_rules; // (the held submits' rule, gpu/vk/barrier_rules.h: RX_VK_BATCH=1 here) barrier_rules::Batch batch_way() { return batch ? barrier_rules::Batch::Hold : barrier_rules::Batch::Off; } int submit_depth=0; -int flush_reason=0;constexpr int FR_OTHER=0,FR_CAP=1; +int flush_reason=0;constexpr int FR_OTHER=0,FR_CAP=1,FR_IDLE=2; +bool batch_idle_on() { return false; } // ((Triforce, 11-22, SYNC-09) RX_VK_BATCH_IDLE: off, as unset; not this test's) +bool batch_gpu_idle() { return false; } bool event_writes_held(VkCommandBuffer) { return false; } // (no event's write held: RX_VK_EVENT_BARRIER's, not this test's) void split_part_submitted(VkCommandBuffer,bool) {} // (RX_VK_SPLIT_BARRIER's count) void end_pass(VkCommandBuffer) { require(false,"a pass ended at a submit"); } diff --git a/tests/dump_readback_test.cpp b/tests/dump_readback_test.cpp index 1538fa2..5b1a208 100644 --- a/tests/dump_readback_test.cpp +++ b/tests/dump_readback_test.cpp @@ -119,7 +119,7 @@ int main(int argc,char** argv) { if(which=="empty") { consume_dump(false);require(waits==0 && invalidations==0 && png.empty(),"empty queue reads/waits");return 0; } - armed=true; + dump_armed=true; if(which=="ssbo" || which=="ssbo-tail") { std::vector words(which=="ssbo"?8:16384); words[1]=1;words[3]=2;words.back()=3;upload(words); diff --git a/tests/fallback_buffer_cleanup_test.cpp b/tests/fallback_buffer_cleanup_test.cpp index 30b387b..6e4e633 100644 --- a/tests/fallback_buffer_cleanup_test.cpp +++ b/tests/fallback_buffer_cleanup_test.cpp @@ -2,8 +2,10 @@ // initialization/BDA functions are included verbatim from current vk_draw.cpp. #include #include +#include #include #include +#include #include #include #include @@ -71,6 +73,8 @@ static uint64_t vkGetBufferDeviceAddress(int device, const VkBufferDeviceAddress return ++address_calls == zero_address ? 0 : info->buffer * 4096; } +static void logf(const char*, ...) {} // ((11-21, MEM-154) the new pairs' line) + #include "fallback_actual.inc" // (the buffers are a recording thread's, not a program's (#1383): a failure is not remembered, the next draw tries again, and @@ -155,6 +159,44 @@ static void run(const std::string& name) { Program next; require(fallback_buffers_of(next, out) && out && out->records_address, "another program inherited failure"); require(live.size() == 2 && destroy_calls == 1, "another program's buffers were freed or leaked"); + } else if (name == "payload_growth_keeps_records") { + // ((11-21, the performance audit's MEM-154) the records are the same size for every program: a program whose + // payloads need more takes a bigger payload buffer, and the records the thread has already) + require(fallback_buffers_of(p, out) && out && out->payload_bytes == 44 && create_calls == 2, "first program's pair"); + const VkBuffer records = out->records; + const uint64_t records_address = out->records_address; + Program bigger; + bigger.fallback_payload_words = 8; + require(fallback_buffers_of(bigger, out) && out && out->payload_bytes == 116, "a bigger payload was not made"); + require(out->records == records && out->records_address == records_address && create_calls == 3 && live.size() == 3, + "a bigger payload made another records buffer"); + const auto calls = create_calls; + require(fallback_buffers_of(p, out) && out->payload_bytes == 116 && create_calls == calls, "the smaller program made more"); + require(destroy_calls == 0, "a growth destroyed a buffer"); + } else if (name == "shared_records_payload_failure") { + // ((11-21, MEM-154) a growth that shares the records and then cannot make its payload: its create fails, and on + // the next try its address comes back 0. The shared records must stay, and the pair before must still serve) + require(fallback_buffers_of(p, out) && out && create_calls == 2 && address_calls == 2, "first program's pair"); + const VkBuffer records = out->records; + const uint64_t payload_bytes = out->payload_bytes; + const auto still_usable = [&](const char* why) { + const auto calls = create_calls; + require(live.contains(records), why); + require(fallback_buffers_of(p, out) && out->records == records && out->payload_bytes == payload_bytes && + create_calls == calls, why); + }; + Program bigger; + bigger.fallback_payload_words = 8; + fail_create = 3; + require(!fallback_buffers_of(bigger, out), "a failed grown payload was accepted"); + require(destroy_calls == 0 && live.size() == 2, "a failed grown payload destroyed something"); + still_usable("the shared records went with a payload that could not be made"); + zero_address = address_calls + 1; + require(!fallback_buffers_of(bigger, out), "a grown payload with no address was accepted"); + require(destroy_calls == 1 && live.size() == 2, "a grown payload with no address was not the one buffer destroyed"); + still_usable("the shared records went with a payload that had no address"); + require(fallback_buffers_of(bigger, out) && out->records == records && out->payload_bytes == 116 && live.size() == 3, + "the growth did not succeed once it could"); } else throw std::runtime_error("unknown authored case"); } int main(int argc, char** argv) { diff --git a/tests/fma_soft_test.c b/tests/fma_soft_test.c new file mode 100644 index 0000000..2a3ca4b --- /dev/null +++ b/tests/fma_soft_test.c @@ -0,0 +1,339 @@ +/* tests/test_fma_soft.py's driver ((Triforce, 11-22, the performance audit's LIFT-06)): the exact fused multiply-add + bodies recomp_fpe_fma64 and recomp_fpe_fma32 with nx2/fma_soft.py's short way out against the same bodies without it + (the kit's text: the way out is the body's first lines and RECOMP_FMA_SOFT_ON() decides it, which the test's hook + header makes a variable). Both run from one Ctx; the value and every byte of the Ctx must match. + + fma_soft_test random n operand triples a width: zeros, subnormals, infinities, NaNs of both kinds with + payloads, the extreme normals, exponents everywhere and matched to the product, + cancellations and half-ulp addends, under random FPCR and FPSR + fma_soft_test sweep edge exponents and fractions for both factors, each with addends placed against the + product (cancelling it within a few ulps, at its half ulp, far below, far above), + under FZ on and off with IXC set and clear + fma_soft_test env (built without the hook) how many of 1000 plain calls the way out answers: what + HOS_FMA_FAST decides + fma_soft_test time ns a call for plain operands, with the way out and without + Exit 0: no mismatch. 1: a mismatch. 3: this CPU cannot run the build. */ +#include "a64.h" +#include +#include +#include +#ifdef _WIN32 +#define WIN32_LEAN_AND_MEAN +#include +#else +#include +#endif + +int fma_soft_test_on = 1; +long long fma_soft_test_exits; + +static uint64_t rs = 1; +static uint64_t rnd(void) { rs ^= rs << 13; rs ^= rs >> 7; rs ^= rs << 17; return rs; } +static uint32_t rbelow(uint32_t n) { return (uint32_t)(((rnd() >> 32) * n) >> 32); } +static double d_of(uint64_t x) { double f; memcpy(&f, &x, 8); return f; } +static uint64_t d_bits(double f) { uint64_t x; memcpy(&x, &f, 8); return x; } +static float f_of(uint32_t x) { float f; memcpy(&f, &x, 4); return f; } +static uint32_t f_bits(float f) { uint32_t x; memcpy(&x, &f, 4); return x; } + +/* ---- states ---- */ +static uint64_t rand_fpcr(void) { + static const uint64_t bits[] = {1u << 22, 1u << 23, 3u << 22, 1u << 25, 1u << 26, 1, 2, 4, 1u << 8, 1u << 9, 1u << 10, + 1u << 11, 1u << 12, 1u << 15, 1u << 19, UINT64_C(1) << 32}; + const uint32_t k = rbelow(100); + if (k < 55) return 0; + if (k < 80) return 1u << 24; /* FZ alone */ + uint64_t v = bits[rbelow(sizeof bits / sizeof bits[0])]; + if (rnd() & 1) v |= 1u << 24; + if (!(rnd() & 3)) v |= bits[rbelow(sizeof bits / sizeof bits[0])]; + return v; +} +static uint64_t rand_fpsr(void) { + uint64_t v = rnd() & 0x9f; + if (!(rnd() & 3)) v |= 1u << 27; + switch (rbelow(4)) { + case 0: return v | 16u; + case 1: return v & ~(uint64_t)16u; + case 2: return 0; + default: return v; + } +} + +/* ---- operands ---- */ +static uint64_t frac64(void) { + switch (rbelow(8)) { + case 0: return 0; + case 1: return (UINT64_C(1) << 52) - 1; + case 2: return rnd() & ((UINT64_C(1) << 52) - 1) & ~((UINT64_C(1) << (1 + rbelow(51))) - 1); /* (low bits clear) */ + case 3: return (UINT64_C(1) << rbelow(52)) | (rnd() & 1); + default: return rnd() & ((UINT64_C(1) << 52) - 1); + } +} +static uint64_t any64(void) { + const uint32_t k = rbelow(100); + const uint64_t sign = (rnd() & 1) << 63; + if (k < 4) return sign; /* zero */ + if (k < 8) return sign | (frac64() ? frac64() : 1); /* subnormal (or zero) */ + if (k < 10) return sign | UINT64_C(0x7ff0000000000000); /* infinity */ + if (k < 12) return sign | UINT64_C(0x7ff8000000000000) | (rnd() & ((UINT64_C(1) << 51) - 1)); /* quiet NaN */ + if (k < 14) return sign | UINT64_C(0x7ff0000000000000) | ((rnd() & ((UINT64_C(1) << 51) - 1)) | 1); /* signalling */ + if (k < 17) return sign | ((rnd() & 1) ? UINT64_C(0x0010000000000000) | (rnd() & 3) : UINT64_C(0x7fefffffffffffff) - (rnd() & 3)); + if (k < 30) return sign | ((uint64_t)(1 + rbelow(2046)) << 52) | frac64(); /* any exponent */ + if (k < 40) { /* around the way out's limits */ + static const uint32_t e[] = {1, 2, 123, 542, 543, 544, 1502, 1503, 1504, 1923, 2045, 2046}; + return sign | ((uint64_t)e[rbelow(sizeof e / sizeof e[0])] << 52) | frac64(); + } + return sign | ((uint64_t)(983 + rbelow(81)) << 52) | frac64(); /* ordinary */ +} +static void triple64(uint64_t *a, uint64_t *b, uint64_t *z) { + *a = any64(), *b = any64(), *z = any64(); + const uint32_t k = rbelow(100); + if (k >= 60) return; + const double p = d_of(*a) * d_of(*b); + const uint64_t pb = d_bits(p); + const uint32_t pe = (uint32_t)(pb >> 52) & 0x7ff; + if (pe == 0 || pe == 0x7ff) return; + if (k < 20) { /* an addend that cancels the product within a few ulps */ + *z = (pb ^ (UINT64_C(1) << 63)) + (uint64_t)(int64_t)((int)rbelow(9) - 4); + } else if (k < 35) { /* an addend at the product's half ulp, quarter ulp, ulp ... (its fraction zero or random) */ + const int e = (int)pe - 55 + (int)rbelow(6); + if (e >= 1) *z = ((rnd() & 1) << 63) | ((uint64_t)e << 52) | ((rnd() & 1) ? 0 : (rnd() & 3) ? frac64() : 1); + } else if (k < 50) { /* an addend near the product's size */ + const int e = (int)pe - 3 + (int)rbelow(7); + if (e >= 1 && e <= 2046) *z = ((rnd() & 1) << 63) | ((uint64_t)e << 52) | frac64(); + } else { /* the product at an addend's half ulp */ + const int e = (int)pe + 50 + (int)rbelow(6); + if (e <= 2046) *z = ((rnd() & 1) << 63) | ((uint64_t)e << 52) | ((rnd() & 1) ? 0 : frac64()); + } +} +static uint32_t frac32(void) { + switch (rbelow(8)) { + case 0: return 0; + case 1: return 0x7fffff; + case 2: return (uint32_t)rnd() & 0x7fffff & ~((1u << (1 + rbelow(22))) - 1); + case 3: return (1u << rbelow(23)) | ((uint32_t)rnd() & 1); + default: return (uint32_t)rnd() & 0x7fffff; + } +} +static uint32_t any32(void) { + const uint32_t k = rbelow(100); + const uint32_t sign = (uint32_t)(rnd() & 1) << 31; + if (k < 4) return sign; + if (k < 8) return sign | (frac32() ? frac32() : 1); + if (k < 10) return sign | 0x7f800000u; + if (k < 12) return sign | 0x7fc00000u | ((uint32_t)rnd() & 0x3fffff); + if (k < 14) return sign | 0x7f800000u | (((uint32_t)rnd() & 0x3fffff) | 1); + if (k < 17) return sign | ((rnd() & 1) ? 0x00800000u | ((uint32_t)rnd() & 3) : 0x7f7fffffu - ((uint32_t)rnd() & 3)); + if (k < 35) return sign | ((1 + rbelow(254)) << 23) | frac32(); + return sign | ((107 + rbelow(41)) << 23) | frac32(); +} +static void triple32(uint32_t *a, uint32_t *b, uint32_t *z) { + *a = any32(), *b = any32(), *z = any32(); + const uint32_t k = rbelow(100); + if (k >= 60) return; + const float p = f_of(*a) * f_of(*b); + const uint32_t pb = f_bits(p), pe = (pb >> 23) & 0xff; + if (pe == 0 || pe == 0xff) return; + if (k < 20) { + *z = (pb ^ 0x80000000u) + (uint32_t)((int)rbelow(9) - 4); + } else if (k < 35) { + const int e = (int)pe - 26 + (int)rbelow(6); + if (e >= 1) *z = ((uint32_t)(rnd() & 1) << 31) | ((uint32_t)e << 23) | ((rnd() & 1) ? 0 : (rnd() & 3) ? frac32() : 1); + } else if (k < 50) { + const int e = (int)pe - 3 + (int)rbelow(7); + if (e >= 1 && e <= 254) *z = ((uint32_t)(rnd() & 1) << 31) | ((uint32_t)e << 23) | frac32(); + } else { + const int e = (int)pe + 21 + (int)rbelow(6); + if (e <= 254) *z = ((uint32_t)(rnd() & 1) << 31) | ((uint32_t)e << 23) | ((rnd() & 1) ? 0 : frac32()); + } +} + +/* ---- one comparison ---- */ +static struct { long long checks, bad, exits, new_ixc, exact, fz; } st[2]; /* [0] binary64, [1] binary32 */ +static void check(int w32, uint64_t a, uint64_t b, uint64_t z, uint64_t fpcr, uint64_t fpsr) { + static Ctx base; + Ctx on, off; + if (!st[0].checks && !st[1].checks) /* (the rest of the context: bytes that must stay as they are) */ + for (size_t i = 0; i < sizeof base; i++) ((uint8_t *)&base)[i] = (uint8_t)rnd(); + base.fpcr = fpcr, base.fpsr = fpsr; + on = base, off = base; + const long long before = fma_soft_test_exits; + fma_soft_test_on = 1; + const uint64_t von = w32 ? recomp_fpe_fma32(&on, a, b, z) : recomp_fpe_fma64(&on, a, b, z); + const int took = fma_soft_test_exits != before; + fma_soft_test_on = 0; + const uint64_t voff = w32 ? recomp_fpe_fma32(&off, a, b, z) : recomp_fpe_fma64(&off, a, b, z); + st[w32].checks++; + if (took) { + st[w32].exits++; + st[w32].new_ixc += !(fpsr & 16u) && (on.fpsr & 16u); + st[w32].exact += !(fpsr & 16u) && !(on.fpsr & 16u); + st[w32].fz += (fpcr >> 24) & 1; + } + if (von != voff || memcmp(&on, &off, sizeof on)) { + if (st[w32].bad++ < 8) + printf("MISMATCH fma%d a %016llx b %016llx z %016llx fpcr %llx fpsr %llx: with the way out %016llx fpsr %llx, " + "without %016llx fpsr %llx\n", w32 ? 32 : 64, (unsigned long long)a, (unsigned long long)b, + (unsigned long long)z, (unsigned long long)fpcr, (unsigned long long)fpsr, (unsigned long long)von, + (unsigned long long)on.fpsr, (unsigned long long)voff, (unsigned long long)off.fpsr); + } +} + +static void random_cases(long long n) { + for (long long i = 0; i < n; i++) { + uint64_t a, b, z; + uint32_t x, y, w; + triple64(&a, &b, &z); + check(0, a, b, z, rand_fpcr(), rand_fpsr()); + triple32(&x, &y, &w); + check(1, x, y, w, rand_fpcr(), rand_fpsr()); + } +} + +/* the sweep's addends, placed against the product p (kind k; i varies the sign and the last bits) */ +static uint64_t addend64(double p, int k, unsigned i) { + const uint64_t pb = d_bits(p), neg = pb ^ (UINT64_C(1) << 63), sign = (uint64_t)(i & 1) << 63; + const int pe = (int)((pb >> 52) & 0x7ff); + static const int rel[] = {-108, -106, -105, -56, -55, -54, -53, -52, -51, -2, -1, 0, 1, 2, 51, 52, 53, 54, 55, 106}; + if (k < 9) return neg + (uint64_t)(int64_t)(k - 4); /* -p within four ulps */ + if (k < 11) return (k == 9 ? pb : neg) ^ ((uint64_t)(i >> 1 & 1) << 52); /* +-p, or its neighbour binade */ + k -= 11; + if (k < 20) { /* 2^rel times the product's binade: fraction 0 */ + const int e = pe + rel[k]; + return e >= 1 && e <= 2046 ? sign | ((uint64_t)e << 52) : sign | UINT64_C(0x3ff0000000000000); + } + k -= 20; + { /* the same with a fraction: 1, all ones */ + const int e = pe + rel[k % 20]; + const uint64_t f = k < 20 ? 1 : (UINT64_C(1) << 52) - 1; + return e >= 1 && e <= 2046 ? sign | ((uint64_t)e << 52) | f : sign | UINT64_C(0x4000000000000000) | f; + } +} +static uint32_t addend32(float p, int k, unsigned i) { + const uint32_t pb = f_bits(p), neg = pb ^ 0x80000000u, sign = (uint32_t)(i & 1) << 31; + const int pe = (int)((pb >> 23) & 0xff); + static const int rel[] = {-50, -48, -47, -27, -26, -25, -24, -23, -22, -2, -1, 0, 1, 2, 22, 23, 24, 25, 26, 48}; + if (k < 9) return neg + (uint32_t)(k - 4); + if (k < 11) return (k == 9 ? pb : neg) ^ ((uint32_t)(i >> 1 & 1) << 23); + k -= 11; + if (k < 20) { + const int e = pe + rel[k]; + return e >= 1 && e <= 254 ? sign | ((uint32_t)e << 23) : sign | 0x3f800000u; + } + k -= 20; + { + const int e = pe + rel[k % 20]; + const uint32_t f = k < 20 ? 1 : 0x7fffff; + return e >= 1 && e <= 254 ? sign | ((uint32_t)e << 23) | f : sign | 0x40000000u | f; + } +} +enum { kKinds = 11 + 20 + 40 }; +static void sweep(void) { + static const uint32_t e64[] = {1, 2, 123, 542, 543, 544, 1000, 1022, 1023, 1024, 1050, 1502, 1503, 1504, 2046}; + static const uint64_t f64[] = {0, 1, 2, 3, (UINT64_C(1) << 26) - 1, UINT64_C(1) << 26, (UINT64_C(1) << 26) + 1, + (UINT64_C(1) << 27) - 1, UINT64_C(1) << 51, (UINT64_C(1) << 52) - 1, (UINT64_C(1) << 52) - 2, + UINT64_C(0x5555555555555), UINT64_C(0xaaaaaaaaaaaaa), UINT64_C(0x8000000000001), + UINT64_C(0xfffffff800000), UINT64_C(0x921fb54442d18)}; + static const uint32_t e32[] = {1, 2, 24, 25, 64, 100, 126, 127, 128, 150, 190, 230, 253, 254}; + static const uint32_t f32[] = {0, 1, 2, 3, 0xfff, 0x1000, 0x1001, 0x400000, 0x7fffff, 0x7ffffe, 0x555555, 0x2aaaaa, + 0x400001, 0x7ff000, 0x490fdb}; + static const uint64_t fpcr[] = {0, 1u << 24, 0, 1u << 24}, fpsr[] = {0x1f, 0x1f, 0, 0x8f}; + const int ne = sizeof e64 / sizeof e64[0], nf = sizeof f64 / sizeof f64[0]; + unsigned i = 0; + for (int ea = 0; ea < ne; ea++) for (int fa = 0; fa < nf; fa++) + for (int eb = 0; eb < ne; eb++) for (int fb = 0; fb < nf; fb++) { + const uint64_t a = ((uint64_t)e64[ea] << 52) | f64[fa] | ((uint64_t)(fa & 1) << 63); + const uint64_t b = ((uint64_t)e64[eb] << 52) | f64[fb] | ((uint64_t)(eb & 1) << 63); + const double p = d_of(a) * d_of(b); + for (int k = 0; k < kKinds; k++, i++) + check(0, a, b, addend64(p, k, i), fpcr[i & 3], fpsr[i & 3]); + } + const int me = sizeof e32 / sizeof e32[0], mf = sizeof f32 / sizeof f32[0]; + for (int ea = 0; ea < me; ea++) for (int fa = 0; fa < mf; fa++) + for (int eb = 0; eb < me; eb++) for (int fb = 0; fb < mf; fb++) { + const uint32_t a = (e32[ea] << 23) | f32[fa] | ((uint32_t)(fa & 1) << 31); + const uint32_t b = (e32[eb] << 23) | f32[fb] | ((uint32_t)(eb & 1) << 31); + const float p = f_of(a) * f_of(b); + for (int k = 0; k < kKinds; k++, i++) + check(1, a, b, addend32(p, k, i), fpcr[i & 3], fpsr[i & 3]); + } +} + +static double now_ns(void) { +#ifdef _WIN32 + LARGE_INTEGER f, t; + QueryPerformanceFrequency(&f), QueryPerformanceCounter(&t); + return (double)t.QuadPart * 1e9 / (double)f.QuadPart; +#else + struct timespec t; + clock_gettime(CLOCK_MONOTONIC, &t); + return t.tv_sec * 1e9 + t.tv_nsec; +#endif +} +static void timing(void) { + enum { kN = 4096, kRounds = 300 }; + static uint64_t a[kN], b[kN], z[kN]; + static Ctx c; + for (int w32 = 0; w32 < 2; w32++) { + for (int i = 0; i < kN; i++) { + if (w32) a[i] = ((107 + rbelow(41)) << 23) | frac32(), b[i] = ((107 + rbelow(41)) << 23) | frac32(), z[i] = ((107 + rbelow(41)) << 23) | frac32(); + else a[i] = ((uint64_t)(983 + rbelow(81)) << 52) | frac64(), b[i] = ((uint64_t)(983 + rbelow(81)) << 52) | frac64(), z[i] = ((uint64_t)(983 + rbelow(81)) << 52) | frac64(); + } + double ns[2]; + for (int on = 1; on >= 0; on--) { + fma_soft_test_on = on; + double best = 1e30; + uint64_t sink = 0; + for (int rep = 0; rep < 3; rep++) { + const double t0 = now_ns(); + for (int r = 0; r < kRounds; r++) + for (int i = 0; i < kN; i++) sink += w32 ? recomp_fpe_fma32(&c, a[i], b[i], z[i]) : recomp_fpe_fma64(&c, a[i], b[i], z[i]); + const double t = (now_ns() - t0) / ((double)kRounds * kN); + if (t < best) best = t; + } + ns[on] = best; + if (!sink) printf(" "); + } + printf(" recomp_fpe_fma%d, plain operands: %.1f ns with the way out, %.1f ns without (%.1fx)\n", w32 ? 32 : 64, ns[1], ns[0], ns[0] / ns[1]); + } +} + +int main(int argc, char **argv) { + unsigned eax, ebx, ecx, edx; +#ifdef __SSE4_2__ + if (!__get_cpuid(1, &eax, &ebx, &ecx, &edx) || !(ecx >> 9 & 1) || !(ecx >> 20 & 1) || !(ecx >> 23 & 1)) { + printf("this CPU has no SSSE3/SSE4.2/POPCNT\n"); + return 3; + } +#endif +#ifdef __FMA__ + if (!__get_cpuid(1, &eax, &ebx, &ecx, &edx) || !(ecx >> 12 & 1)) { + printf("this CPU has no FMA\n"); + return 3; + } +#endif + (void)eax, (void)ebx, (void)ecx, (void)edx; + if (argc == 2 && !strcmp(argv[1], "env")) { /* (built without the hook: HOS_FMA_FAST as the game reads it) */ + Ctx c; + memset(&c, 0, sizeof c); + uint64_t sink = 0; + for (int i = 0; i < 1000; i++) + sink += recomp_fpe_fma64(&c, d_bits(1.5 + i), d_bits(3.25), d_bits(0.1)) + recomp_fpe_fma32(&c, f_bits(1.5f + i), f_bits(3.25f), f_bits(0.1f)); + printf("fma64=%d exits=%lld sum=%llx\n", RECOMP_FPX_FMA64, fma_soft_test_exits, (unsigned long long)sink); + return 0; + } + if (argc == 2 && !strcmp(argv[1], "time")) { timing(); return 0; } + if (argc == 4 && !strcmp(argv[1], "random")) { + rs = strtoull(argv[3], 0, 10); + if (!rs) rs = 1; + random_cases(atoll(argv[2])); + } else if (argc == 2 && !strcmp(argv[1], "sweep")) { + sweep(); + } else { + fprintf(stderr, "usage: random | sweep | env | time\n"); + return 2; + } + for (int w = 0; w < 2; w++) + printf("%s fma%d fma64=%d checks=%lld mismatches=%lld exits=%lld new_ixc=%lld exact=%lld fz=%lld\n", argv[1], w ? 32 : 64, + RECOMP_FPX_FMA64, st[w].checks, st[w].bad, st[w].exits, st[w].new_ixc, st[w].exact, st[w].fz); + return st[0].bad || st[1].bad ? 1 : 0; +} diff --git a/tests/fma_steps_side.c b/tests/fma_steps_side.c new file mode 100644 index 0000000..cc51f0f --- /dev/null +++ b/tests/fma_steps_side.c @@ -0,0 +1,26 @@ +/* One side of tests/test_fma_steps.py's comparison ((Triforce, 11-22, the performance audit's LIFT-05)): the guest words' + lifted C compiled against one runtime header. Built twice, -DSIDE=now with the header nx2/fma_steps.py writes and + -DSIDE=old with the kit's (A64_NO_FMA_STEPS=1); the same lifted text and the same a64_fpx.h in both. */ +#include "a64.h" +#include "fma_steps_lifted.h" + +#define CAT_(a, b) a##b +#define CAT(a, b) CAT_(a, b) + +/* form 0: as lifted; 1: as a hot function has it; 2: the exact body inline (translated with the fast path off) */ +void CAT(SIDE, _run)(int k, int form, Ctx *c) { (form == 0 ? CASES[k].lifted : form == 1 ? CASES[k].hot : CASES[k].exact)(c); } +int CAT(SIDE, _count)(void) { return (int)(sizeof CASES / sizeof CASES[0]); } +const char *CAT(SIDE, _name)(int k) { return CASES[k].name; } +int CAT(SIDE, _rn)(int k) { return CASES[k].rn; } +int CAT(SIDE, _rm)(int k) { return CASES[k].rm; } +int CAT(SIDE, _dbl)(int k) { return CASES[k].dbl; } +int CAT(SIDE, _recps32)(uint64_t a, uint64_t b, uint32_t fz, uint64_t *v) { return recomp_fpx_recps32(a, b, fz, v); } +int CAT(SIDE, _rsqrts32)(uint64_t a, uint64_t b, uint32_t fz, uint64_t *v) { return recomp_fpx_rsqrts32(a, b, fz, v); } +/* whether this build's steps are the fused instruction (the guard nx2/fma_steps.py gives them) */ +int CAT(SIDE, _fma)(void) { +#if RECOMP_FPX_HOST == 2 && defined(__FMA__) && !defined(A64_FPX_NO_FMA32) + return 1; +#else + return 0; +#endif +} diff --git a/tests/fma_steps_test.c b/tests/fma_steps_test.c new file mode 100644 index 0000000..a7d5283 --- /dev/null +++ b/tests/fma_steps_test.c @@ -0,0 +1,211 @@ +/* tests/test_fma_steps.py's driver ((Triforce, 11-22, the performance audit's LIFT-05)): guest FRECPS and FRSQRTS words + compiled against the runtime header nx2/fma_steps.py writes (now_*) and against the kit's (old_*), tests/fma_steps_side.c + built twice. + + fma_steps_test lifted n random states for each guest word: the word as lifted and as a hot function has + it, on both sides, and the exact body inline; the whole Ctx must be the same in all + fma_steps_test direct the two fast paths by themselves on n operand pairs each, FZ on and off: what both + sides keep must be the same bits, what either keeps must be the exact body's value + with FPSR untouched, and the old side must keep nothing the new one declines, but + for an FRSQRTS step of 2^127 or more: there the fused 3 - a*b is past binary32 + before its halving, the new side declines, and the exact body answers + Operands: zeros, subnormals, infinities, NaNs of both kinds with payloads, the extreme normals, any exponent, products + near 2 and near 3 (the step cancels), and pairs whose step is exactly a binary32 midpoint in binary64. + Exit 0: no mismatch. 1: a mismatch. 3: this CPU cannot run the build. */ +#include "a64.h" +#include +#include +#include + +#define SIDE_DECL(s) \ + void s##_run(int k, int form, Ctx *c); int s##_count(void); const char *s##_name(int k); int s##_rn(int k); \ + int s##_rm(int k); int s##_dbl(int k); int s##_recps32(uint64_t, uint64_t, uint32_t, uint64_t *); \ + int s##_rsqrts32(uint64_t, uint64_t, uint32_t, uint64_t *); int s##_fma(void); +SIDE_DECL(now) +SIDE_DECL(old) + +static uint64_t rs = 1; +static uint64_t rnd(void) { rs ^= rs << 13; rs ^= rs >> 7; rs ^= rs << 17; return rs; } +static uint32_t rbelow(uint32_t n) { return (uint32_t)(((rnd() >> 32) * n) >> 32); } +static float f_of(uint32_t x) { float f; memcpy(&f, &x, 4); return f; } +static uint32_t f_bits(float f) { uint32_t x; memcpy(&x, &f, 4); return x; } + +static uint64_t rand_fpcr(void) { + static const uint64_t bits[] = {1u << 22, 1u << 23, 3u << 22, 1u << 25, 1u << 26, 1, 2, 4, 1u << 8, 1u << 9, 1u << 10, + 1u << 11, 1u << 12, 1u << 15, 1u << 19, UINT64_C(1) << 32}; + const uint32_t k = rbelow(100); + if (k < 50) return 0; + if (k < 85) return 1u << 24; /* FZ alone */ + uint64_t v = bits[rbelow(sizeof bits / sizeof bits[0])]; + if (rnd() & 1) v |= 1u << 24; + return v; +} +static uint64_t rand_fpsr(void) { /* (IXC set, the fast path's gate, three times in four) */ + uint64_t v = rnd() & 0x9f; + if (!(rnd() & 3)) v |= 1u << 27; + return (rnd() & 3) ? v | 16u : v & ~(uint64_t)16u; +} + +static uint32_t frac32(void) { + switch (rbelow(8)) { + case 0: return 0; + case 1: return 0x7fffff; + case 2: return (uint32_t)rnd() & 0x7fffff & ~((1u << (1 + rbelow(22))) - 1); + case 3: return (1u << rbelow(23)) | ((uint32_t)rnd() & 1); + default: return (uint32_t)rnd() & 0x7fffff; + } +} +static uint32_t any32(void) { + const uint32_t k = rbelow(100); + const uint32_t sign = (uint32_t)(rnd() & 1) << 31; + if (k < 4) return sign; + if (k < 8) return sign | (frac32() ? frac32() : 1); + if (k < 10) return sign | 0x7f800000u; + if (k < 12) return sign | 0x7fc00000u | ((uint32_t)rnd() & 0x3fffff); + if (k < 14) return sign | 0x7f800000u | (((uint32_t)rnd() & 0x3fffff) | 1); + if (k < 17) return sign | ((rnd() & 1) ? 0x00800000u | ((uint32_t)rnd() & 3) : 0x7f7fffffu - ((uint32_t)rnd() & 3)); + if (k < 35) return sign | ((1 + rbelow(254)) << 23) | frac32(); + return sign | ((117 + rbelow(21)) << 23) | frac32(); +} +/* a pair for a step: step 2 for FRECPS (2 - a*b), 3 for FRSQRTS ((3 - a*b) / 2) */ +static void pair32(int step, uint32_t *a, uint32_t *b) { + const uint32_t k = rbelow(100); + *a = any32(), *b = any32(); + if (k < 40) return; + if (k < 60) { /* the step exactly a binary32 midpoint in binary64 (the pairs the binary64 form declines) */ + const uint32_t i = rbelow(512) * 2 + 1, j = rbelow(256) * 2 + 1; + if (step == 2) *a = 0x3f800000u | (i << 11), *b = 0xbf800000u | (j << 12); /* (1 + i/2^12) * -(1 + j/2^11) */ + else *a = 0x3f800000u | (i << 11), *b = 0x3f800000u | (j << 11); /* (1 + i/2^12) * (1 + j/2^12) */ + if (rnd() & 1) { const uint32_t t = *a; *a = *b, *b = t; } + return; + } + if (k < 80) { /* a*b near the step's constant: the step cancels (a tiny, subnormal or zero result) */ + const uint32_t x = ((120 + rbelow(15)) << 23) | frac32(); + const uint32_t y = f_bits((float)step / f_of(x)) + (uint32_t)((int)rbelow(9) - 4); + *a = x, *b = y; + if (rnd() & 1) *a ^= 0x80000000u, *b ^= 0x80000000u; + return; + } + if (k < 90) { /* a large or a small product */ + *a = ((rnd() & 1 ? 180 + rbelow(74) : 1 + rbelow(60)) << 23) | frac32() | ((uint32_t)(rnd() & 1) << 31); + *b = ((rnd() & 1 ? 180 + rbelow(74) : 1 + rbelow(60)) << 23) | frac32() | ((uint32_t)(rnd() & 1) << 31); + return; + } + *a = ((122 + rbelow(11)) << 23) | frac32() | ((uint32_t)(rnd() & 1) << 31); /* ordinary: the step is an ordinary number */ + *b = ((122 + rbelow(11)) << 23) | frac32() | ((uint32_t)(rnd() & 1) << 31); +} +static uint64_t any64(void) { + const uint32_t k = rbelow(100); + const uint64_t sign = (rnd() & 1) << 63, frac = rnd() & ((UINT64_C(1) << 52) - 1); + if (k < 4) return sign; + if (k < 8) return sign | (frac >> rbelow(52)) | 1; + if (k < 10) return sign | UINT64_C(0x7ff0000000000000); + if (k < 14) return sign | UINT64_C(0x7ff0000000000000) | (frac | 1); + if (k < 30) return sign | ((uint64_t)(1 + rbelow(2046)) << 52) | frac; + return sign | ((uint64_t)(1013 + rbelow(21)) << 52) | ((rnd() & 3) ? frac : 0); +} + +static int lifted(long n) { + const int ncases = now_count(); + long long checks = 0, bad = 0; + for (int k = 0; k < ncases; k++) { + const int rn = now_rn(k), rm = now_rm(k), dbl = now_dbl(k); + const int step = strstr(now_name(k), "FRSQRTS") ? 3 : 2; + for (long i = 0; i < n; i++) { + static Ctx base, r[5]; + for (int v = 0; v < 32; v++) base.vreg[v][0] = rnd(), base.vreg[v][1] = rnd(); + if (dbl) { + for (int h = 0; h < 2; h++) base.vreg[rn][h] = any64(), base.vreg[rm][h] = any64(); + } else { + uint32_t a[4], b[4]; + for (int l = 0; l < 4; l++) pair32(step, &a[l], &b[l]); + memcpy(base.vreg[rm], b, 16); /* (rm first: where rn is rm the pairs are (a, a)) */ + memcpy(base.vreg[rn], a, 16); + } + for (int x = 0; x < 31; x++) base.x[x] = rnd(); + base.n = (uint8_t)(i & 1), base.z = (uint8_t)(i >> 1 & 1), base.c = (uint8_t)(i >> 2 & 1), base.v = (uint8_t)(i >> 3 & 1); + base.fpcr = rand_fpcr(), base.fpsr = rand_fpsr(); + for (int j = 0; j < 5; j++) r[j] = base; + now_run(k, 0, &r[0]), now_run(k, 1, &r[1]), old_run(k, 0, &r[2]), old_run(k, 1, &r[3]), now_run(k, 2, &r[4]); + checks++; + int diff = 0; + for (int j = 0; j < 4; j++) diff |= memcmp(&r[j], &r[4], sizeof r[0]) ? 1 << j : 0; + if (diff && bad++ < 8) { + static const char *const who[4] = {"now", "now hot", "old", "old hot"}; + for (int j = 0; j < 4; j++) + if (diff >> j & 1) + printf("MISMATCH %s (%s against the exact body): fpcr %llx fpsr %llx; rn %016llx%016llx rm %016llx%016llx\n", + now_name(k), who[j], (unsigned long long)base.fpcr, (unsigned long long)base.fpsr, + (unsigned long long)base.vreg[rn][1], (unsigned long long)base.vreg[rn][0], + (unsigned long long)base.vreg[rm][1], (unsigned long long)base.vreg[rm][0]); + } + } + } + printf("lifted fused=%d shapes=%d checks=%lld mismatches=%lld\n", now_fma(), ncases, checks, bad); + return bad ? 1 : 0; +} + +static struct { long long checks, kept_now, kept_old, now_only, old_only, bad; } st; +static void one(int half, uint32_t a, uint32_t b, uint32_t fz) { + uint64_t vn = 0, vo = 0; + const int kn = half ? now_rsqrts32(a, b, fz, &vn) : now_recps32(a, b, fz, &vn); + const int ko = half ? old_rsqrts32(a, b, fz, &vo) : old_recps32(a, b, fz, &vo); + st.checks++, st.kept_now += kn, st.kept_old += ko, st.now_only += kn && !ko, st.old_only += ko && !kn; + int bad = (kn && ko && vn != vo) || (ko && !kn && !(half && (vo & 0x7f800000u) == 0x7f000000u)); + if (kn || ko) { /* (what a fast path keeps is Arm's value, and the exact body raises no new flag for it) */ + Ctx c; + memset(&c, 0, sizeof c); + c.fpcr = fz, c.fpsr = 0x10 | (rnd() & 0x8f); + const uint64_t s = c.fpsr; + const uint64_t e = half ? recomp_fpe_rsqrts32(&c, a, b) : recomp_fpe_recps32(&c, a, b); + bad |= (kn && vn != e) || (ko && vo != e) || c.fpsr != s; + } + if (bad && st.bad++ < 8) + printf("MISMATCH %s a %08x b %08x fz %d: now %d %08llx, old %d %08llx\n", half ? "rsqrts32" : "recps32", a, b, fz != 0, kn, + (unsigned long long)vn, ko, (unsigned long long)vo); +} +static int direct(long n) { + /* the audit's two midpoint pairs first: kept by the fused form, declined by the binary64 one */ + static const uint32_t named[2][2] = {{0x3f800800u, 0xbf801000u}, {0x3f800800u, 0x3f800800u}}; + int named_ok = 1; + for (int half = 0; half < 2; half++) { + uint64_t vn = 0, vo = 0; + const int kn = half ? now_rsqrts32(named[1][0], named[1][1], 0, &vn) : now_recps32(named[0][0], named[0][1], 0, &vn); + const int ko = half ? old_rsqrts32(named[1][0], named[1][1], 0, &vo) : old_recps32(named[0][0], named[0][1], 0, &vo); + if (ko || kn != now_fma()) named_ok = 0; + one(half, named[half][0], named[half][1], 0); + } + for (long i = 0; i < n; i++) + for (int half = 0; half < 2; half++) { + uint32_t a, b; + pair32(half ? 3 : 2, &a, &b); + one(half, a, b, (rnd() & 1) ? 0x01000000u : 0); + } + printf("direct fused=%d named=%d checks=%lld kept_now=%lld kept_old=%lld now_only=%lld old_only=%lld mismatches=%lld\n", now_fma(), + named_ok, st.checks, st.kept_now, st.kept_old, st.now_only, st.old_only, st.bad); + return st.bad ? 1 : 0; +} + +int main(int argc, char **argv) { + unsigned eax, ebx, ecx, edx; +#ifdef __SSE4_2__ + if (!__get_cpuid(1, &eax, &ebx, &ecx, &edx) || !(ecx >> 9 & 1) || !(ecx >> 20 & 1) || !(ecx >> 23 & 1)) { + printf("this CPU has no SSSE3/SSE4.2/POPCNT\n"); + return 3; + } +#endif +#ifdef __FMA__ + if (!__get_cpuid(1, &eax, &ebx, &ecx, &edx) || !(ecx >> 12 & 1)) { + printf("this CPU has no FMA\n"); + return 3; + } +#endif + (void)eax, (void)ebx, (void)ecx, (void)edx; + if (argc != 4 || (strcmp(argv[1], "lifted") && strcmp(argv[1], "direct"))) { + fprintf(stderr, "usage: lifted | direct \n"); + return 2; + } + rs = strtoull(argv[3], 0, 10); + if (!rs) rs = 1; + return argv[1][0] == 'l' ? lifted(atol(argv[2])) : direct(atol(argv[2])); +} diff --git a/tests/gpu_waits_test.cpp b/tests/gpu_waits_test.cpp new file mode 100644 index 0000000..6d83153 --- /dev/null +++ b/tests/gpu_waits_test.cpp @@ -0,0 +1,160 @@ +// tests/test_gpu_waits.py: gpu/vk/gpu_wait_kinds.h (the host's time in calls that block on the GPU, by kind) and vk_work.h's +// five wrappers, put in the place of WRAPPERS and run over stand-ins for the driver's calls. No game, no GPU. +// (Triforce, 11-22, the performance audit's SYNC-27, after Source A's.) +// +// The calls below are vk_work.h's own five macros (their lines put in the place of MACROS), over its wrappers. +// on RX_VK_GPU_WAITS unset: everything below +// off RX_VK_GPU_WAITS=0 (set by the test): the calls are made, their results and arguments pass, nothing is counted +#include +#include +#include +#include +#include +#include +#include +#include "gpu_wait_kinds.h" + +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +namespace gw = gpu_wait_kinds; + +// ---- stand-ins for the driver ---- +typedef int VkResult; +typedef void* VkQueue; +typedef void* VkDevice; +typedef void* VkFence; +typedef void* VkQueryPool; +typedef uint32_t VkBool32; +typedef uint64_t VkDeviceSize; +typedef uint32_t VkQueryResultFlags; +struct VkSemaphoreWaitInfo { int tag; }; +constexpr VkQueryResultFlags VK_QUERY_RESULT_64_BIT = 1, VK_QUERY_RESULT_WAIT_BIT = 2; +static int calls[5]; +static int64_t sleep_us = 0; // (what each driver call takes) +static uint64_t seen_timeout; +static VkQueryResultFlags seen_flags; +static void work() { + if (sleep_us) std::this_thread::sleep_for(std::chrono::microseconds(sleep_us)); +} +namespace vkx { +VkResult vkQueueWaitIdle(VkQueue q) { calls[0]++, work(); return q ? 11 : -1; } +VkResult vkDeviceWaitIdle(VkDevice d) { calls[1]++, work(); return d ? 12 : -1; } +VkResult vkWaitForFences(VkDevice d, uint32_t n, const VkFence* f, VkBool32 all, uint64_t timeout) { + calls[2]++, seen_timeout = timeout, work(); + return d && n == 3 && f && all == 1 ? 13 : -1; +} +VkResult vkWaitSemaphores(VkDevice d, const VkSemaphoreWaitInfo* info, uint64_t timeout) { + calls[3]++, seen_timeout = timeout, work(); + return d && info && info->tag == 77 ? 14 : -1; +} +VkResult vkGetQueryPoolResults(VkDevice d, VkQueryPool pool, uint32_t first, uint32_t count, size_t size, void* data, VkDeviceSize stride, + VkQueryResultFlags flags) { + calls[4]++, seen_flags = flags, work(); + return d && pool && first == 4 && count == 5 && size == 40 && data && stride == 8 ? 15 : -1; +} +} // namespace vkx + +/* WRAPPERS */ +// (the five macros as vk_work.h has them, the recorder's boundary a count here: the test puts their lines in the place of MACROS) +static int boundaries = 0; +namespace mkwvk::rec { +enum { B_vkQueueWaitIdle, B_vkDeviceWaitIdle, B_vkWaitForFences, B_vkWaitSemaphores, B_vkGetQueryPoolResults }; +inline void boundary(int) { boundaries++; } +} +/* MACROS */ + +static int64_t ns_of(gw::Kind k) { return gw::totals.ns[k].load(); } +static int64_t n_of(gw::Kind k) { return gw::totals.n[k].load(); } +static int64_t n_all() { return n_of(gw::FENCE) + n_of(gw::TIMELINE) + n_of(gw::IDLE) + n_of(gw::QUERY); } + +int main(int argc, char** argv) { + const bool on = !(argc > 1 && std::string(argv[1]) == "off"); + int dev = 0, x = 0; + VkFence list[3] = {&x, &x, &x}; + VkSemaphoreWaitInfo info{77}; + uint64_t data[5]; + CHECK(gw::on() == on); + + // ---- the wrappers: the driver is called with what they were given, its result comes back, and they count by kind when they block + sleep_us = 0; + CHECK(vkWaitForFences(&dev, 3, list, 1, 0) == 13 && calls[2] == 1 && seen_timeout == 0 && n_all() == 0); // (a poll: nothing) + CHECK(vkWaitSemaphores(&dev, &info, 0) == 14 && calls[3] == 1 && seen_timeout == 0 && n_all() == 0); // (a poll: nothing) + CHECK(vkGetQueryPoolResults(&dev, &x, 4, 5, 40, data, 8, VK_QUERY_RESULT_64_BIT) == 15 && calls[4] == 1 && seen_flags == VK_QUERY_RESULT_64_BIT && n_all() == 0); + sleep_us = 3000; + CHECK(vkWaitForFences(&dev, 3, list, 1, 250000000ull) == 13 && seen_timeout == 250000000ull); + CHECK(n_of(gw::FENCE) == (on ? 1 : 0) && n_of(gw::TIMELINE) == 0 && n_of(gw::IDLE) == 0 && n_of(gw::QUERY) == 0); + if (on) CHECK(ns_of(gw::FENCE) >= 3000000 && ns_of(gw::TIMELINE) == 0 && ns_of(gw::IDLE) == 0 && ns_of(gw::QUERY) == 0); + CHECK(vkWaitSemaphores(&dev, &info, ~0ull) == 14 && seen_timeout == ~0ull && n_of(gw::TIMELINE) == (on ? 1 : 0)); + CHECK(vkQueueWaitIdle(&dev) == 11 && vkDeviceWaitIdle(&dev) == 12 && calls[0] == 1 && calls[1] == 1 && n_of(gw::IDLE) == (on ? 2 : 0)); + CHECK(vkGetQueryPoolResults(&dev, &x, 4, 5, 40, data, 8, VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WAIT_BIT) == 15 && n_of(gw::QUERY) == (on ? 1 : 0)); + if (on) CHECK(ns_of(gw::TIMELINE) >= 3000000 && ns_of(gw::IDLE) >= 6000000 && ns_of(gw::QUERY) >= 3000000); + CHECK(n_all() == (on ? 5 : 0) && gw::depth == 0 && gw::excluded == 0); + CHECK(boundaries == 8); // (the recorder's boundary before each of the eight calls so far, counted or not) + if (!on) { + CHECK(ns_of(gw::FENCE) + ns_of(gw::TIMELINE) + ns_of(gw::IDLE) + ns_of(gw::QUERY) == 0); + gw::Reader r; + CHECK(r.take(nullptr, nullptr) == 0); + std::printf("off: the five calls made, nothing counted\n"); + return fails != 0; + } + + // ---- two readers: each sees every nanosecond and every wait, whenever it reads + gw::Reader a, b; + int64_t ns[gw::KINDS], n[gw::KINDS]; + const int64_t a1 = a.take(ns, n); + CHECK(a1 == ns[0] + ns[1] + ns[2] + ns[3] && n[gw::FENCE] == 1 && n[gw::TIMELINE] == 1 && n[gw::IDLE] == 2 && n[gw::QUERY] == 1); + CHECK(a.take(ns, n) == 0 && n[0] + n[1] + n[2] + n[3] == 0); // (nothing since) + vkWaitForFences(&dev, 3, list, 1, 1); + const int64_t a2 = a.take(ns, n); + CHECK(a2 >= 3000000 && n[gw::FENCE] == 1 && ns[gw::FENCE] == a2); + const int64_t b1 = b.take(ns, n); // (the other reader reads once, later: all of it) + CHECK(b1 == a1 + a2 && n[gw::FENCE] == 2 && n[0] + n[1] + n[2] + n[3] == 6); + CHECK(b1 == ns_of(gw::FENCE) + ns_of(gw::TIMELINE) + ns_of(gw::IDLE) + ns_of(gw::QUERY)); + + // ---- a wait inside a wait counts once, as the outer one + { + const int64_t f0 = n_of(gw::FENCE), t0 = n_of(gw::TIMELINE), fns = ns_of(gw::FENCE), tns = ns_of(gw::TIMELINE); + const int r = gw::timed(gw::FENCE, true, [&] { return vkWaitSemaphores(&dev, &info, 5) + 100; }); + CHECK(r == 114 && n_of(gw::FENCE) == f0 + 1 && n_of(gw::TIMELINE) == t0 && ns_of(gw::TIMELINE) == tns); + CHECK(ns_of(gw::FENCE) - fns >= 3000000 && gw::depth == 0); + } + // ---- excluded waits count nothing, and the count comes back after the scope (one inside another too) + { + const int64_t all0 = n_all(); + { + const gw::Exclude pacing; + CHECK(vkWaitSemaphores(&dev, &info, 100000000ull) == 14 && n_all() == all0); + { + const gw::Exclude again; + CHECK(vkWaitForFences(&dev, 3, list, 1, 9) == 13 && n_all() == all0); + } + CHECK(vkQueueWaitIdle(&dev) == 11 && n_all() == all0); + } + CHECK(gw::excluded == 0 && vkQueueWaitIdle(&dev) == 11 && n_all() == all0 + 1); + } + // ---- every thread's waits are counted; one thread inside a wait or an Exclude does not hide another's + { + sleep_us = 0; + const int64_t all0 = n_all(); + std::atomic go{false}, inside{false}, done{false}; + std::thread outer([&] { + const gw::Exclude pacing; // (this thread: excluded for as long as the others run) + inside = true; + while (!done) std::this_thread::yield(); + }); + while (!inside) std::this_thread::yield(); + std::vector threads; + for (int t = 0; t < 4; t++) + threads.emplace_back([&] { + while (!go) std::this_thread::yield(); + for (int i = 0; i < 500; i++) vkWaitForFences(&dev, 3, list, 1, 1), vkWaitForFences(&dev, 3, list, 1, 0); + }); + go = true; + for (auto& t : threads) t.join(); + done = true; + outer.join(); + CHECK(n_all() == all0 + 4 * 500); + } + std::printf("on: polls and excluded waits count nothing, a nested wait once, the kinds apart, two readers the whole each, 2,000 waits of 4 threads\n"); + return fails != 0; +} diff --git a/tests/helper_spin_test.cpp b/tests/helper_spin_test.cpp index ca83f40..7896e6f 100644 --- a/tests/helper_spin_test.cpp +++ b/tests/helper_spin_test.cpp @@ -64,8 +64,8 @@ static bool all_done(int seconds) { } static void case_until() { - CHECK(helper_spin::bound_ns(nullptr) == 100000); - CHECK(helper_spin::bound_ns("") == 100000); + CHECK(helper_spin::bound_ns(nullptr) == 20000); + CHECK(helper_spin::bound_ns("") == 20000); CHECK(helper_spin::bound_ns("0") == 0); CHECK(helper_spin::bound_ns("-5") == 0); CHECK(helper_spin::bound_ns("250") == 250000); diff --git a/tests/input_background_test.c b/tests/input_background_test.c index 09b15d5..a7026f1 100644 --- a/tests/input_background_test.c +++ b/tests/input_background_test.c @@ -87,7 +87,10 @@ void binds_load(void) { read_input calls */ int hos_mouse_native(void) { return -1; } void hos_mouse_native_free(int free) { (void)free; } -void hos_mouse_native_stick(int held) { (void)held; } +/* ((Triforce, 11-22) whether the right stick turns the camera, told after every read: behind another window too, where the + pads alone turned it, and the native mouse put the angles back each frame) */ +static int native_held = -1; +void hos_mouse_native_stick(int held) { native_held = held; } static void mouse_setup(void) {} static void mouse_sens_keys(void) {} int mouse_stick(int32_t *x, int32_t *y, int held) { (void)x; (void)y; (void)held; mouse_queries++; return 0; } @@ -113,8 +116,10 @@ static void packet(uint64_t buttons, int32_t lx, int32_t ly, int32_t rx) { context.x[0] = 64; context.x[1] = 2; context.x[2] = 16; context.x[30] = 0x1234; G32(16) = NPAD_NO1; memset(memory + 64, 0xa5, 2 * STATE_SIZE + 8); + native_held = -1; npad_states(&context); CHECK(context.x[0] == 2); + CHECK(native_held == (rx != 0)); /* (the stick the game is given, told to the native mouse, the window in front or not) */ expected_sampling++; for (int i = 0; i < 2; i++) { const uint64_t address = 64 + (uint64_t)i * STATE_SIZE; diff --git a/tests/invalidate_vcore_test.cpp b/tests/invalidate_vcore_test.cpp index 18deee3..27333bd 100644 --- a/tests/invalidate_vcore_test.cpp +++ b/tests/invalidate_vcore_test.cpp @@ -64,13 +64,24 @@ static void renderer(uint64_t gpu, uint64_t size) { } static void vulkan_invalidate(uint64_t gpu, uint64_t size) { vulkan_calls++; renderer(gpu, size); } static void wgr_nvn_invalidate(uint64_t gpu, uint64_t size) { fallback_calls++; renderer(gpu, size); } +// ((Triforce, THREAD-25) the back end's answer to "has this invalidate nothing to wait for?" (mkwvk_invalidate_quick, found by name +// in the product): asked with the caller's core still held, before anything is given up +static int quick_calls, quick_answer; +static int quick(uint64_t gpu, uint64_t size) { + quick_calls++; + CHECK(gpu == want_gpu && size == want_size); + CHECK(!callback_active); + if (cur && enabled) CHECK(cur->vc_held == 1 && cur->releases == 0); + return quick_answer; +} +static int (*volatile invalidate_quick_fn)(uint64_t, uint64_t); #include "actual_invalidate.inc" int main(int argc, char** argv) { CHECK(argc == 2); const char* name = argv[1]; - const bool fallback = !std::strcmp(name, "fallback") || !std::strcmp(name, "fallback-immediate"); + const bool fallback = !std::strcmp(name, "fallback") || !std::strcmp(name, "fallback-immediate") || !std::strcmp(name, "quick-fallback"); const bool already_away = !std::strcmp(name, "already-away"); const bool non_guest = !std::strcmp(name, "non-guest"); enabled = std::strcmp(name, "disabled") != 0; @@ -78,6 +89,11 @@ int main(int argc, char** argv) { // A separate case demonstrates producer starvation on the original wrapper, beyond the entry invariant. require_no_lease = std::strcmp(name, "producer") != 0; if (!std::strcmp(name, "zero")) want_gpu = want_size = 0; + // ((Triforce, THREAD-25) quick: the back end says there is nothing to wait for: no call, no release, no take. quick-declined: it + // says there may be: everything as before. quick-fallback: no Vulkan back end is running: it is not asked) + const bool quick_yes = !std::strcmp(name, "quick"), quick_no = !std::strcmp(name, "quick-declined"); + const bool quick_fallback = !std::strcmp(name, "quick-fallback"); + if (quick_yes || quick_no || quick_fallback) invalidate_quick_fn = quick, quick_answer = !quick_no; MkwVkRenderer backend{}; backend.nvn_invalidate = vulkan_invalidate; vk = fallback ? nullptr : &backend; @@ -111,11 +127,12 @@ int main(int argc, char** argv) { } rx_nvn_invalidate(want_gpu, want_size); if (worker.joinable()) worker.join(); - CHECK(vulkan_calls == (fallback ? 0 : 1)); + CHECK(vulkan_calls == (fallback || quick_yes ? 0 : 1)); CHECK(fallback_calls == (fallback ? 1 : 0)); + CHECK(quick_calls == (quick_yes || quick_no ? 1 : 0)); const bool owned_on_entry = cur && enabled && !already_away; - CHECK(caller.releases - before_release == (owned_on_entry ? 1 : 0)); - CHECK(caller.takes - before_take == (owned_on_entry ? 1 : 0)); + CHECK(caller.releases - before_release == (owned_on_entry && !quick_yes ? 1 : 0)); + CHECK(caller.takes - before_take == (owned_on_entry && !quick_yes ? 1 : 0)); CHECK(caller.vc_held == (owned_on_entry ? 1 : 0)); CHECK(owner == (owned_on_entry ? &caller : nullptr)); if (need_producer) CHECK(producer_done && producer.takes == 1 && producer.releases == 1 && !producer.vc_held); diff --git a/tests/mirror_bookkeeping_test.cpp b/tests/mirror_bookkeeping_test.cpp new file mode 100644 index 0000000..0ccc9e1 --- /dev/null +++ b/tests/mirror_bookkeeping_test.cpp @@ -0,0 +1,119 @@ +// tests/test_mirror_bookkeeping.py puts vk_memory.cpp's own wb_notes_off, memory_cpu_owned (with its two merges), +// shadow_written_to and its two callers, guest_cpu_write and guest_cpu_wrote in the place of ACTUAL and runs them with +// and without mirrors. No game, no GPU. (Triforce, 11-22, the performance audit's MEM-51, after Source A's.) +// +// off mirrors off, the device made: nothing is noted, no guest page is looked up, the write still lands +// early mirrors off, the device not made yet: noted as before, and mirror_on() is not asked +// dump mirrors on: a fixed random run of calls, then every list printed. The test builds this file twice, +// from the code as it is and from the code with the new condition taken out (the code as it was): +// the two prints must be the same +#include +#include +#include +#include +#include +#include +#include + +#define VK_NULL_HANDLE nullptr +#define CHECK(ok) do { if (!(ok)) { std::printf("FAIL:%d %s\n", __LINE__, #ok); fails++; } } while (0) +static int fails = 0; +constexpr uint64_t kPageShift = 12; + +struct Dev { void* device = nullptr; } dev; +static bool mirrors = true; +static int mirror_asked = 0; +bool mirror_on() { return mirror_asked++, mirrors; } +bool wb_diff_on() { return true; } + +static int looked_up = 0, writes = 0; +struct Host { + static uint64_t gpu_guest(uint64_t gpu) { return looked_up++, gpu >= 0x100000 ? gpu - 0x100000 + 0x4000000 : 0; } + static int gpu_write(uint64_t, const void*, uint64_t) { return writes++, 1; } +}; +static Host the_host; +static Host* host = &the_host; + +std::mutex cpu_words_mu; +std::vector cpu_words, cpu_words_recent, cpu_words_new; +std::mutex shadow_written_mu; +std::vector shadow_written_pages, shadow_written_collected_list, shadow_written_by_replay; +bool shadow_written_all = false; +void shadow_written_to(std::vector& list, const uint32_t* pages, int64_t n); +void guest_cpu_wrote(uint64_t gpu, uint64_t n); + +/* ACTUAL */ + +static uint64_t rnd_state = 0x1234567887654321ull; +static uint64_t rnd() { + rnd_state ^= rnd_state << 13, rnd_state ^= rnd_state >> 7, rnd_state ^= rnd_state << 17; + return rnd_state; +} +static bool nothing_noted() { + return cpu_words.empty() && cpu_words_recent.empty() && cpu_words_new.empty() && shadow_written_pages.empty() && + shadow_written_collected_list.empty() && shadow_written_by_replay.empty() && !shadow_written_all; +} +/// The calls the replay and the capture make, n of them, from the generator's state +static void calls(int n) { + uint32_t pages[80]; + const uint32_t word = 0; + for (int i = 0; i < n; i++) { + const uint64_t at = 0x100000 + (rnd() % 4096) * 8, len = 1 + rnd() % 20000; + switch (rnd() % 5) { + case 0: memory_cpu_owned(rnd() % 9 == 0 ? 0 : 0x4000000 + (rnd() % 70000) * 8 + rnd() % 8, 1 + rnd() % 40); break; + case 1: { + const int k = (int)(rnd() % 80); + for (int j = 0; j < k; j++) pages[j] = (uint32_t)(rnd() % 100000); + shadow_written(pages, k); + break; + } + case 2: { + const int k = (int)(rnd() % 80); + for (int j = 0; j < k; j++) pages[j] = (uint32_t)(rnd() % 100000); + shadow_written_collected(pages, k); + break; + } + case 3: guest_cpu_wrote(rnd() % 7 == 0 ? 0x1000 : at, rnd() % 11 == 0 ? 0 : len); break; + default: guest_cpu_write(at, &word, 4); break; + } + } +} +template static void dump(const char* name, const std::vector& v) { + uint64_t h = 1469598103934665603ull; + for (const T& e : v) h = (h ^ (uint64_t)e) * 1099511628211ull; + std::printf("%s %zu %016llx\n", name, v.size(), (unsigned long long)h); +} + +int main(int argc, char** argv) { + const std::string test = argc == 2 ? argv[1] : ""; + if (test == "off") { + dev.device = &dev, mirrors = false; + calls(20000); + CHECK(nothing_noted()); + CHECK(looked_up == 0); // (guest_cpu_wrote asks the host for no page) + CHECK(writes > 1000); // (guest_cpu_write still writes) + CHECK(mirror_asked > 0); + } else if (test == "early") { + dev.device = nullptr, mirrors = false; + calls(2000); + CHECK(!nothing_noted() && looked_up > 0); + CHECK(mirror_asked == 0); // (mirror_on() decides once, by the card: not asked before there is one) + } else if (test == "dump") { + dev.device = &dev, mirrors = true; + for (int round = 0; round < 40; round++) { + calls(3000); + if (round % 8 == 7) { // (a write-back reads them: the lists start again, as shadows_forget_written leaves them) + std::lock_guard lk(shadow_written_mu); + shadow_written_pages.clear(), shadow_written_collected_list.clear(), shadow_written_by_replay.clear(), shadow_written_all = false; + } + dump("words", cpu_words), dump("recent", cpu_words_recent), dump("new", cpu_words_new); + dump("game", shadow_written_pages), dump("collected", shadow_written_collected_list), dump("replay", shadow_written_by_replay); + std::printf("all %d looked_up %d writes %d\n", (int)shadow_written_all, looked_up, writes); + } + CHECK(!nothing_noted() && looked_up > 0); + } else { + std::printf("a known case\n"); + return 1; + } + return fails != 0; +} diff --git a/tests/occ_answer_test.cpp b/tests/occ_answer_test.cpp index 45acbe4..2875adf 100644 --- a/tests/occ_answer_test.cpp +++ b/tests/occ_answer_test.cpp @@ -359,6 +359,91 @@ int main() { d.note(1), d.note(1), d.note(2); CHECK(d.take() == 2 && d.take() == 0); } + { // ((Triforce, 11-22, the performance audit's GPUWORK-01) a window with a draw that writes colour + bool keep_now = false; + Rep r; + // the rule off: counted for the line, and the count measured as before, the colour draw under its open query + Counter c; + c.reset(4, A, 10); + c.draw_enter(false); + Counter::Colour m = c.colour_draw(false, false); // (the window's first draw writes colour: no query open yet) + CHECK(m.first && !m.open && !m.lose && c.colour && !c.lost); + CHECK(c.may_begin(false) && c.begin(false, A)); + c.open = 10; + c.draw_issued(false); + c.draw_enter(false); + m = c.colour_draw(false, false); + CHECK(!m.first && m.open && !m.lose && !c.lost && c.open == 10); // (a second one, under the open query) + c.draw_issued(false); + c.open = -1; + CHECK(c.report(false, A, 11, &r, &keep_now) == Counter::kMeasured && r.first == 10 && r.n == 1 && c.why == Counter::kNone); + // the rule on: a depth-only window is measured as before (no colour draw is ever told to it) + Counter d; + d.reset(4, A, 20); + d.draw_enter(false); + CHECK(d.begin(false, A)); + d.open = 20; + d.draw_issued(false); + d.open = -1; + CHECK(d.report(false, A, 21, &r, &keep_now) == Counter::kMeasured && r.first == 20 && r.n == 1 && !d.colour); + // the rule on: a colour draw gives the count up; no query begins again until the next reset; its report is + // incomplete for that reason, so nothing is written for it + Counter e; + e.reset(4, A, 30); + e.draw_enter(false); + CHECK(e.begin(false, A)); + e.open = 30; + e.draw_issued(false); + e.draw_enter(false); + m = e.colour_draw(false, true); + CHECK(m.first && m.open && m.lose); + e.open = -1; // (the caller ends the open query: occ_close) + e.lose_colour(); + CHECK(e.lost && e.why == Counter::kColour && !e.may_begin(false) && !e.begin(false, A)); + e.draw_issued(false); + e.draw_enter(false); + m = e.colour_draw(false, true); // (a later colour draw of the window: nothing more) + CHECK(!m.first && !m.open && !m.lose); + e.draw_issued(false); + r = Rep{}; + CHECK(e.report(false, A, 31, &r, &keep_now) == Counter::kIncomplete && e.why == Counter::kColour && r.n == 0); + e.reset(4, A, 31); // (the next window begins afresh) + CHECK(!e.lost && !e.colour && e.why == Counter::kNone && e.may_begin(false)); + // a colour draw before the window's first query, the rule on: the same, with no query to end + Counter f; + f.reset(4, A, 40); + f.draw_enter(false); + m = f.colour_draw(false, true); + CHECK(m.first && !m.open && m.lose); + f.lose_colour(); + f.draw_issued(false); + CHECK(!f.may_begin(false) && f.report(false, A, 40, &r, &keep_now) == Counter::kIncomplete && f.why == Counter::kColour); + // not counting (no reset yet), or lost already: a colour draw is nothing to the count + Counter g; + m = g.colour_draw(false, true); + CHECK(!m.first && !m.open && !m.lose && !g.colour); + g.reset(4, 0, 0); // (no recording's: lost) + m = g.colour_draw(false, true); + CHECK(!m.first && !m.lose && g.why == Counter::kOtherCmd); + // counting past a report (keep): a colour draw after the report is told too + Counter h; + h.reset(4, A, 50); + CHECK(h.report(true, A, 50, &r, &keep_now) == Counter::kMeasured); + m = h.colour_draw(true, false); + CHECK(m.first && !m.lose); + m = h.colour_draw(false, false); // (without keep a reported count is not counting) + CHECK(!m.first && !m.open && !m.lose); + // the line + Stats st; + st.colour_windows = 12, st.colour_open = 640, st.colour_lost = 5, st.inc_colour = 4; + char line[512]; + st.colour_line(line, sizeof line, 600, true); + CHECK(std::strstr(line, "vk: occlusion queries (RX_VK_OCC_QUERY), 600 presents (3): 12 counting windows met a draw that writes " + "colour, 640 such draws were recorded under an open query; depth-only counting (RX_VK_OCC_DEPTH_ONLY) is " + "on: 5 counts given up at a colour draw, 4 reports kept 4096 for it") == line); + st.colour_line(line, sizeof line, 600, false); + CHECK(std::strstr(line, "(RX_VK_OCC_DEPTH_ONLY) is off: ") != nullptr); + } if (failures) std::fprintf(stderr, "%d failed\n", failures); else std::printf("occ_answer: all cases pass\n"); return failures ? 1 : 0; diff --git a/tests/page_table_stream_test.cpp b/tests/page_table_stream_test.cpp new file mode 100644 index 0000000..63ee5f9 --- /dev/null +++ b/tests/page_table_stream_test.cpp @@ -0,0 +1,162 @@ +// tests/test_page_table_stream.py puts vk_memory.cpp's own page table code (TableRun to memory_table_flush) in the place of +// ACTUAL and runs its flush, streamed (gpu/vk/table_stream.h) and sorted as it was, on the same changes. No game, no GPU. +// (Triforce, 11-22, the performance audit's SYNC-04, after Source A's.) +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "table_stream.h" + +// (the largest single allocation while `tracking` is set: the flush's scratch) +static bool tracking = false; +static size_t largest = 0; +static void* counted(size_t n) { + if (tracking && n > largest) largest = n; + void* p = std::malloc(n ? n : 1); + if (!p) std::abort(); + return p; +} +void* operator new(size_t n) { return counted(n); } +void* operator new[](size_t n) { return counted(n); } +void* operator new(size_t n, const std::nothrow_t&) noexcept { return counted(n); } +void* operator new[](size_t n, const std::nothrow_t&) noexcept { return counted(n); } +void operator delete(void* p) noexcept { std::free(p); } +void operator delete[](void* p) noexcept { std::free(p); } +void operator delete(void* p, size_t) noexcept { std::free(p); } +void operator delete[](void* p, size_t) noexcept { std::free(p); } + +namespace fixture { +int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) + +uint64_t* guest_pages = nullptr; // (nothing is written at once here: every change goes through the flush) +using VkBuffer = int; +using VkCommandBuffer = int; +VkBuffer guest_pages_buffer = 7; +std::atomic par_active{false}; +struct Update { + uint64_t first; + std::vector values; + bool operator==(const Update& o) const { return first == o.first && values == o.values; } +}; +std::vector updates; +size_t largest_update = 0; +void vkCmdUpdateBuffer(VkCommandBuffer, VkBuffer b, uint64_t offset, uint64_t bytes, const void* data) { + CHECK(b == guest_pages_buffer && offset % 8 == 0 && bytes % 8 == 0 && bytes > 0); + largest_update = std::max(largest_update, (size_t)bytes); + const bool was = tracking; + tracking = false; // (the stand-in's own copy is not the flush's scratch) + updates.push_back({offset / 8, std::vector((const uint64_t*)data, (const uint64_t*)data + bytes / 8)}); + tracking = was; +} +VkCommandBuffer upload_command_buffer() { return 1; } +struct GprofSpan { + int items = 0; + uint64_t bytes = 0; + GprofSpan(VkCommandBuffer, const char*) {} +}; +void logf(const char*, ...) {} + +/* ACTUAL */ + +uint64_t rnd_state = 0xD1B54A32D192ED03ull; +uint64_t rnd() { + rnd_state ^= rnd_state << 13, rnd_state ^= rnd_state >> 7, rnd_state ^= rnd_state << 17; + return rnd_state; +} +const uint64_t kNone = ~0ull; // (a page no change touched) + +/// The runs flushed both ways: the same transfers, entry for entry, no transfer over 64 KiB, and the table they leave is +/// the last writer's of every page +void both_ways(const std::vector& runs, uint64_t span) { + std::vector model(span, kNone); + for (const TableRun& r : runs) + for (uint64_t i = 0; i < r.count; i++) model[r.first + i] = r.address ? r.address + (i << 12) : 0; + updates.clear(), largest_update = 0; + table_pending = runs; + const bool any = memory_table_flush(); + CHECK(any == !runs.empty() && table_pending.empty()); + const std::vector streamed = updates; + CHECK(largest_update <= 65536); + std::vector table(span, kNone); + uint64_t next_free = 0; + for (const Update& u : streamed) { + CHECK(u.first >= next_free); // (ascending, no page twice) + for (size_t i = 0; i < u.values.size(); i++) table[u.first + i] = u.values[i]; + next_free = u.first + u.values.size(); + } + CHECK(table == model); + updates.clear(); + table_pending = runs; + if (!runs.empty()) table_flush_sorted(); + CHECK(table_pending.empty()); + CHECK(updates == streamed); +} + +int run(int argc, char** argv) { + const std::string test = argc == 2 ? argv[1] : ""; + if (test == "random") { + // 50 trials of 128 runs: anywhere in 4,000 pages, so most overlap, in any order + for (int t = 0; t < 50; t++) { + std::vector runs; + for (int i = 0; i < 128; i++) { + const uint64_t count = rnd() % 5 == 0 ? rnd() % 1200 : rnd() % 40; // (some of 0 pages) + const uint64_t first = rnd() % (4000 - count); + runs.push_back({first, count, rnd() % 4 == 0 ? 0 : ((rnd() % 0xFFFFFF) + 1) << 12}); + } + both_ways(runs, 4000); + } + // in page order and apart (the usual case: no sweep), some touching, some with gaps, some long + for (int t = 0; t < 50; t++) { + std::vector runs; + uint64_t at = rnd() % 16; + for (int i = 0; i < 128; i++) { + const uint64_t count = rnd() % 9 == 0 ? 8000 + rnd() % 9000 : rnd() % 64; + runs.push_back({at, count, rnd() % 4 == 0 ? 0 : ((rnd() % 0xFFFFFF) + 1) << 12}); + at += count + (rnd() % 3 == 0 ? rnd() % 50 : 0); + } + both_ways(runs, at + 1); + // and the same runs with two of them swapped, or one repeated: the sweep + std::swap(runs[rnd() % 128], runs[rnd() % 128]); + runs.push_back(runs[rnd() % 128]); + runs.back().address = 0x7770000000ull; + both_ways(runs, at + 1); + } + // 256 single pages in any order, some twice + std::vector singles; + for (int i = 0; i < 256; i++) singles.push_back({rnd() % 300, 1, ((rnd() % 0xFFFF) + 1) << 12}); + both_ways(singles, 300); + both_ways({}, 1); + both_ways({{5, 0, 0x1000}}, 8); + both_ways({{0, 8192, 0x4000000}, {8192, 1, 0x9000000}}, 8200); // (a full transfer, then one entry) + std::printf("ok: the streamed flush writes the sorted flush's transfers, entry for entry\n"); + } else if (test == "four" || test == "four-sorted") { + // four overlapping ranges over 150,000 pages (a pool set, zeroed at its merge and set again in parts) + const std::vector runs = {{0, 150000, 0x100000000ull}, {20000, 70000, 0}, {50000, 70000, 0x900000000ull}, {100000, 50000, 0xA00000000ull}}; + table_pending = runs; + updates.clear(), largest = 0; + tracking = true; + if (test == "four") memory_table_flush(); // (as the process's environment says: streamed unless RX_VK_TABLE_STREAM=0) + else table_flush_sorted(); + tracking = false; + std::printf("largest %zu\n", largest); + const std::vector got = updates; + updates.clear(); + table_pending = runs; + table_flush_sorted(); + CHECK(got == updates && !got.empty()); + } else { + std::printf("a known case\n"); + return 1; + } + return fails != 0; +} +} // namespace fixture +int main(int argc, char** argv) { return fixture::run(argc, argv); } diff --git a/tests/parallel_init_order_test.cpp b/tests/parallel_init_order_test.cpp index 3286840..50a1491 100644 --- a/tests/parallel_init_order_test.cpp +++ b/tests/parallel_init_order_test.cpp @@ -221,8 +221,12 @@ std::chrono::steady_clock::time_point bc_used_at; uint64_t bc_last_value = 0; std::atomic bc_have{false}; constexpr uint32_t kBcLayers = 8; +uint32_t bc_scratch_layers() { return kBcLayers; } // ((11-21, TEX-10) the scratch's layers: a run's) VkImage bc_scratch = VK_NULL_HANDLE, bc_scratch2 = VK_NULL_HANDLE; bool bc_scratch_fresh = false, bc_scratch2_fresh = false; +VkImage bc_small = VK_NULL_HANDLE; // ((11-21, TEX-09) the small scratch: none here) +bool bc_small_fresh = false; +uint32_t bc_small_layers() { return 1; } bool bc_scratch_make() { bc_scratch = kScratch, bc_scratch_fresh = true, bc_have = true; return true; diff --git a/tests/parallel_table_order_test.cpp b/tests/parallel_table_order_test.cpp index b4ec11d..d0b91f5 100644 --- a/tests/parallel_table_order_test.cpp +++ b/tests/parallel_table_order_test.cpp @@ -3,6 +3,7 @@ // change the same pages in one order and the queue runs their uploads in the other. No game, no GPU. #include #include +#include #include #include #include @@ -13,8 +14,10 @@ #include #include #include +#include "table_stream.h" // ((Triforce, 11-22, SYNC-04) the flush's stream: gpu/vk, on the include path) namespace fixture { +void logf(const char*, ...) {} // (the flush's line for one over 1 ms) void require(bool ok, const char* why) { if (!ok) { std::fprintf(stderr, "FAIL: %s\n", why); diff --git a/tests/pipeline_cache_save_test.cpp b/tests/pipeline_cache_save_test.cpp index b31e705..b334854 100644 --- a/tests/pipeline_cache_save_test.cpp +++ b/tests/pipeline_cache_save_test.cpp @@ -177,6 +177,7 @@ std::uint64_t pcache_own_stamp() { return 0xabcdef0123456789ull; } std::atomic pcache_main_built{0}; std::atomic pcache_preparing{0}; bool pcache_play_save(int64_t) { return true; } +thread_local bool pcache_play_only_now = false; // ((Triforce, 11-22, PIPE-23) vk_draw.cpp: precompile_end_save's play-only save) int pcache_play_cache = 0; std::atomic pcache_play_saved{0}; void pcache_main_written(int64_t) {} diff --git a/tests/present_source_crop_test.cpp b/tests/present_source_crop_test.cpp index 9b347fb..4874e71 100644 --- a/tests/present_source_crop_test.cpp +++ b/tests/present_source_crop_test.cpp @@ -16,6 +16,7 @@ #define VK_USE_64_BIT_PTR_DEFINES 0 #include #include "vk_present_rect.h" +#include "framegen_refresh.h" // ((Triforce, 11-21, the upscaler audit's UPS-05) a fixed multiplier and the display's refresh #include "framegen_rows.h" // (Triforce: the Frame generation row's AMD FSR 4, which framegen_record asks first) using namespace mkwvk; using VmaAllocation = void*; @@ -28,7 +29,7 @@ constexpr bool vkCmdPushDescriptorSetKHR = true; constexpr uint32_t HDR_ENCODE_COMP_SPV[] = {0}; bool init_scale() { std::abort(); } bool make_pipe(Pipe&, const uint32_t*, size_t, std::initializer_list, int) { std::abort(); } -void logf(const char*) { std::abort(); } +void logf(const char*, ...) { std::abort(); } // ((Triforce, UPS-05) framegen_record's line takes arguments; never reached here) void device_wait_idle_locked() { std::abort(); } void drop_img(Img&) { std::abort(); } bool make_img(Img&, uint32_t, uint32_t, VkFormat, VkImageUsageFlags) { std::abort(); } @@ -64,8 +65,10 @@ void vkCmdClearColorImage(VkCommandBuffer, VkImage, VkImageLayout, const VkClear CHECK(c->float32[0] == 0 && c->float32[1] == 0 && c->float32[2] == 0 && c->float32[3] == 1); ++clears; } bool init() { ++initializations; return true; } +int multiplier(); bool ensure_size(VkCommandBuffer, uint32_t w, uint32_t h) { - ++sizes; allocated_w = w; allocated_h = h; return true; + // (as the real one: the sets' frames are made for the multiplier, which framegen_record holds its own to) + ++sizes; allocated_w = w; allocated_h = h; fg.mult = multiplier(); return true; } int64_t now_ns() { return 1000000; } int multiplier() { return 2; } diff --git a/tests/program_cache_cleanup_test.cpp b/tests/program_cache_cleanup_test.cpp index ae4a07a..b7ff6a5 100644 --- a/tests/program_cache_cleanup_test.cpp +++ b/tests/program_cache_cleanup_test.cpp @@ -108,6 +108,9 @@ const std::vector* renodx_replacement(uint32_t) { return nullptr; } #define PROF_EXACT() // (vk_prof.h's exact-timing scope, since #989: nothing to time here) // (local, texture detail: vk_shader.cpp's note of which bindings a module reads by exact texel. Nothing to note here) void exact_note(VkShaderModule, const std::vector&) {} +void lanes_note(VkShaderModule, const std::vector&) {} // (PIPE-52: make_module's note of a module's Subgroup barriers) +void size_note(VkShaderModule, const std::vector&) {} // (PIPE-56: its OpPhi and OpFunctionCall counts) +void size_move(VkShaderModule, VkShaderModule) {} // (Triforce, RX_VK_SPV_PACK: vk_shader.cpp's packing of a ready program's stages. Off here, as with RX_VK_SPV_PACK=0: // every stage keeps its module, which is what this test counts. tests/test_spv_store.py has the packing) void pack_note(VkShaderModule, const uint32_t*, size_t) {} diff --git a/tests/recording_admission_test.cpp b/tests/recording_admission_test.cpp index 0a24424..42f28ea 100644 --- a/tests/recording_admission_test.cpp +++ b/tests/recording_admission_test.cpp @@ -20,6 +20,9 @@ #include #include #include +#include "flat_map.h" // ((Triforce) RecordingRes's c1_at) +#include "desc_pool_grow.h" // ((Triforce, 11-22, DESC-33) a slot's pool at half, whole once a recording ran out) +namespace desc_pool_grow = mkwvk::desc_pool_grow; void require(bool ok, const char* why) { if (!ok) { std::fprintf(stderr, "FAIL: %s\n", why); std::exit(1); } @@ -44,6 +47,7 @@ using VmaAllocation = Allocation*; struct { VkDevice device{}; VkPhysicalDeviceProperties props{}; } dev; std::map buffers; std::set pools; +std::map pool_sets; // ((Triforce, 11-22, DESC-33) each pool's maxSets as it was made) std::map sets; uintptr_t serial = 100; template T handle() { return reinterpret_cast(++serial); } @@ -63,10 +67,10 @@ void destroy_buffer(VkBuffer buffer, VmaAllocation allocation) { require(buffer && buffers.count(buffer) && &buffers.at(buffer) == allocation, "buffer ownership/double free"); buffers.erase(buffer); ++destroys; } -VKAPI_ATTR VkResult VKAPI_CALL vkCreateDescriptorPool(VkDevice, const VkDescriptorPoolCreateInfo*, const VkAllocationCallbacks*, VkDescriptorPool* pool) { +VKAPI_ATTR VkResult VKAPI_CALL vkCreateDescriptorPool(VkDevice, const VkDescriptorPoolCreateInfo* ci, const VkAllocationCallbacks*, VkDescriptorPool* pool) { // Vulkan output handles must not be trusted after a failing creation result. if (fail_pool) { *pool = reinterpret_cast(uintptr_t(0xdead)); return VK_ERROR_OUT_OF_HOST_MEMORY; } - *pool = handle(); pools.insert(*pool); return VK_SUCCESS; + *pool = handle(); pools.insert(*pool); pool_sets[*pool] = ci->maxSets; return VK_SUCCESS; } void drop_sets(VkDescriptorPool pool) { for (auto it = sets.begin(); it != sets.end();) { @@ -149,6 +153,7 @@ int64_t descs_of(const VkWriteDescriptorSet* w, size_t n) { int64_t k = 0; for ( bool bind_tail() { VkCommandBuffer cmd{}; VkPipelineBindPoint bp = VK_PIPELINE_BIND_POINT_GRAPHICS; struct { VkDescriptorSetLayout set_layout{}; VkPipelineLayout layout{}; } p; + auto& dwr = dw(); // ((Triforce, 11-22, DESC-20) as bind_descriptors takes it at its top) /* DESCRIPTORS */ } void begin(int slot = 0, uint64_t value = 1) { draw_begin_recording(slot, VkCommandBuffer{}, value); } @@ -256,6 +261,37 @@ int main(int argc, char** argv) { require(cur->extra.empty() && pools.size() == pool_count, "failed descriptor pool ownership retained/leaked"); fail_pool = fail_pool_commit = false; fail_sets = 1; require(bind_tail() && cur->extra.size() == 1 && updates == 1 && binds == 1, "descriptor retry did not recover"); + } else if (mode == "pool-grow" || mode == "pool-grow-off") { + // ((Triforce, 11-22, the performance audit's DESC-33) a slot's pool at half, whole from the reset after a recording + // ran out of it; with RX_VK_DESC_POOL_GROW=0 (the test's driver sets it for pool-grow-off) whole from the start + const bool on = mode == "pool-grow"; + const auto first = res[0].pool; + writes.writes[0].descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, writes.writes[0].descriptorCount = 3; + require(pool_sets.at(first) == (on ? 4096u : 8192u) && res[0].shift == (on ? 1u : 0u), "a slot's first pool: half with the rule, whole without"); + require(bind_tail() && cur->extra.empty(), "a set from the slot's own pool"); + begin(0, 2); + require(res[0].pool == first && pools.count(first) && pools_whole == 0 && pools_added == 0, "a slot that did not run out keeps its pool"); + require(pool_most.sets == 1 && pool_most.n[desc_pool_grow::kSampled] == 3, "what the first recording took is not counted"); + fail_sets = 1; require(bind_tail() && cur->extra.size() == 1 && pool_sets.at(cur->extra[0]) == 8192u, "the extra pool is not a whole one"); + require(bind_tail() && cur->extra.size() == 1, "a second set from the extra pool"); + const auto extra = cur->extra[0]; + begin(0, 3); + require(cur->extra.empty() && !pools.count(extra) && pools_added == 1, "the extra pool is not destroyed and counted at the slot's reset"); + require(pool_most.sets == 2 && pool_most.n[desc_pool_grow::kSampled] == 6 && res[0].took.sets == 0, "the peaks do not follow the recordings"); + if (on) require(res[0].pool != first && !pools.count(first) && pool_sets.at(res[0].pool) == 8192u && res[0].shift == 0 && pools_whole == 1, + "a slot that ran out does not have a whole pool from its next reset"); + else require(res[0].pool == first && pools.count(first) && pools_whole == 0, "without the rule the slot's pool is replaced"); + const auto whole = res[0].pool; + fail_sets = 1; require(bind_tail() && cur->extra.size() == 1, "a whole slot that runs out gets an extra pool as before"); + begin(0, 4); + require(res[0].pool == whole && pools.count(whole) && pools_whole == (on ? 1u : 0u) && pools_added == 2, "a whole slot's pool is made again"); + // (the whole pool cannot be made: the half one is reset and kept, the slot never without a pool) + begin(1, 5); const auto other = res[1].pool; + fail_sets = 1; require(bind_tail() && cur->extra.size() == 1, "the second slot's extra pool"); + fail_pool = true; begin(1, 6); fail_pool = false; + require(res[1].pool == other && pools.count(other) && res[1].extra.empty() && res[1].shift == (on ? 1u : 0u) && + pools_whole == (on ? 1u : 0u), "a whole pool that could not be made cost the slot its pool"); + require(bind_tail(), "the kept pool does not serve"); } else if (mode == "healthy") { require(bind_tail() && cur->extra.empty() && updates == 1 && binds == 1, "healthy primary descriptor binding"); copied(17, 51); require(cur->stream == old && cur->used == 273, "alignment/prefix preservation"); diff --git a/tests/replay_threads_tuner_test.cpp b/tests/replay_threads_tuner_test.cpp index 420e67c..2a3fd96 100644 --- a/tests/replay_threads_tuner_test.cpp +++ b/tests/replay_threads_tuner_test.cpp @@ -14,6 +14,11 @@ // better one as often as without it, and as soon after a change; a short trial never changes the way, one that finds the // other way shorter is followed at once by a full one; a change the scene's means cannot see is found by a backed-off // trial; the rule's no-trial verdicts hold; a load is no scene change. +// "settled": (Triforce, 11-22) Tuner::settled, after Kaefergeneral's #dev-fixes post (a 5900X and an RTX 3090, where the +// helpers lose in every settled scene): an area's load window (long frames, the replay most of them, nearly every draw +// cuttable) gets no verdict, nor the first window after it; a settled heavy scene where the helpers win still turns them on +// without a trial, a window after it settled; a trial the scene changed under changes nothing and the next comes soon +// after the scene settles. With settled off, each as before. #include #include #include @@ -193,11 +198,12 @@ struct Run { long long first_on = -1, first_trial = -1; }; /// frames presents of a scene through the tuner, the way in use applied as par_tune_present applies it -/// (backoff: the tuner's back-off; off for the checks of the trials' schedule as it was) -Run run_scene(const Scene& sc, int frames, bool use_rule, uint32_t seed, Tuner* keep = nullptr, bool backoff = false) { +/// (backoff: the tuner's back-off; off for the checks of the trials' schedule as it was. settled: Tuner::settled) +Run run_scene(const Scene& sc, int frames, bool use_rule, uint32_t seed, Tuner* keep = nullptr, bool backoff = false, + bool settled = true) { Tuner local; Tuner& t = keep ? *keep : local; - t.use_rule = use_rule, t.threads = 4, t.backoff = backoff; + t.use_rule = use_rule, t.threads = 4, t.backoff = backoff, t.settled = settled; std::mt19937 r(seed); bool on = false; Run out; @@ -222,7 +228,7 @@ void same_case() { for (int run = 0; run < 60; run++) { OldTuner old; Tuner t; - t.use_rule = false, t.backoff = false; + t.use_rule = false, t.backoff = false, t.settled = false; bool on = false, order_only = false; const double base = 6.0 + (r() % 900) / 100.0, par = 0.85 + (r() % 30) / 100.0; // (the helpers 15% faster to 15% slower) for (int i = 0; i < 12000; i++) { @@ -277,11 +283,18 @@ void rule_case() { CHECK(v.gain == v.saved && v.gain < 1.0 && v.kind != Verdict::kOn); } // the tuner on each scene, 20,000 presents + // (Triforce, 11-22: settled. On at the second window, the first with one before it to tell the scene settled; with + // settled off at the first, as before) const Run heavy = run_scene(test4_heavy, 20000, true, 1); - CHECK(heavy.first_on == Tuner::kWindow); // (on at the first window, without a trial) + CHECK(heavy.first_on == 2 * Tuner::kWindow); // (on at the second window, without a trial) CHECK(heavy.first_trial > heavy.first_on); // (a trial came after, kEvery later) CHECK(heavy.changes == 0 && heavy.trials >= 6); // (and kept them, 12% shorter frames: no trial took them off) - CHECK(heavy.on_frames > 20000 - 120 - (heavy.trials + 1) * (Tuner::kSettle + Tuner::kTake)); // (but in the trials' serial parts) + CHECK(heavy.on_frames > 20000 - 240 - (heavy.trials + 1) * (Tuner::kSettle + Tuner::kTake)); // (but in the trials' serial parts) + { + const Run was = run_scene(test4_heavy, 20000, true, 1, nullptr, false, false); + CHECK(was.first_on == Tuner::kWindow && was.first_trial > was.first_on && was.changes == 0 && was.trials >= 6); + CHECK(was.on_frames > 20000 - 120 - (was.trials + 1) * (Tuner::kSettle + Tuner::kTake)); + } const Run gpu = run_scene(gpu_bound, 20000, true, 2); CHECK(gpu.trials == 0 && gpu.flips == 0 && gpu.rule_off > 100); // (no trial at all: it cannot pay) const Run game = run_scene(test7_trial4, 20000, true, 3); @@ -299,7 +312,7 @@ void rule_case() { // cannot pay, nothing at the title), the trials fewer where they decide, and none changed the way where it held { const Run h = run_scene(test4_heavy, 20000, true, 1, nullptr, true); - CHECK(h.first_on == Tuner::kWindow && h.first_trial > h.first_on && h.changes == 0); + CHECK(h.first_on == 2 * Tuner::kWindow && h.first_trial > h.first_on && h.changes == 0); CHECK(h.trials >= 2 && h.trials < heavy.trials); CHECK(h.on_frames > heavy.on_frames); // (fewer frames the other way) const Run g = run_scene(gpu_bound, 20000, true, 2, nullptr, true); @@ -394,7 +407,7 @@ void prev_case() { PrevTuner prev; Tuner t; const bool use_rule = run % 2 == 0; - prev.use_rule = use_rule, t.use_rule = use_rule, t.backoff = false; + prev.use_rule = use_rule, t.use_rule = use_rule, t.backoff = false, t.settled = false; prev.threads = t.threads = 2 + run % 3; bool on_p = false, on_t = false, order_only = false; Scene sc = scenes[r() % 6]; @@ -624,12 +637,119 @@ void backoff_case() { } } +// ---- (Triforce, 11-22) settled ---- + +// (from the post: the window of the teleport to the Market, "a frame of 14.84 ms ... the replay thread 11.37, 77% of it ... +// 1394 of its 1464 draws and dispatches in command buffers to cut"; the settled Market with the replay 62% of the frame and the +// helpers ~10% slower, 71.4 fps without them; Kokiri 85.8 fps, the helpers ~6% slower) +const Scene arrival{14.84, 11.37, 6.0, 1464, 1394, 6.0, 1.0}; +const Scene market_settled{14.0, 8.7, 7.0, 3250, 3150, 6.5, 1.10}; +const Scene kokiri{11.66, 7.4, 5.8, 2000, 1900, 4.0, 1.06}; + +struct Arrival { + int on_verdicts = 0, trials = 0, changes = 0; + long long first_on = -1, on_frames = 0; + std::vector unsettled; // (the presents of the windows the rule did not judge, before the first trial) +}; +/// Kokiri for two windows, the load's window, then `to` (each window the rule's: the first trial comes at present 600) +Arrival run_arrival(const Scene& to, bool settled, uint32_t seed) { + Tuner t; + t.use_rule = true, t.backoff = true, t.settled = settled, t.threads = 4; + std::mt19937 r(seed); + bool on = false; + Arrival out; + for (int i = 0; i < 6000; i++) { + const Scene& sc = i < 2 * Tuner::kWindow ? kokiri : i < 3 * Tuner::kWindow ? arrival : to; + const Tuner::Step s = t.step(sample(sc, on, r), on, false); + if (s.ruled && s.verdict.kind == Verdict::kOn) out.on_verdicts++; + if (s.ruled && s.flip && out.first_on < 0) out.first_on = t.frames(); + if (s.unsettled && !s.trial && !out.trials) out.unsettled.push_back(t.frames()); + if (s.trial) out.trials++, out.changes += s.better; + if (s.flip) on = !on; + out.on_frames += on; + } + return out; +} + +struct Straddle { + bool flipped = false, unsettled = false, second_ok = false, second_changed = true, on_after = true; + long long end1 = -1, end2 = -1, settle_at = -1; +}; +/// A trial the scene changes under: its first part in test7's busy scene, its other way in a fade (5 ms frames either way, +/// as short as a loading screen's), its last part in a lighter area where the helpers lose by 8% (the trials decide there). +/// The trial's parts are told apart by its own changes of way +Straddle run_straddle(bool settled, uint32_t seed) { + const Scene fade{5.0, 0.6, 0.9, 30, 0, 0.1, 1.0}; + const Scene lose{12.0, 9.6, 8.4, 1500, 1400, 3.4, 1.08}; + Tuner t; + t.use_rule = true, t.backoff = true, t.settled = settled, t.threads = 4; + std::mt19937 r(seed); + bool on = false; + int part = 0; // (0: before the trial's first change of way, 1: its other way, 2: after) + Straddle out; + for (int i = 0; i < 6000; i++) { + const Scene& sc = part == 0 ? test7_busy : part == 1 ? fade : lose; + const Tuner::Step s = t.step(sample(sc, on, r), on, false); + if (s.settle && out.settle_at < 0) out.settle_at = t.frames(); + if (s.trial && out.end1 < 0) { + out.end1 = t.frames(), out.flipped = s.flip, out.unsettled = s.unsettled; + } else if (s.trial && out.end2 < 0) { + out.end2 = t.frames(), out.second_ok = s.ok && !s.unsettled, out.second_changed = s.better; + } + if (s.flip && !s.trial && !s.ruled && part < 2) part++; + if (s.flip) on = !on; + if (out.end2 > 0 && t.frames() == out.end2) out.on_after = on; + } + return out; +} + +void settled_case() { + CHECK(rule(mean_of(arrival), 4).kind == Verdict::kOn); // (as the post's log has it, judged alone) + CHECK(rule(mean_of(market_settled), 4).kind == Verdict::kTrial && rule(mean_of(kokiri), 4).kind == Verdict::kTrial); + // the load's window: no verdict, nor the first window after it; the settled Market's are the trials' + { + const Arrival n = run_arrival(market_settled, true, 61), o = run_arrival(market_settled, false, 61); + CHECK(n.on_verdicts == 0 && n.first_on < 0); + // (the first window, none before it; the load's; the Market's first) + CHECK((n.unsettled == std::vector{Tuner::kWindow, 3 * Tuner::kWindow, 4 * Tuner::kWindow})); + CHECK(n.trials >= 2 && n.changes == 0); // (the trials keep the replay thread alone) + CHECK(o.on_verdicts >= 1 && o.first_on == 3 * Tuner::kWindow && o.unsettled.empty()); // (before: on at the load's window) + std::printf("settled: the load's window into the Market: %d \"on\" verdicts (%d with settled off, on at present %lld), %d " + "windows not judged before the first trial; the helper threads in %lld of 6,000 frames (%lld)\n", + n.on_verdicts, o.on_verdicts, o.first_on, (int)n.unsettled.size(), n.on_frames, o.on_frames); + } + // a settled heavy scene where the helpers win (test4's): on without a trial, a window after it settled, and kept + { + const Arrival n = run_arrival(test4_heavy, true, 62), o = run_arrival(test4_heavy, false, 62); + CHECK(n.first_on == 5 * Tuner::kWindow); // (the heavy scene's second window: the first with one like it before) + CHECK(n.changes == 0 && n.on_frames > 6000 - 600 - (n.trials + 1) * (Tuner::kSettle + Tuner::kTake)); + CHECK(o.first_on == 3 * Tuner::kWindow); + std::printf("settled: the load's window into test4's heavy scene: the helper threads on at present %lld without a trial " + "(%lld with settled off), kept by %d trials\n", n.first_on, o.first_on, n.trials); + } + // a trial the scene changed under: nothing changes, and the next comes soon after the scene settles + { + const Straddle n = run_straddle(true, 63), o = run_straddle(false, 63); + CHECK(n.end1 > 0 && n.unsettled && !n.flipped); + CHECK(n.settle_at > n.end1 && n.settle_at <= n.end1 + 2 * Tuner::kWindow); + // (the next trial kWindow after the settled window, and its 80 + 24 + 80 + 24 + 80 frames) + CHECK(n.end2 == n.settle_at + Tuner::kWindow + Tuner::kTake + 2 * (Tuner::kSettle + Tuner::kTake)); + CHECK(n.second_ok && !n.second_changed && !n.on_after); // (the settled area's trial keeps the replay thread alone) + CHECK(o.flipped && !o.unsettled); // (before: the fade's short frames passed for the helpers' gain) + CHECK(o.end2 - o.end1 >= Tuner::kEvery); + std::printf("settled: a trial the scene changed under: %s (%s with settled off); the scene settled %lld presents after " + "it, the next trial ended %lld after it (%lld)\n", n.flipped ? "a change" : "no change", + o.flipped ? "the helper threads on" : "no change", n.settle_at - n.end1, n.end2 - n.end1, o.end2 - o.end1); + } +} + int main(int argc, char** argv) { const std::string c = argc > 1 ? argv[1] : ""; if (c == "same") same_case(); else if (c == "prev") prev_case(); else if (c == "rule") rule_case(); else if (c == "backoff") backoff_case(); + else if (c == "settled") settled_case(); else return std::printf("which case?\n"), 2; return failures ? 1 : 0; } diff --git a/tests/sampler_admission_test.cpp b/tests/sampler_admission_test.cpp index 4b2cf95..8b36fbe 100644 --- a/tests/sampler_admission_test.cpp +++ b/tests/sampler_admission_test.cpp @@ -159,6 +159,7 @@ void vkCmdPushDescriptorSetKHR(VkCommandBuffer, VkPipelineBindPoint, VkPipelineL check_writes(n, writes); descriptor_pushes++; } VkDescriptorPool make_pool() { return reinterpret_cast(uintptr_t(11)); } +void pool_took(const VkWriteDescriptorSet*, size_t) {} // ((Triforce, 11-22, DESC-33) vk_draw.cpp: the sets a recording took, counted) void vkDestroyDescriptorPool(VkDevice, VkDescriptorPool pool, const VkAllocationCallbacks*) { assert(pool); } VkResult vkAllocateDescriptorSets(VkDevice, const VkDescriptorSetAllocateInfo*, VkDescriptorSet* out) { descriptor_allocs++; *out = reinterpret_cast(uintptr_t(12)); return VK_SUCCESS; diff --git a/tests/test_achievements.py b/tests/test_achievements.py new file mode 100644 index 0000000..859fd84 --- /dev/null +++ b/tests/test_achievements.py @@ -0,0 +1,213 @@ +"""Achievements (host/achievements.h, host/achievements.c, host/hos_main.c, gpu/vk/achievement_rows.h, gpu/vk/vk_menu.cpp; +authored lists, no game). + +Things done in the game that are marked as earned, once, with a notice on the screen and a sound, and a page of the +options that lists them all with their icons: on unless HOS_ACHIEVEMENTS=0. Compiled and run here: + +- tests/achievements_test.c: host/achievements.h alone. The built-in list is read whole; the form of a list and each + thing that leaves an achievement out (no condition, a line that is none of the form's, a name that is empty, too long + or another's, a count that is none, an icon that is none, too many conditions, more than there is room for); when one + is earned (every condition, an item's count, a flag set and in the save data), that it stays earned and is not asked + about again; and the file of the earned ones, written and read back, with the lines of achievements no list has now + kept as they were; +- tests/achievements_host_test.c: host/achievements.c's own text against a stand-in save data, menu and game folder. + Without the switch nothing is asked, read, written, said or named; with it a player's own achievements.txt is read + after the built-in list, the list is named to the menu once, whole, when the renderer answers, six earned at once are + written at once and shown one at a time with the sound and never before the menu has the list, and at the next start + (a second run on the same folder) they are neither said nor shown again; +- tests/achievement_rows_test.cpp: gpu/vk/achievement_rows.h alone. Nothing shown until the list is whole, then as + named; one earned, its day and its notice once; names and lines cut to their room; the page's choice and pages under + the keys, the wheel and a click; how a notice comes in and goes. + +Read in the sources: the built-in list asks only for items gpu/vk/oot_cheat_items.h already names, and for no flag, so +that no other name of the game's comes into the repository; the save data is asked at Link's update, once a second, +not before the switch is looked at, only while a save is loaded, and through nothing that writes; the host and the menu +have the same icons, the same room and the same option ids; the menu has the row, the page and the notice. + + python -I -B tests/test_achievements.py [--cc gcc] [--cxx g++] +""" +import argparse +import os +from pathlib import Path +import re +import subprocess +import tempfile + +RUN_ALL_ARGS = ["--cc", "gcc", "--cxx", "g++"] + +ROOT = Path(__file__).resolve().parents[1] + +OWN_LIST = "[Mine]\nabout: A flag of the player's own choosing.\nicon: gem blue\nflag My_Flag\n[Not one]\n[Nor this]\nitem\n" + + +def check(cond, what): + if not cond: + raise SystemExit("FAIL: " + what) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def run(command, says=None, **more): + """A compiler or a fixture run; a fixture must say `says`""" + flags = (subprocess.CREATE_NO_WINDOW | subprocess.BELOW_NORMAL_PRIORITY_CLASS) if os.name == "nt" else 0 + result = subprocess.run(command, creationflags=flags, capture_output=True, text=True, encoding="utf-8", errors="replace", **more) + check(result.returncode == 0 and (says is None or says in result.stdout), + "%s:\n%s%s" % (" ".join(str(c) for c in command[:1] + command[-2:]), result.stdout, result.stderr)) + return result.stdout + + +def check_sources(): + read = lambda *parts: ROOT.joinpath(*parts).read_text(encoding="utf-8") + host, own, header, api = read("host", "hos_main.c"), read("host", "achievements.c"), read("host", "achievements.h"), read("gpu", "vk", "vk_api.h") + native, rows, menu, replay = read("host", "nvn_native.c"), read("gpu", "vk", "achievement_rows.h"), read("gpu", "vk", "vk_menu.cpp"), read("gpu", "vk", "vk_replay.cpp") + # at Link's update, after the other tools' reads; the switch before anything is asked of the game; only with a save loaded + update = block(host, "static void oot_update_player_cheats(") + check("oot_achievements_tick(c, now);" in update and update.index("oot_link_tick(c, actor, now);") < update.index("oot_achievements_tick(c, now);"), + "host/hos_main.c does not ask about the achievements at Link's update") + check(host.count("oot_achievements_tick(") == 2 and host.count("hos_achievements_check(") == 1, "the achievements are asked about from somewhere else too") + tick = block(host, "static void oot_achievements_tick(") + check("if (!hos_achievements_on() || now < next) return;" in tick and "next = now + 1000000000LL;" in tick + and tick.index("hos_achievements_on()") < tick.index("oot_pouch_count(") + and "if (oot_pouch_count(c, oot_item_names[0]) < 0) return;" in tick + and tick.index("oot_pouch_count(") < tick.index("hos_achievements_check("), + "the game is asked before the switch is, more than once a second, or while no save is loaded") + # reading only: the probe and host/achievements.c write nothing of the game's + probe = block(host, "static int oot_achievement_probe(") + for word in ("recomp_store", "oot_gd_set", "oot_item_give", "oot_flag_set", "A64_PTR"): + check(word not in probe and word not in tick, "asking about an achievement writes into the game: " + word) + for word in ("recomp_", "A64_PTR", "GP(", "a64_lookup"): + check(word not in own, "host/achievements.c reaches into the game: " + word) + check("on = !(e && e[0] == '0' && !e[1]);" in block(own, "int hos_achievements_on("), "the achievements are not on unless HOS_ACHIEVEMENTS=0") + for name in ("void hos_achievements_frame(", "void hos_achievements_check("): # (the switch before the lock, the lists and the menu) + body = block(own, name) + check("hos_achievements_on()" in body and body.index("hos_achievements_on()") < body.index("lock();") < body.index("lists_read();") + and body.count("unlock();") == 1 and "return;" not in body[body.index("lock();"):], + "%s the lists are read before the switch is looked at, or the lock is not let go" % name) + # the list named at the game's present, after the mods' cheats, from one place + present = block(native, "NV(nvnQueuePresentTexture) {") + check(present.index("hos_mod_cheats_frame();") < present.index("hos_achievements_frame();") and native.count("hos_achievements_frame();") == 1, + "the achievements are not named to the menu at the game's present") + # the host and the menu: the same ids, the same room, the same icons in the same order + enum = block(api, "enum { HOS_OPT_ACH_TEXT") + names = re.findall(r"\bHOS_OPT_\w+", enum) + check("HOS_OPT_ACH_TEXT = 165" in enum and names == ["HOS_OPT_ACH_TEXT", "HOS_OPT_ACH_ADD", "HOS_OPT_ACH_COUNT", "HOS_OPT_ACH_EARNED", "HOS_OPT_ACH_ASK"], + "the option ids are not 165 to 169") + others = re.sub(r"enum \{ HOS_OPT_ACH_TEXT[^}]*\}", "", api) + for value in range(165, 170): + check(not re.search(r"= %d\b" % value, others), "another option has the id %d" % value) + check("enum { HOS_ACH_EARNED = 256 };" in api and "constexpr int kEarned = 256;" in rows, "the earned bit is not the same on both sides") + number = lambda text, name: int(re.search(r"\b%s = (\d+)\b" % name, text).group(1)) + check(number(header, "ACH_MAX") == number(rows, "kSlots") and number(header, "ACH_NAME") + 1 == number(rows, "kName") + and number(header, "ACH_ABOUT") + 1 == number(rows, "kAbout"), "the host's list and the menu's have not the same room") + words = lambda text, name: re.findall(r'"(\w+)"', re.search(r"#define %s (.*)" % name, text).group(1)) + shapes = re.search(r"enum Shape \{ ([^}]*) \}", rows).group(1).split(", ") + tones = re.search(r"enum Tone \{ ([^}]*) \}", rows).group(1).split(", ") + check(["k" + w.capitalize() for w in words(header, "ACH_SHAPES")] + ["kShapes"] == shapes, "the icons' shapes differ: %s" % shapes) + check(["k" + w.capitalize() for w in words(header, "ACH_TONES")] + ["kTones"] == tones, "the icons' tones differ: %s" % tones) + check(len(shapes) - 1 <= 16 and len(tones) - 1 <= 16, "an icon no longer fits its eight bits") + icon = block(menu, "void ach_icon(") + for shape in shapes[1:-1]: # (kStar is the default) + check("case ar::%s" % shape in icon, "the menu does not draw the shape %s" % shape) + check(len(re.findall(r"ui::rgba\(", icon.split("};")[0])) == len(tones) - 1, "the menu has not a colour for every tone") + # the menu: told by the host, the row shown once it was, its page, the notice + check("if (id == HOS_OPT_ACH_ASK) return 1;" in replay, "the renderer does not say that it shows the achievements") + setter = block(menu, "void menu_option_set(") + for line in ("if (id == HOS_OPT_ACH_TEXT) return ach_list.text(value);", "if (id == HOS_OPT_ACH_ADD) return ach_list.add(value);", + "ach_list.done(value);", "if (ach_list.earn(value)) menu_refresh = true;"): + check(line in setter, "the menu does not take the host's achievements: " + line) + table = menu[menu.index("Row rows[] = {"):] + check(table.index('"Bug capture (F8)"') < table.index('{TAB_SYSTEM, "Achievements", nullptr, 0, 0, 0, nullptr,') < table.index('MENU_HEADING(TAB_SYSTEM, "Mod cheats")') + and 'constexpr char kAchRow[] = "Achievements";' in menu, "the Achievements row is not on the System tab after Bug capture (F8)") + check("(strcmp(rows[i].name, kAchRow) || rows[i].tab != TAB_SYSTEM || ach_list.shown() > 0) &&" in block(menu, "bool row_shown("), + "the Achievements row shows before the host named them") + press = menu[menu.index("else if (press[7] && is_ach_row(r)) page = kAchPage"):] + check(press.index("is_ach_row(r)") < press.index("else if (press[7] && !r.value && can_bind()) page = 1"), + "the Achievements row opens the Controls page: it is asked after the rows without a value") + check("else if (page == kAchPage) draw_achievements(k);" in block(menu, "void draw_menu(ui::Ctx& c, bool perf_under) {"), "the Achievements page is not drawn") + page = number(menu, "kAchPage") + check(page not in (0, 1, 3, 4, 5, 6, number(menu, "kModsPage"), number(menu, "kModsRowPage")), "kAchPage is another page's number") + for record in ("void menu_record(", "void menu_record_image("): + body = block(menu, record) + check("ach_notice >= 0 && prof_now() - ach_notice_at < kAchNoticeMs * 1000000" in body and "&& !earned)" in body + and "if (earned) draw_ach_notice(c);" in body, "%s an achievement's notice is not drawn, or not while nothing else shows" % record) + check(number(own, "SHOWN_MS") > int(re.search(r"kAchNoticeMs = (\d+);", menu).group(1)), "the host shows the next notice before this one is gone") + return api, own + + +def check_names(exe): + """The built-in list asks only for items the Inventory page already names, a count only of one that has a count, and no flag""" + items = dict(re.findall(r'OOT_ITEM\("[^"]+", "([^"]+)", ([01])\)', (ROOT / "gpu" / "vk" / "oot_cheat_items.h").read_text(encoding="utf-8"))) + check(len(items) > 60, "gpu/vk/oot_cheat_items.h: the items were not read") + asked = [line.split() for line in run([str(exe), "names"]).splitlines()] + check(len(asked) >= 25, "the built-in list's conditions were not printed") + for kind, what, least in asked: + check(kind == "I", "the built-in list asks for a flag: %s" % what) + check(what in items, "the built-in list names what gpu/vk/oot_cheat_items.h does not: %s" % what) + check(least == "1" or items[what] == "1", "the built-in list counts an item that has no count: %s" % what) + header = (ROOT / "host" / "achievements.h").read_text(encoding="utf-8") + quoted = set(re.findall(r"\b((?:Item|Weapon|Armor)_[A-Za-z0-9_]+)", header)) + check(quoted and quoted <= set(items), "host/achievements.h names what gpu/vk/oot_cheat_items.h does not: %s" % sorted(quoted - set(items))[:3]) + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--cc", default="cc") + p.add_argument("--cxx", default="c++") + a = p.parse_args() + api, own = check_sources() + with tempfile.TemporaryDirectory(prefix="triforce-achievements-") as tmp: + tmp = Path(tmp) + warnings = ["-std=c11", "-Wall", "-Wextra", "-Werror"] + # the list, the conditions and the file + exe = tmp / "list.exe" + run([a.cc] + warnings + ["-I" + str(ROOT / "host"), str(ROOT / "tests" / "achievements_test.c"), "-o", str(exe)]) + run([str(exe)], says="PASS") + check_names(exe) + # the menu's side + exe = tmp / "rows.exe" + run([a.cxx, "-std=c++20", "-Wall", "-Wextra", "-Werror", "-I" + str(ROOT / "gpu" / "vk"), str(ROOT / "tests" / "achievement_rows_test.cpp"), + "-o", str(exe)]) + run([str(exe)], says="PASS") + # the host's side: host/achievements.c from after its includes on, with the menu's ids from vk_api.h + marker = '#include "menu_sound.h"\n' + check(own.count(marker) == 1, "host/achievements.c no longer includes menu_sound.h last") + (tmp / "achievements_actual.inc").write_text(own[own.index(marker) + len(marker):], encoding="utf-8") + (tmp / "achievements_constants.inc").write_text( + "".join(block(api, start) + ";\n" for start in ("enum { HOS_OPT_ACH_TEXT = 165", "enum { HOS_ACH_EARNED = 256", "enum { HOS_MENU_SOUND_OPEN = 1")), + encoding="utf-8") + exe = tmp / "host.exe" + run([a.cc] + warnings + ["-Wno-unused-function", "-I" + str(ROOT / "host"), "-I" + str(tmp), str(ROOT / "tests" / "achievements_host_test.c"), + "-o", str(exe)]) + game = tmp / "game folder" + game.mkdir() + (game / "achievements.txt").write_bytes(OWN_LIST.encode("utf-8")) + earned = game / "triforce_achievements.txt" + env = {k: v for k, v in os.environ.items() if k != "HOS_ACHIEVEMENTS"} + run([str(exe), str(game), "off"], says="PASS", cwd=str(tmp), env=dict(env, HOS_ACHIEVEMENTS="0")) # (on unless it is 0) + check(not earned.exists(), "a file was written with the achievements off") + on = dict(env, HOS_ACHIEVEMENTS="1") + run([str(exe), str(game), "first"], says="PASS", cwd=str(tmp), env=env) # (on by default: the switch not set) + kept = earned.read_text(encoding="utf-8").splitlines() + check(kept[0].startswith("#") and [line.split(" ", 2)[2] for line in kept[1:]] == + ["A Sword of Your Own", "Forest's Thanks", "Heart of the Mountain", "Jewel of the River", "Three Stones", "Mine"], + "what the game folder keeps of the earned ones: %r" % kept) + check(all(re.fullmatch(r"[0-9a-f]{8} \d{9,} .+", line) for line in kept[1:]), "an earned one is not its key, its time and its name: %r" % kept) + with open(earned, "ab") as f: # (an achievement of a list that was taken away since) + f.write(b"deadbeef 5 Gone\n") + run([str(exe), str(game), "second"], says="PASS", cwd=str(tmp), env=on) + check(sorted(f.name for f in game.iterdir()) == ["achievements.txt", "triforce_achievements.txt"], "the game folder has other files now") + check((game / "achievements.txt").read_bytes() == OWN_LIST.encode("utf-8"), "the player's own list was changed") + print("PASS: achievements: the list and what it refuses, earning, the file of the earned ones, the host's naming and notices, " + "the menu's list and page, and on unless HOS_ACHIEVEMENTS=0") + + +if __name__ == "__main__": + main() diff --git a/tests/test_alt_menu.py b/tests/test_alt_menu.py new file mode 100644 index 0000000..fc02eba --- /dev/null +++ b/tests/test_alt_menu.py @@ -0,0 +1,104 @@ +"""A lone Alt does not freeze the start-up screen (host/display.c keymenu_dropped, wndproc; no game, no window). + +After Casual Gamer's #dev-fixes post. Pressed and let go alone, Alt has DefWindowProc enter the menu loop (WM_SYSCOMMAND with +SC_KEYMENU) on the window's thread, which during the start-up screen is the one waiting for the shaders: the screen froze +until a key was pressed. The window has no menu bar, so the command is answered 0 there. + +Cuts keymenu_dropped out of display.c and runs it with stand-ins for the Windows types: a lone Alt's command (lp 0) and one +for a letter with Alt (lp the character) are dropped, Alt+Space's (the system menu) is not, nor are the other system commands +(SC_CLOSE: Alt+F4; SC_SCREENSAVE; SC_MAXIMIZE), whatever their low bits; HOS_ALT_MENU=1 drops none. Then the source: the +window procedure asks it first in WM_SYSCOMMAND and returns 0, and Alt+Enter and F10 are still WM_SYSKEYDOWN's; the switch is +in docs/development.md. + + python -I -B tests/test_alt_menu.py [--cxx g++] +""" +import argparse +import os +import subprocess +import tempfile +from pathlib import Path + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] + + +def block(text, marker): + if text.count(marker) != 1: + raise SystemExit("FAIL: not found once in display.c: " + marker) + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +FIXTURE = r""" +#include +#include +#include +typedef uintptr_t WPARAM; +typedef intptr_t LPARAM; +#define SC_KEYMENU 0xF100 +#define SC_CLOSE 0xF060 +#define SC_SCREENSAVE 0xF140 +#define SC_MAXIMIZE 0xF030 +@@DROPPED@@ +#define CHECK(x) do { if (!(x)) { std::printf("line %d: %s\n", __LINE__, #x); return 1; } } while (0) +int main(int argc, char**) { + if (argc > 1) { // (HOS_ALT_MENU=1: as before) + CHECK(!keymenu_dropped(SC_KEYMENU, 0) && !keymenu_dropped(SC_KEYMENU, 'f')); + std::printf("alt menu: HOS_ALT_MENU=1 leaves the menu loop to Windows, as before\n"); + return 0; + } + CHECK(keymenu_dropped(SC_KEYMENU, 0)); // (a lone Alt, or F10 let through) + CHECK(keymenu_dropped(SC_KEYMENU | 3, 0)); // (the low four bits are Windows' own) + CHECK(keymenu_dropped(SC_KEYMENU, 'f')); // (Alt and a letter: a menu's mnemonic, and there is no menu) + CHECK(!keymenu_dropped(SC_KEYMENU, ' ')); // (Alt+Space: the system menu) + CHECK(!keymenu_dropped(SC_CLOSE, 0) && !keymenu_dropped(SC_SCREENSAVE, 0) && !keymenu_dropped(SC_MAXIMIZE, 0)); + std::printf("alt menu: a lone Alt enters no menu loop\n"); + return 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="g++") + args = parser.parse_args() + display = (ROOT / "host/display.c").read_text(encoding="utf-8") + fn = block(display, "static int keymenu_dropped(WPARAM wp, LPARAM lp) {") + proc = block(display, "static LRESULT CALLBACK wndproc(HWND w, UINT msg, WPARAM wp, LPARAM lp) {") + sys_cmd = proc[proc.index("case WM_SYSCOMMAND:"):] + sys_cmd = sys_cmd[:sys_cmd.index("break;")] + if "if (keymenu_dropped(wp, lp)) return 0;" not in sys_cmd or \ + sys_cmd.index("keymenu_dropped") > sys_cmd.index("SC_SCREENSAVE"): + raise SystemExit("FAIL: the window procedure does not answer a lone Alt's menu command itself") + sys_key = proc[proc.index("case WM_SYSKEYDOWN:"):] + if "if ((wp == VK_RETURN && (lp & (1 << 29))) || wp == VK_F10) {" not in sys_key[:sys_key.index("break;")]: + raise SystemExit("FAIL: Alt+Enter and F10 are no longer WM_SYSKEYDOWN's") + if "`HOS_ALT_MENU=1`" not in (ROOT / "docs/development.md").read_text(encoding="utf-8"): + raise SystemExit("FAIL: HOS_ALT_MENU is not in docs/development.md's switches") + flags = subprocess.CREATE_NO_WINDOW | subprocess.BELOW_NORMAL_PRIORITY_CLASS if os.name == "nt" else 0 + with tempfile.TemporaryDirectory(prefix="alt-menu-") as tmp: + src, exe = Path(tmp) / "alt.cpp", Path(tmp) / "alt.exe" + src.write_text(FIXTURE.replace("@@DROPPED@@", fn), encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", str(src), "-o", str(exe)], check=True) + env = dict(os.environ) + env.pop("HOS_ALT_MENU", None) + ran = 0 + for argv, value in (([str(exe)], None), ([str(exe), "before"], "1")): + if value is not None: + env["HOS_ALT_MENU"] = value + r = subprocess.run(argv, env=env, capture_output=True, text=True, timeout=120, creationflags=flags) + print(r.stdout, end="") + ran = ran or r.returncode + if ran: + raise SystemExit("FAIL: a lone Alt's menu command") + print("PASS: a lone Alt enters no menu loop; Alt+Space, Alt+F4, Alt+Enter and F10 as they were") + + +if __name__ == "__main__": + main() diff --git a/tests/test_astc_bc_policy.cpp b/tests/test_astc_bc_policy.cpp index 75dd2a1..1e99fc9 100644 --- a/tests/test_astc_bc_policy.cpp +++ b/tests/test_astc_bc_policy.cpp @@ -1,4 +1,4 @@ -// astc_bc_policy::all_by_vram (gpu/vk/astc_bc_policy.h): the cards that keep every large ASTC texture as BC3. +// astc_bc_policy::all_by_vram and all (gpu/vk/astc_bc_policy.h): the cards that keep every large ASTC texture as BC3. // tests/test_astc_bc_policy.py #include "../gpu/vk/astc_bc_policy.h" @@ -24,6 +24,19 @@ int main() { expect(!all_by_vram(9400 * MB), "a 10 GB card"); expect(!all_by_vram(11200 * MB), "a 12 GB card"); expect(!all_by_vram(15000 * MB), "a 16 GB card"); + // (Triforce, 11-22, the performance audit's TEX-12, after Source A's) every integrated GPU, whatever it is allowed + using astc_bc_policy::all; + expect(all(10232 * MB, true), "an integrated GPU allowed 10.2 GB (a 16 GB handheld)"); + expect(all(0, true), "an integrated GPU whose allowance is not known"); + expect(all(7402 * MB, true), "an integrated GPU allowed 7.4 GB"); + expect(all(21000 * MB, true), "an integrated GPU allowed 21 GB (a PC with 32 GB)"); + expect(!all(15000 * MB, false), "a discrete 16 GB card: as before"); + expect(!all(0, false), "a discrete card whose allowance is not known: as before"); + expect(!all(9400 * MB, false) && !all(11200 * MB, false) && !all(31000 * MB, false), "discrete 10, 12 and 32 GB cards: as before"); + expect(all(3379 * MB, false) && all(7133 * MB, false) && all(8703 * MB, false) && !all(8704 * MB, false), + "discrete cards under 8.5 GB: as before, and the line itself"); + for (uint64_t mb = 0; mb <= 49152; mb += 37) // (a discrete card is the old rule at every size) + expect(all(mb * MB, false) == all_by_vram(mb * MB), "a discrete card differs from the rule by the allowance alone"); if (failures) return EXIT_FAILURE; std::printf("astc bc policy: all checks passed\n"); return EXIT_SUCCESS; diff --git a/tests/test_astc_bc_policy.py b/tests/test_astc_bc_policy.py index 341a125..efb867d 100644 --- a/tests/test_astc_bc_policy.py +++ b/tests/test_astc_bc_policy.py @@ -4,7 +4,7 @@ import subprocess import tempfile from pathlib import Path -RUN_ALL_ARGS = ["--cxx", "g++"] +RUN_ALL_ARGS = ["--cxx", "{cxx}"] ROOT = Path(__file__).resolve().parents[1] @@ -12,10 +12,20 @@ ROOT = Path(__file__).resolve().parents[1] def check_source(): """The texture setup still asks the rule, and the switch that overrides it is still read""" src = (ROOT / "gpu/vk/vk_texture.cpp").read_text(encoding="utf-8") - for needle in ('#include "astc_bc_policy.h"', "astc_bc_policy::all_by_vram(allowed)", 'getenv("RX_VK_ASTC_BC_ALL")', - "(bw * bh >= 36 || all) ? low : 64ull << 20"): + # ((Triforce, 11-22, the performance audit's TEX-12) the rule is asked with the card's type; an unknown allowance is 0 + for needle in ('#include "astc_bc_policy.h"', "astc_bc_policy::all(allowed, integrated)", 'getenv("RX_VK_ASTC_BC_ALL")', + "(bw * bh >= 36 || all) ? low : 64ull << 20", + "if (!vram_usage(&used, &allowed)) allowed = 0;", + "const bool integrated = dev.props.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU;\n" + " const bool small = astc_bc_policy::all(allowed, integrated);", + "if (const char* e = getenv(\"RX_VK_ASTC_BC_ALL\")) return e[0] == '1';\n uint64_t used = 0, allowed = 0;"): if needle not in src: raise SystemExit("vk_texture.cpp no longer has: " + needle) + if "astc_bc_policy::all_by_vram(" in src: + raise SystemExit("vk_texture.cpp still asks the rule by the allowance alone: an integrated GPU allowed 8.5 GB or more is missed") + docs = (ROOT / "docs/development.md").read_text(encoding="utf-8") + if "| `RX_VK_ASTC_BC_ALL=1` or `0` |" not in docs: + raise SystemExit("docs/development.md: RX_VK_ASTC_BC_ALL is not in the switches table") def main(): diff --git a/tests/test_astc_reuse.py b/tests/test_astc_reuse.py new file mode 100644 index 0000000..5615af4 --- /dev/null +++ b/tests/test_astc_reuse.py @@ -0,0 +1,197 @@ +"""Decoded ASTC used again, its first step: the blocks hashed, the finalized textures listed, a hit counted +(gpu/vk/astc_reuse.h, vk_texture.cpp; no game, no GPU). + +(Triforce, 11-21, the performance audit's TEX-14, after Source A's.) When the game comes back to an area it makes its +textures again, mostly at other addresses, and the same ASTC blocks are decoded and encoded again while the image made +of them the first time may still be in VRAM. This step counts what taking that image would save (RX_VK_ASTC_REUSE=2); +nothing is copied. + +This compiles astc_reuse.h alone: the switch's modes; the two lanes are different hashes and both change with one byte; +a storage hashed in pieces of any size gives one hash, and another order or length another; never all zero; shapes that +differ in any one field never match; a texture is listed once, found by hash, both lanes and shape, never itself, and +unlisted by its pointer alone. + +And reads the source: with the switch off nothing of it runs; a new ASTC texture's blocks are hashed once at its first +upload and the count never stands in for the decode; a finalized one is listed only when it was uploaded from memory, +never drawn into, and of the finalized texture's own description; it is unlisted when used again or gone; the switch +is in the switches table. + + python -I -B tests/test_astc_reuse.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, head): + at = text.index(head) + return text[at:text.index("\n}\n", at) + 3] + + +def in_order(text, parts, why): + at = 0 + for p in parts: + at = text.find(p, at) + check(at >= 0, why + " (missing, or out of order: " + p + ")") + at += len(p) + + +PROGRAM = r""" +#include +#include +#include +#include "astc_reuse.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +using namespace astc_reuse; +struct Tex { Shape s; int id; }; +static Hash of(const std::vector& d, uint64_t piece) { + Hasher h; + for (uint64_t at = 0; at < d.size(); at += piece) h.add(d.data() + at, d.size() - at < piece ? d.size() - at : piece); + return h.done(); +} +int main() { + CHECK(mode(nullptr) == kOff && mode("") == kOff && mode("0") == kOff && mode("1") == kCopy && mode("2") == kCount && mode("3") == kOff && + mode("on") == kOff); + + std::vector d(3 * kStep + 12345); + uint32_t r = 7; + for (uint8_t& b : d) r = r * 1664525u + 1013904223u, b = (uint8_t)(r >> 24); + // the lanes: two hashes, each moved by one byte anywhere + CHECK(lane2(d.data(), d.size()) != upload_same::content_hash(d.data(), d.size())); + for (const size_t at : {(size_t)0, (size_t)63, (size_t)4096, d.size() / 2, d.size() - 1}) { + const Hash before = of(d, 64u << 20); + const uint64_t l2 = lane2(d.data(), d.size()); + d[at] ^= 0x10; + const Hash after = of(d, 64u << 20); + CHECK(after.a != before.a && after.b != before.b && lane2(d.data(), d.size()) != l2); + d[at] ^= 0x10; + CHECK(of(d, 64u << 20) == before); + } + // pieces: whatever multiple of the step the storage is read in, one hash + const Hash whole = of(d, 64u << 20); + CHECK(of(d, kStep) == whole && of(d, 2 * kStep) == whole && of(d, 1u << 20) == whole); + CHECK(bool(whole)); + { // the length and the order count + std::vector shorter(d.begin(), d.end() - 1), swapped(d); + for (size_t i = 0; i < kStep; i++) std::swap(swapped[i], swapped[i + kStep]); + CHECK(!(of(shorter, 64u << 20) == whole) && !(of(swapped, 64u << 20) == whole)); + Hasher none; + CHECK(bool(none.done())); // (never all zero: zero is "not hashed") + CHECK(!bool(Hash{})); + } + + // shapes: every field + const Shape s{131, 1024, 512, 1, 11, 6, 0, 0x79, 2, true, false, false, false, false}; + CHECK(s == s); + { Shape o = s; o.format = 145; CHECK(!(o == s)); } + { Shape o = s; o.width = 512; CHECK(!(o == s)); } + { Shape o = s; o.height = 1024; CHECK(!(o == s)); } + { Shape o = s; o.depth = 2; CHECK(!(o == s)); } + { Shape o = s; o.levels = 10; CHECK(!(o == s)); } + { Shape o = s; o.layers = 1; CHECK(!(o == s)); } + { Shape o = s; o.skip = 1; CHECK(!(o == s)); } + { Shape o = s; o.nvn_format = 0x7a; CHECK(!(o == s)); } + { Shape o = s; o.layout = 3; CHECK(!(o == s)); } + { Shape o = s; o.bc = false; CHECK(!(o == s)); } + { Shape o = s; o.bc_resized = true; CHECK(!(o == s)); } + { Shape o = s; o.bc1 = true; CHECK(!(o == s)); } + { Shape o = s; o.bc1_no = true; CHECK(!(o == s)); } + { Shape o = s; o.detail_skip = true; CHECK(!(o == s)); } + + // the list + Freed freed; + Tex a{s, 1}, b{s, 2}, c{s, 3}; + c.s.width = 2048; + const Hash h1{11, 22}, h2{11, 33}, h3{44, 22}; + const auto shape_of = [](const Tex* t) { return t->s; }; + CHECK(freed.size() == 0 && freed.find(h1, s, nullptr, shape_of) == nullptr); + freed.list(h1, &a); + CHECK(freed.size() == 1 && freed.find(h1, s, nullptr, shape_of) == &a); + CHECK(freed.find(h1, s, &a, shape_of) == nullptr); // (never itself) + CHECK(freed.find(h2, s, nullptr, shape_of) == nullptr); // (the second lane differs) + CHECK(freed.find(h3, s, nullptr, shape_of) == nullptr); // (the first lane differs) + CHECK(freed.find(h1, c.s, nullptr, shape_of) == nullptr); // (another shape) + freed.list(h1, &c); // (the same blocks decoded to another shape) + CHECK(freed.find(h1, c.s, nullptr, shape_of) == &c && freed.find(h1, s, nullptr, shape_of) == &a); + freed.list(h2, &b); // (the first lane alike, the second not) + CHECK(freed.size() == 3 && freed.find(h2, s, nullptr, shape_of) == &b && freed.find(h1, s, nullptr, shape_of) == &a); + Tex never{s, 4}; + freed.unlist(h1, &never), freed.unlist(h3, &b); // (not listed, or not under that hash: nothing goes) + CHECK(freed.size() == 3); + freed.unlist(h1, &a); + CHECK(freed.size() == 2 && freed.find(h1, s, nullptr, shape_of) == nullptr && freed.find(h1, c.s, nullptr, shape_of) == &c); + freed.unlist(h1, &a); // (once is all) + CHECK(freed.size() == 2); + freed.unlist(h2, &b), freed.unlist(h1, &c); + CHECK(freed.size() == 0); + return fails ? 1 : 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + tex = (VK / "vk_texture.cpp").read_text(encoding="utf-8").replace("\r\n", "\n") + head = (VK / "vk_texture.h").read_text(encoding="utf-8") + check('static const astc_reuse::Mode m = astc_reuse::mode(getenv("RX_VK_ASTC_REUSE"));' in tex, "the switch is not the rule's") + check("astc_reuse::Hash blocks_hash;" in head and "bool freed_listed = false;" in head, "a texture does not keep its blocks' hash") + + inner = block(tex, "bool upload_inner(Texture& tex, const Ranges* dirty) {") + ask = ("if (tex.astc && !tex.uploaded && !tex.blocks_hash && astc_reuse_mode() != astc_reuse::kOff && reuse_upload(tex, total))") + in_order(inner, [ask, "if (tex.astc && (up_route = UP_ASTC, upload_astc_gpu(tex, total, dirty))) return true;"], + "upload_inner does not hash a new ASTC texture's blocks once, only with the switch on, before its decode") + up = block(tex, "bool reuse_upload(Texture& tex, uint64_t total) {") + in_order(up, ["h.add(guest_bytes(tex.gpu + at, n, scratch), n);", "tex.blocks_hash = h.done();", + "if (tex.sparse || tex.type != VK_IMAGE_TYPE_2D || !tex.image) return false;", "std::lock_guard lk(reuse_mu);", + "reuse_freed.find(tex.blocks_hash, reuse_shape(tex), &tex,", + "if (!from || !from->image || !from->uploaded || from->gpu_written) return false;", + "reuse_freed.unlist(from->blocks_hash, from);", "from->freed_listed = false;", "st_reuse_hits"], + "reuse_upload does not hash the blocks as the decoder is given them, or look for a listed texture under the lock") + check("return true;" not in up, "reuse_upload stands in for the decode: this step only counts, and nothing was copied") + shape = block(tex, "astc_reuse::Shape reuse_shape(const Texture& t) {") + check("(uint32_t)t.format, t.width, t.height, t.depth, t.levels, t.layers, t.skip, (uint32_t)t.nvn_format, t.key_layout," in shape and + "t.bc, t.bc_resized, t.bc1, t.bc1_no, t.detail_skip};" in shape, "a texture's shape leaves a field of its image out") + + rel = block(tex, "void reuse_released(const nvn_tex* t) {") + in_order(rel, ["if (!x->astc || !x->blocks_hash || !x->uploaded || x->gpu_written || x->freed_listed) continue;", + "x->d.format != t->d.format", "x->d.levels != t->d.levels || x->d.target != t->d.target", "std::lock_guard lk(reuse_mu);", + "reuse_freed.list(x->blocks_hash, x);", "x->freed_listed = true;"], + "a finalized texture is listed though it was drawn into, not uploaded, not hashed, or of another description") + # ((Triforce, 11-22, the performance audit's MEM-85) texture_released's body is texture_released_at, from a given generation) + released = block(tex, "void texture_released_at(const nvn_tex* t, WriteGen gen) {") + check("if (astc_reuse_mode() != astc_reuse::kOff) reuse_released(t);" in released, "texture_released does not list, or lists with the switch off") + for name, fn in (("forget_gpu", "void forget_gpu(Texture* tex) {"), ("texture_touch", "Texture* texture_touch(Texture* tex, bool for_sampling) {"), + ("texture_touch_quick", "bool texture_touch_quick(Texture* tex, bool for_sampling) {")): + body = block(tex, fn) + check("if (tex->freed_listed) reuse_drop(*tex);" in body and body.index("if (tex->freed_listed) reuse_drop(*tex);") < 400, + name + " does not unlist a texture used again or gone: a later texture could be given a dangling or a live one's image") + drop = block(tex, "void reuse_drop(Texture& t) {") + in_order(drop, ["std::lock_guard lk(reuse_mu);", "if (t.freed_listed) reuse_freed.unlist(t.blocks_hash, &t), t.freed_listed = false;"], + "reuse_drop does not unlist under the lock") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`RX_VK_ASTC_REUSE=2`" in dev, "docs/development.md: RX_VK_ASTC_REUSE is not in the switches table") + + with tempfile.TemporaryDirectory(prefix="astc-reuse-") as tmp: + c, exe = Path(tmp) / "reuse.cpp", Path(tmp) / "reuse.exe" + c.write_text(PROGRAM, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O2", "-Wall", "-Wextra", "-Werror", "-I" + str(VK), str(c), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + check(r.returncode == 0, "the rules as made:\n" + r.stdout) + print("PASS: decoded ASTC's blocks are hashed, the finalized textures listed and a hit counted") + + +if __name__ == "__main__": + main() diff --git a/tests/test_audio_mix_effects.py b/tests/test_audio_mix_effects.py new file mode 100644 index 0000000..5cbc1a0 --- /dev/null +++ b/tests/test_audio_mix_effects.py @@ -0,0 +1,82 @@ +"""Compile the actual mix effects (biquad filter, delay, plain reverb), the splitter destinations' biquad filters and +their nn::audio setters from host/nn_audio.c around authored mixes and samples. + +The "Volcano eruption audio bugged" ticket: the game's AddBiquadFilter, AddDelay and AddReverb on its sub mixes and +SetSplitterBiquadFilterParameter were import stubs, so what it routes through them played unfiltered and too loud. +Checked: a low pass passes a low tone and takes a high one down, a delay's echo lands at the delay time and decays by +the feedback gain, the channel spread moves the echo to the other channel, the plain reverb adds a tail on top of the +dry signal, a splitter destination's filter reaches the voice's mix through render_voice, and with +HOS_AUDIO_EFFECTS=0 every sample is left as it was. No game, SDK or output device is used. +""" +import argparse +import importlib.util +import os +from pathlib import Path +import re +import subprocess +import sys +import tempfile + +RUN_ALL_ARGS = ["--cc", "gcc"] + +ROOT = Path(__file__).resolve().parents[1] +SPLIT_BIQUAD = ("_ZN2nn5audio32SetSplitterBiquadFilterParameterEPNS0_12SplitterTypeEiPNS0_10SubMixTypeEiRKNS0_22" + "TBiquadFilterParameterIfEE") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cc", default="cc") + parser.add_argument("--source", type=Path, default=ROOT / "host/nn_audio.c") + args = parser.parse_args() + spec = importlib.util.spec_from_file_location("audio_block_helper", ROOT / "tests/test_audio_deadline.py") + helper = importlib.util.module_from_spec(spec) + spec.loader.exec_module(helper) + source = args.source.read_text(encoding="utf-8") + constants = [] + for name in ("FRAME", "RATE", "AU_MAGIC", "MAX_VOICES", "MAX_MIXES", "MAX_SPLITTERS", "MAX_SPLIT_DESTS", + "MAX_CH", "MAX_BUF", "MAX_WB", "MAX_REL"): + matches = re.findall(r"^#define " + name + r"\s+[^\n]+", source, re.M) + assert len(matches) == 1, name + constants.extend(matches) + types = source[source.index("enum { K_VOICE"):source.index("static CRITICAL_SECTION lock;")] + queue = source[source.index("enum { OP_VOICE_RELEASE"):source.index("static int64_t last_update_ns;")] + effects = source[source.index("/* ---- I3DL2 reverb (on a sub mix"):source.index("/* ---- the mixer")] + setters = source[source.index("/* ---- I3DL2 reverb: AddI3dl2Reverb"):] + actual = ["\n".join(constants), types, """ +static Voice voices[MAX_VOICES]; +static Mix mixes[MAX_MIXES]; +static Splitter splitters[MAX_SPLITTERS]; +static int sink_mix = -1; +static void release_wb(Voice *voice, uint64_t buffer) { (void)voice; (void)buffer; } +"""] + actual.extend(helper.block(source, marker) for marker in ( + "static void put_handle(", "static int get_handle(", "static float farg(", "static int mix_index(", + "static Mix *mix_of(", "static int splitter_index(", "static Splitter *splitter_of(")) + actual.extend([queue, helper.block(source, "static void apply("), helper.block(source, "static void submit("), + helper.block(source, "static void commit("), effects, + "static int next_source_frame(Voice *v); /* (the fixture's tone) */", + helper.block(source, "static void render_voice("), + helper.block(source, "static void set_split_biquad("), + helper.block(source, "HLE(" + SPLIT_BIQUAD + ")"), setters]) + flags = subprocess.CREATE_NO_WINDOW | subprocess.BELOW_NORMAL_PRIORITY_CLASS if os.name == "nt" else 0 + with tempfile.TemporaryDirectory(prefix="audio-mix-effects-") as temporary: + folder = Path(temporary) + (folder / "actual_audio_effects.inc").write_text("\n\n".join(actual), encoding="utf-8") + executable = folder / "fixture.exe" + build = [args.cc, "-std=c11", "-O2", "-Wall", "-Wextra", "-Werror", "-Wno-unused-parameter", + "-Wno-unused-function", "-I" + str(folder), str(ROOT / "tests/audio_mix_effects_test.c"), "-lm", "-o", + str(executable)] + env = {k: v for k, v in os.environ.items() if not k.startswith("HOS_AUDIO")} + runs = [(build, env), ([str(executable)], env), ([str(executable)], dict(env, HOS_AUDIO_EFFECTS="0"))] + for command, environment in runs: + result = subprocess.run(command, creationflags=flags, capture_output=True, text=True, env=environment, + encoding="utf-8", errors="replace", timeout=60) + print(result.stdout, end="") + print(result.stderr, end="", file=sys.stderr) + if result.returncode: + raise SystemExit(result.returncode) + + +if __name__ == "__main__": + main() diff --git a/tests/test_barrier_rules.py b/tests/test_barrier_rules.py index 0104b34..9d679d7 100644 --- a/tests/test_barrier_rules.py +++ b/tests/test_barrier_rules.py @@ -106,8 +106,25 @@ def check_source(): submit = block(replay, "void submit_recording(Recording& r, bool signal_value = true, uint64_t signal_at = 0, bool part = false) {") in_order(submit, ["if (event_writes_held(r.cmd)) {", "end_pass(r.cmd), full_barrier(r.cmd,", "draw_flush_recording(r.slot);", "if (batch_on()) {", "barrier_rules::batch_hold(way, !signal_value, submit_depth > 0);", - "if (hold || !pend.cbs.empty()) {", "pend.recs++;", "barrier_rules::batch_flush(way, hold,", "return;", + "if (hold || !pend.cbs.empty()) {", "pend.recs++;", "barrier_rules::batch_flush(way, hold,", + # ((Triforce, 11-22, the performance audit's SYNC-09) RX_VK_BATCH_IDLE: after the cap's flush, the rule before the GPU's poll + "if (barrier_rules::batch_idle(way, !pend.cbs.empty(), batch_idle_on()) && batch_gpu_idle()) flush_reason = FR_IDLE, queue_flush_locked();", + "return;", "vkQueueSubmit2(dev.queue, 1, &si, r.fence);"], "vk_replay.cpp submit_recording") + # ((Triforce, 11-22, the performance audit's SYNC-09) RX_VK_BATCH_IDLE: read once through the rule, off unless set; the GPU's + # poll under the queue's lock; its flushes counted as "idle"; the switches table + idle_on = block(replay, "bool batch_idle_on() {") + if 'barrier_rules::batch_idle_on(getenv("RX_VK_BATCH_IDLE"));' not in idle_on or "static const bool on" not in idle_on: + raise SystemExit("vk_replay.cpp batch_idle_on does not read RX_VK_BATCH_IDLE once through the rule") + gpu_idle = block(replay, "bool batch_gpu_idle() {") + if "vkGetSemaphoreCounterValue(dev.device, batch_tl, &done) == VK_SUCCESS && done >= batch_flushed;" not in gpu_idle: + raise SystemExit("vk_replay.cpp batch_gpu_idle is not the batches' timeline against the last batch flushed") + if replay.count("batch_gpu_idle()") != 2 or "FR_VALUE, FR_IDLE, FR_COUNT };" not in replay or \ + "(long long)(st_reason[FR_IDLE] - r0[FR_IDLE])," not in replay or "value %lld, idle %lld, other %lld" not in replay: + raise SystemExit("vk_replay.cpp: the idle flush is asked for somewhere else, or not counted in the batched submits' line") + dev_md = (VK.parent.parent / "docs/development.md").read_text(encoding="utf-8") + if not any(line.startswith("| `RX_VK_BATCH_IDLE=1` |") and "**off unless set on every driver**" in line for line in dev_md.splitlines()): + raise SystemExit("docs/development.md: no row for RX_VK_BATCH_IDLE with its default") job = block(replay, "void submit_job(std::vector& handles, uint64_t value, uint64_t queue) {") in_order(job, ["submit_depth++;", "if (!--submit_depth && batch_way() == barrier_rules::Batch::Parts) queue_flush_for(FR_OTHER);", "run_submit("], "vk_replay.cpp submit_job") @@ -116,7 +133,7 @@ def check_source(): raise SystemExit("vk_replay.cpp queue_flush_locked: the held recordings' count is not cleared with them") begin = block(draw, "void begin_pass(") in_order(begin, ["end_pass(cmd);", "if (pass_resuming) pass_resume_barrier(cmd);", "else pass_start_barrier(cmd);", - "work_since_barrier = true;", "pass_at = at;", "pass().sampled.clear(), pass().sampled_targets = 0, pass().written = 0;", + "work_since_barrier = true;", "pass_at = at;", "ps.sampled.clear(), ps.sampled_targets = 0, ps.written = 0;", "vkCmdBeginRendering(cmd, &ri);"], "vk_draw.cpp begin_pass") take = block(draw, "bool pass_carry_take() {") if "pass_carry_on = pass().open && !par_active.load(std::memory_order_relaxed);" not in take or \ @@ -208,6 +225,14 @@ int main() { CHECK(batch_flush(Batch::Off, false, 0, 32)); CHECK(!batch_flush(Batch::Hold, true, 31, 32) && batch_flush(Batch::Hold, true, 32, 32)); CHECK(batch_flush(Batch::Parts, false, 1, 4) && !batch_flush(Batch::Parts, true, 3, 4) && batch_flush(Batch::Parts, true, 4, 4)); + // ((Triforce, 11-22, the performance audit's SYNC-09) RX_VK_BATCH_IDLE: off unless set to a number other than 0; the rule + // only in Hold mode, with something held, the switch on + CHECK(!batch_idle_on(nullptr) && !batch_idle_on("") && !batch_idle_on("0") && !batch_idle_on("off") && batch_idle_on("1") && batch_idle_on("2")); + CHECK(batch_idle(Batch::Hold, true, true)); + CHECK(!batch_idle(Batch::Hold, true, false) && !batch_idle(Batch::Hold, false, true)); + CHECK(!batch_idle(Batch::Parts, true, true) && !batch_idle(Batch::Off, true, true)); + for (int m = 0; m < 3; m++) // (unset, nothing asks the GPU, in any mode) + for (int held = 0; held < 2; held++) CHECK(!batch_idle((Batch)m, held != 0, batch_idle_on(nullptr))); // (submit_recording's way with RX_VK_BATCH=parts, over a game's submits: the command buffers reach the queue in the order they // were submitted, a part never waits, at most 4 wait, and none is left once the submit's replay returns) struct Sub { int id; bool part; }; diff --git a/tests/test_bc_scratch_layers.py b/tests/test_bc_scratch_layers.py new file mode 100644 index 0000000..1909120 --- /dev/null +++ b/tests/test_bc_scratch_layers.py @@ -0,0 +1,114 @@ +"""The BC3 scratch has the layers one run uses, not 8 (gpu/vk/bc_scratch.h, vk_texture.cpp bc_layers, bc_scratch_layers, +bc_scratch_make, bc_scratch2_make, bc_scratch_layout, texture_copy_into_bc; no game, no GPU). + +(Triforce, 11-21, the performance audit's TEX-10, after Source A's.) Big ASTC textures are decoded into a 2048 x 2048 +RGBA8 scratch and encoded again as BC3 into an output buffer. Both were made with 8 layers, 160 MB, while a run decodes +one layer: 140 MB held and never written, for good once made. + +This compiles bc_scratch.h alone and checks the rule: a run is 1 layer unless RX_VK_BC_LAYERS says 1 to 8; the scratch +has a run's layers, or 8 with RX_VK_BC_SCRATCH_FULL; one layer is 16 MB of image and 4 MB of output, 20 MB, 36 with the +second scratch, and 8 layers are the 160 and 288 MB it was. + +And reads the source: both scratch images, their views, the layout's barriers and the output buffer are sized by +bc_scratch_layers(), nowhere by kBcLayers; a run's layers are bc_layers(), which the scratch is sized from; the copy into +a BC3 texture takes no more layers at once than the scratch has; the made line says the size; the switches are in the +switches table. + + python -I -B tests/test_bc_scratch_layers.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +PROGRAM = r""" +#include +#include +#include +#include "bc_scratch.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +int main() { + using namespace bc_scratch; + CHECK(kDim == 2048 && kMost == 8); + CHECK(run_layers(nullptr) == 1 && run_layers("") == 1 && run_layers("x") == 1 && run_layers("0") == 1 && run_layers("-3") == 1); + CHECK(run_layers("1") == 1 && run_layers("4") == 4 && run_layers("8") == 8 && run_layers("99") == 8); + for (uint32_t run : {1u, 2u, 8u}) CHECK(layers(run, false) == run && layers(run, true) == 8); + CHECK(layers(0, false) == 1 && layers(64, false) == 8); + CHECK(out_bytes(1) == 4ull << 20 && out_bytes(8) == 32ull << 20); + CHECK(mb(1, false) == 20 && mb(1, true) == 36); // (16 MB of image, 4 MB of output; the second image 16 more) + CHECK(mb(8, false) == 160 && mb(8, true) == 288); // (what it was) + return fails ? 1 : 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + tex = (VK / "vk_texture.cpp").read_text(encoding="utf-8") + + check('#include "bc_scratch.h"' in tex, "vk_texture.cpp does not use the rule") + run = block(tex, "uint32_t bc_layers() {") + check('bc_scratch::run_layers(getenv("RX_VK_BC_LAYERS"))' in run, "a run's layers are not the rule's of RX_VK_BC_LAYERS") + size = block(tex, "uint32_t bc_scratch_layers() {") + check('bc_scratch::layers(bc_layers(), getenv("RX_VK_BC_SCRATCH_FULL") != nullptr)' in size, + "the scratch's layers are not a run's, with all 8 behind RX_VK_BC_SCRATCH_FULL") + for name in ("bool bc_scratch_make() {", "bool bc_scratch2_make() {"): + body = block(tex, name) + check("kBcLayers" not in body and "ii.arrayLayers = bc_scratch_layers();" in body and + "vi.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, bc_scratch_layers()};" in body, + name.split("(")[0] + ": the image or its view is not sized by bc_scratch_layers()") + make = block(tex, "bool bc_scratch_make() {") + # ((11-21, TEX-09) the one buffer holds a run of either scratch: the first's, or the small one's when it is larger) + check("const uint64_t out = std::max(bc_scratch::out_bytes(bc_scratch_layers()), small_out);" in make, + "the output buffer is not the rule's size") + check('logf("vk: the BC3 scratch is made: %u layer' in make, "the made scratch's size is not logged") + layout = block(tex, "bool bc_scratch_layout(") + check("kBcLayers" not in layout and layout.count("0, 1, 0, bc_scratch_layers()};") == 2, + "bc_scratch_layout's barriers cover layers the scratch may not have") + copy = block(tex, "bool texture_copy_into_bc(") + check("kBcLayers" not in copy and "l0 += bc_scratch_layers()" in copy and "std::min(bc_scratch_layers(), layers - l0)" in copy, + "a copy into a BC3 texture takes more layers at once than the scratch has") + # ((11-21, TEX-09) a level the small scratch holds runs its layers; any other, bc_layers(), what the first is sized by) + check("const uint32_t run = small ? bc_small_layers() : bc_layers();" in tex, + "the upload's run through the first scratch is not bc_layers(), what that scratch is sized by") + idle = block(tex, "void bc_scratch_idle() {") + check("bc_scratch::mb(bc_scratch_layers(), second)" in idle, "the given-back line still states 160 or 288 MB") + stub = (ROOT / "tests" / "parallel_init_order_test.cpp").read_text(encoding="utf-8") + check("uint32_t bc_scratch_layers() { return kBcLayers; }" in stub, "parallel_init_order_test.cpp cannot compile bc_scratch_layout") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`RX_VK_BC_LAYERS=`, `RX_VK_BC_SCRATCH_FULL=1`" in dev, "docs/development.md: the BC scratch's switches are not in the table") + + with tempfile.TemporaryDirectory(prefix="bc-scratch-") as tmp: + src, exe = Path(tmp) / "scratch.cpp", Path(tmp) / "scratch.exe" + src.write_text(PROGRAM, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(VK), str(src), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + check(r.returncode == 0, "the rule as made:\n" + r.stdout) + print("PASS: the BC3 scratch has the layers one run uses: 20 MB, not 160") + + +if __name__ == "__main__": + main() diff --git a/tests/test_bc_small_layers.py b/tests/test_bc_small_layers.py new file mode 100644 index 0000000..0ad6203 --- /dev/null +++ b/tests/test_bc_small_layers.py @@ -0,0 +1,148 @@ +"""BC mip levels of 1024 or less are re-encoded 8 layers a run, through a small scratch (gpu/vk/bc_scratch.h, vk_texture.cpp +bc_small_layers, bc_scratch_make, bc_scratch_layout, bc_scratch_idle, bc_encode_into, upload_astc_gpu; no game, no GPU). + +(Triforce, 11-21, the performance audit's TEX-09, after Source A's.) A run is a decode, an encode, a copy and three full +barriers, and one layer a run made a terrain array of 90 layers and 11 levels about 990 of them. + +This compiles bc_scratch.h alone and checks the rule: the small scratch's layers are 8 unless RX_VK_BC_SMALL_LAYERS says 1 +to 8, and 1 on AMD's cards when it is not set; a level of 1024 or less each way goes through it, a 1365 level, a level +1025 one way, a texture encoded at another size and a start with no small scratch do not; a layer of a run begins where the +encoder wrote it (16 bytes a block of 4 x 4, 8 as BC1, uneven sizes rounded up to blocks); and it counts a 1365 x 1365 x 90 +array's runs: 990 at one layer a run, 210 with 8 for its levels from the second down. + +And reads the source: the small scratch is made with the first, only at full texture detail and with more than one layer +asked, and given back with it; its layout is set before its first use; the one output buffer holds a run of either; a +level asks the rule which scratch and takes that scratch's run, view and encode source; the copy has a region a layer at +the rule's offsets; the switch is in the switches table. + + python -I -B tests/test_bc_small_layers.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +PROGRAM = r""" +#include +#include +#include +#include "bc_scratch.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +// the runs of one array: every level, its layers in runs of the scratch it goes through +static int runs(uint32_t w, uint32_t h, uint32_t layers, uint32_t levels, bool have_small, uint32_t small_n, uint32_t big_n) { + int n = 0; + for (uint32_t l = 0; l < levels; l++) { + const uint32_t lw = std::max(1u, w >> l), lh = std::max(1u, h >> l); + const uint32_t run = bc_scratch::small_level(lw, lh, have_small, false) ? small_n : big_n; + n += (int)((layers + run - 1) / run); + } + return n; +} +int main() { + using namespace bc_scratch; + CHECK(kSmallDim == 1024); + const uint32_t nvidia = 0x10DE, amd = 0x1002, intel = 0x8086; + CHECK(small_layers(nullptr, nvidia) == 8 && small_layers(nullptr, intel) == 8 && small_layers("", nvidia) == 8); + CHECK(small_layers(nullptr, amd) == 1 && small_layers("", amd) == 1); // (until the picture is checked there) + CHECK(small_layers("8", amd) == 8 && small_layers("4", amd) == 4); // (asked for: on any card) + CHECK(small_layers("1", nvidia) == 1 && small_layers("0", nvidia) == 1 && small_layers("99", nvidia) == 8); + CHECK(small_level(1024, 1024, true, false) && small_level(682, 682, true, false) && small_level(1, 1, true, false)); + CHECK(!small_level(1365, 1365, true, false) && !small_level(1025, 4, true, false) && !small_level(4, 1025, true, false)); + CHECK(!small_level(512, 512, false, false)); // (no small scratch: RX_VK_BC_SMALL_LAYERS=1, AMD, reduced detail) + CHECK(!small_level(512, 512, true, true)); // (encoded at another size: the second scratch's) + CHECK(small_out_bytes(8) == 8ull << 20 && small_out_bytes(1) == 1ull << 20 && small_image_bytes(8) == 32ull << 20); + CHECK(small_out_bytes(8) > out_bytes(1)); // (so the one output buffer has to be the larger of the two) + // where a run's layers are in the encoder's output + CHECK(layer_offset(0, 682, 682, false) == 0); + CHECK(layer_offset(1, 1024, 1024, false) == 256ull * 256 * 16 && layer_offset(7, 1024, 1024, false) == 7 * 256ull * 256 * 16); + CHECK(layer_offset(1, 682, 682, false) == 171ull * 171 * 16); // (682 texels: 171 blocks, rounded up) + CHECK(layer_offset(3, 85, 43, false) == 3 * 22ull * 11 * 16); + CHECK(layer_offset(2, 512, 512, true) == 2 * 128ull * 128 * 8); // (kept as BC1: the colour half alone) + CHECK(layer_offset(7, 1024, 1024, false) + 256ull * 256 * 16 <= small_out_bytes(8)); // (the last layer fits the buffer) + // a terrain array, 1365 x 1365 x 90 with 11 levels + CHECK(runs(1365, 1365, 90, 11, false, 8, 1) == 990); + CHECK(runs(1365, 1365, 90, 11, true, 8, 1) == 90 + 10 * 12); // (its first level one layer a run, the rest 8) + CHECK(runs(1365, 1365, 90, 11, true, 8, 1) == 210); + return fails ? 1 : 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + tex = (VK / "vk_texture.cpp").read_text(encoding="utf-8") + + layers = block(tex, "uint32_t bc_small_layers() {") + check('bc_scratch::small_layers(getenv("RX_VK_BC_SMALL_LAYERS"), dev.props.vendorID)' in layers, + "the small scratch's layers are not the rule's of the switch and the card's vendor") + make = block(tex, "bool bc_scratch_make() {") + check("if (bc_small_layers() > 1 && texture_detail_level() == tex_detail::kFull) {" in make, + "the small scratch is made with one layer asked, or at reduced texture detail") + check("si.extent = {bc_scratch::kSmallDim, bc_scratch::kSmallDim, 1};" in make and "si.arrayLayers = bc_small_layers();" in make and + "sv.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, bc_small_layers()};" in make, + "the small scratch or its view is not 1024 x 1024 with a run's layers") + check("const uint64_t out = std::max(bc_scratch::out_bytes(bc_scratch_layers()), small_out);" in make and + "bc_small ? bc_scratch::small_out_bytes(bc_small_layers()) : 0" in make, + "the output buffer does not hold a run of either scratch") + check("bc_small_fresh = bc_small != VK_NULL_HANDLE;" in make, "a new small scratch's layout is not marked to be set") + failed = make[make.index("!make_out()) {"):make.index("bc_scratch_fresh = true;")] + check("if (bc_small) destroy_image(bc_small, bc_small_alloc);" in failed, "the small scratch stays when the first could not be made") + layout = block(tex, "bool bc_scratch_layout(") + check("|| (bc_small_fresh && bc_small));" in layout and "b.image = bc_small;" in layout and + "b.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, bc_small_layers()};" in layout and + layout.index("b.image = bc_small;") < layout.index("if (!bc_scratch_fresh) return true;"), + "the small scratch's layout is not set before its first use") + idle = block(tex, "void bc_scratch_idle() {") + check("dead.push_back(Dead{bc_last_value, bc_small, bc_small_alloc, {bc_small_view}, {}});" in idle and + "bc_small = VK_NULL_HANDLE" in idle, "the small scratch is not given back with the first") + start = tex.index(" if (tex.bc) { // (decoded into the scratch a run of consecutive layers at a time, re-encoded, copied in)") + loop = tex[start:tex.index(" st_astc++;", start)] + check("const bool small = bc_scratch::small_level(g.w, g.h, bc_small != VK_NULL_HANDLE, tex.bc_resized);" in loop and + "const uint32_t run = small ? bc_small_layers() : bc_layers();" in loop, + "a level does not ask the rule which scratch it goes through, or does not take that scratch's run") + check("VkDescriptorImageInfo ii{VK_NULL_HANDLE, small ? bc_small_view : bc_scratch_view, VK_IMAGE_LAYOUT_GENERAL};" in loop and + "bc_encode_into(cmd, iw, ih, n, tex, level, layers[k0], 0, 0, small ? bc_small_view : VK_NULL_HANDLE);" in loop, + "a small level is not decoded into the small scratch and encoded from it") + check("bc_encode_resized(cmd, g.w, g.h, iw, ih, n, tex, level, layers[k0]);" in loop, "a resized texture's levels lost their own path") + enc = block(tex, "void bc_encode_into(") + check("VkBufferImageCopy c[kBcLayers];" in enc and + "bc_scratch::layer_offset(k, w, h, dst.bc1), 0, 0, {VK_IMAGE_ASPECT_COLOR_BIT, level, layer + k, 1}" in enc and + "vkCmdCopyBufferToImage(cmd, bc_out, dst.image, VK_IMAGE_LAYOUT_GENERAL, std::min(n, kBcLayers), c);" in enc, + "a run's layers are not copied by a region each, at the encoder's layer stride") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`RX_VK_BC_SMALL_LAYERS=`" in dev, "docs/development.md: RX_VK_BC_SMALL_LAYERS is not in the switches table") + + with tempfile.TemporaryDirectory(prefix="bc-small-") as tmp: + c, exe = Path(tmp) / "small.cpp", Path(tmp) / "small.exe" + c.write_text(PROGRAM, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(VK), str(c), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + check(r.returncode == 0, "the rule as made:\n" + r.stdout) + print("PASS: BC levels of 1024 or less run 8 layers at once through the small scratch: a 90-layer array's 990 runs are 210") + + +if __name__ == "__main__": + main() diff --git a/tests/test_bg_import_held.py b/tests/test_bg_import_held.py index d1364b3..778cee6 100644 --- a/tests/test_bg_import_held.py +++ b/tests/test_bg_import_held.py @@ -82,6 +82,10 @@ void logf(const char*, ...) {} void guest_pages_set(uint64_t, uint64_t, uint64_t) {} void retire_pool(std::map::iterator it) { retired_pools.push_back(imported.extract(it)); } std::vector> bg_held; +std::deque::node_type> evicted_pools; // (the imports given back and not freed yet: none here) +@@DEAD_IMPORT@@ +std::deque dead_imports; +@@LEAVING@@ @@SEND@@ @@LET_GO@@ @@DRAIN@@ @@ -180,6 +184,8 @@ def main(): fixture = (FIXTURE.replace("@@SEND@@", send) .replace("@@LET_GO@@", block(memory, "void bg_let_go(uint64_t lo, uint64_t hi) {")) .replace("@@DRAIN@@", block(memory, "void bg_drain() {")) + .replace("@@DEAD_IMPORT@@", re.search(r"^struct DeadImport \{[^\n]*\};$", memory, re.M).group(0)) + .replace("@@LEAVING@@", block(memory, "bool imports_leaving_overlap(uint64_t lo, uint64_t hi) {")) .replace("@@IMPORTED@@", block(memory, "int memory_guest_imported(uint64_t lo, uint64_t hi) {"))) with tempfile.TemporaryDirectory(prefix="bg-import-held-") as tmp: src, exe = Path(tmp) / "held.cpp", Path(tmp) / "held.exe" diff --git a/tests/test_build_priority.py b/tests/test_build_priority.py new file mode 100644 index 0000000..9cbd715 --- /dev/null +++ b/tests/test_build_priority.py @@ -0,0 +1,125 @@ +"""The builders start the build tools at normal priority, and below normal only on request (no game, no build). + +(Triforce, 11-22, the performance audit's OS-57, after Source A's.) Every Windows build started nx2/make_game.py below +normal priority, from four places, with no switch. Source A starts its tools at normal priority and reports, without a +measurement, that below normal Windows kept the one-process steps of a build on a hybrid Intel CPU's efficiency cores. +HOOKSHOT_LOW_PRIORITY=1 brings the below-normal start back. + +Reads the source: in each of the four files the below-normal set is there once and sits directly under a check of +HOOKSHOT_LOW_PRIORITY == "1"; the switches table lists the switch; prepare_priority's comment and the design notes +say what the default is now. + +And runs the Python builder's own start (builder/py/hookshot_builder/core.py Runner._run) with a stand-in for the tools: +a child that prints its own priority class. By default the child has the class a plain start gives it; with +HOOKSHOT_LOW_PRIORITY=1 it is below normal; =0 and an empty value are the default. The rule build_priority is asked the +same on every OS; the start itself only on Windows. + + python -I -B tests/test_build_priority.py +""" +import os +import re +import subprocess +import sys +import tempfile +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +SWITCH = "HOOKSHOT_LOW_PRIORITY" +BELOW_NORMAL, IDLE, NORMAL = 0x4000, 0x40, 0x20 +# (a child that says the priority class Windows gave it) +CHILD = "import ctypes; k = ctypes.WinDLL('kernel32'); k.GetCurrentProcess.restype = ctypes.c_void_p; " \ + "k.GetPriorityClass.argtypes = [ctypes.c_void_p]; print('class', k.GetPriorityClass(k.GetCurrentProcess()))" + + +def fail(text): + raise SystemExit("FAIL: " + text) + + +def check_csharp(rel, set_line): + text = (ROOT / rel).read_text(encoding="utf-8") + code = [line.strip() for line in text.splitlines() if line.strip() and not line.strip().startswith("//")] + sets = [i for i, line in enumerate(code) if "ProcessPriorityClass.BelowNormal" in line] + if len(sets) != 1: + fail("%s sets the below normal class %d times, not once" % (rel, len(sets))) + i = sets[0] + if code[i] != set_line: + fail("%s: the below normal set is no longer the line the check knows: %s" % (rel, code[i])) + if code[i - 1] != 'if (Environment.GetEnvironmentVariable("%s") == "1")' % SWITCH: + fail("%s starts the build tools below normal without asking %s: the line above the set is: %s" % (rel, SWITCH, code[i - 1])) + if "the performance audit's OS-57" not in text: + fail("%s no longer says where the switch comes from" % rel) + + +def own_class(): + import ctypes + k = ctypes.WinDLL("kernel32") + k.GetCurrentProcess.restype = ctypes.c_void_p + k.GetPriorityClass.argtypes = [ctypes.c_void_p] + return k.GetPriorityClass(k.GetCurrentProcess()) + + +def main(): + # ---- the three C# starts + check_csharp("builder/HookshotBuilder/Runner.cs", "try { p.PriorityClass = ProcessPriorityClass.BelowNormal; } catch (Exception) { }") + check_csharp("setup/Setup.cs", "try { p.PriorityClass = ProcessPriorityClass.BelowNormal; } catch (Exception) { }") + check_csharp("launcher/Logic/Build/BuildRunner.cs", + "try { if (ctx.IsWindows) p.PriorityClass = ProcessPriorityClass.BelowNormal; } catch (Exception) { }") + head = (ROOT / "launcher/Logic/Build/BuildRunner.cs").read_text(encoding="utf-8").split("using System", 1)[0] + if "below-normal priority (Windows)" in head or SWITCH not in head: + fail("BuildRunner.cs's header still says the tools always run below normal") + + # ---- the Python builder: the source, the rule, then its own start + core_text = (ROOT / "builder/py/hookshot_builder/core.py").read_text(encoding="utf-8") + uses = re.findall(r"\bBELOW_NORMAL_PRIORITY_CLASS\b", core_text) + if len(uses) != 2 or 'return BELOW_NORMAL_PRIORITY_CLASS if env.get("%s") == "1" else 0' % SWITCH not in core_text: + fail("core.py uses BELOW_NORMAL_PRIORITY_CLASS outside build_priority's %s check" % SWITCH) + if "creationflags=CREATE_NO_WINDOW | build_priority())" not in core_text: + fail("core.py's Runner no longer starts the tools with build_priority()") + sys.path.insert(0, str(ROOT / "builder" / "py")) + from hookshot_builder import core + for env, want in (({}, 0), ({SWITCH: "1"}, BELOW_NORMAL), ({SWITCH: "0"}, 0), ({SWITCH: ""}, 0), ({SWITCH: "yes"}, 0)): + if core.build_priority(env) != want: + fail("build_priority(%r) is 0x%x, want 0x%x" % (env, core.build_priority(env), want)) + started = "the rule only (the start is Windows')" + if os.name == "nt": + mine = own_class() + plain = mine if mine in (BELOW_NORMAL, IDLE) else NORMAL # (what a child gets with no priority flag) + with tempfile.TemporaryDirectory() as folder: + tools = SimpleNamespace(python=sys.executable, python_args=[], root=folder) + for value, want in ((None, plain), ("0", plain), ("1", BELOW_NORMAL if mine != IDLE else None)): + if want is None: # (an idle-class parent: Windows does not raise the child; nothing to tell apart) + continue + env = {k: v for k, v in os.environ.items() if k != SWITCH} + if value is not None: + env[SWITCH] = value + lines, ended = [], [] + with patch.dict(os.environ, env, clear=True), \ + patch.object(core, "build_command", return_value=[sys.executable, "-I", "-c", CHILD]), \ + patch.object(core, "build_env", return_value=dict(env)): + core.Runner(tools, [], on_line=lines.append, on_finished=ended.append)._run() + if ended != [None] or ["class %d" % want] != [l for l in lines if l.startswith("class ")]: + fail("the tools started with %s=%s have %s (ended %s), want class %d" % (SWITCH, value, lines, ended, want)) + started = "the Python builder's start gives its child class 0x%x by default and 0x%x with the switch" % (plain, BELOW_NORMAL) + + # ---- the docs and the comments that described the old start + docs = (ROOT / "docs/development.md").read_text(encoding="utf-8") + rows = [line for line in docs.splitlines() if line.startswith("| `%s=1` |" % SWITCH)] + if len(rows) != 1 or "OS-57" not in rows[0] or "normal priority" not in rows[0]: + fail("docs/development.md's switches table has no %s=1 row" % SWITCH) + building = docs[docs.index("### Building"):docs.index("## Mods: code patches and cheats")] + if rows[0] not in building: + fail("the %s row is not in the Building table" % SWITCH) + design = (ROOT / "docs/design.md").read_text(encoding="utf-8") + if SWITCH not in design: + fail("docs/design.md's first-build line does not say the builders' start changed") + draw = (ROOT / "gpu/vk/vk_draw.cpp").read_text(encoding="utf-8") + comment = draw[draw.index("/// (Triforce) The priority of the workers of the shader preparation"):draw.index("int prepare_priority() {")] + if SWITCH not in comment or "builders start their build tools so: Runner.cs" in comment: + fail("prepare_priority's comment still says the builders always start their tools below normal") + print("ok: the four builders ask %s before a below-normal start; %s" % (SWITCH, started)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_built_check.py b/tests/test_built_check.py new file mode 100644 index 0000000..dd7777b --- /dev/null +++ b/tests/test_built_check.py @@ -0,0 +1,413 @@ +"""Whether the caches still have what the light precompile's record lists (gpu/vk/built_check.h; vk_draw.cpp +built_check_cold, built_check_made, built_play_publish, built_play_build, precompile_run; no game, no GPU). + +(Triforce, 11-22, the performance audit's PIPE-21, after Source A's and Source B's.) A light precompile (little RAM) builds +the noted pipelines for the caches, lets them go and records their keys in .built; later starts left everything the +record lists to its first draw, trusting it on its stamp alone. When the caches lost them, each of those first draws +compiled in play. Now a sample of the recorded notes is asked of Triforce's own caches first (as a draw asks, without +compiling, where the device can) and some listed compute programs as a dispatch asks; at half or more cold, everything is +built again before the title. Cold again right after that: left to the draws, with a hint. RX_VK_BUILT_CHECK=0: as before. + +The header is compiled and run as it is: the switch; a pipeline cold by the caches' answer, or without the flag by its +first build against its own second; the sample (24 answers: 12 cold is cold, 11 warm; 15 answers say nothing); the places; +the record of strikes; starts in a row (cold: built again; cold again: left to the draws; warm forgets; another stamp anew; +mode 2 always builds again); the counter's rules. + +Then built_check_cold, built_check_made, built_play_publish and built_play_build are cut out of vk_draw.cpp and run over a +stand-in driver whose caches hold pipelines by key: + - warm caches: not cold, 24 notes and 6 compute programs asked, each the note's own variant, nothing compiled, every + pipeline asked for let go, the caches as they were; + - lost caches: cold, still nothing compiled (asked without compiling: the same pipelines exist afterwards); the record + of the rebuild written; cold again: left to the draws with the hint, not cold; warm: the strike forgotten; + - 12 of 24 cold is cold, 11 is not; 15 answers (9 refused) say nothing, 16 do; + - without the flag: each recorded note built twice and timed, the cold ones compiled into the caches (exactly the + recorded pipelines, nothing else), compute programs not asked; + - the switch at 0, or an empty record: nothing asked; at 2: cold with warm caches; + - the counter: nothing before the record is published (nor with the switch at 0), then only the record's keys and its + compute programs, one line at most every 10 s. +And reads the source: precompile_run asks before the erase and forgets the record when cold, records the strikes after the +record is written again, publishes the record for the counter; build_graphics and the dispatch count; the switch is read +once; the switches table has the row. + + python -I -B tests/test_built_check.py [--cxx clang++] +""" +import argparse +import os +import subprocess +import tempfile +from pathlib import Path + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] + +RULES = r""" +#include +#include +#include "built_check.h" +#define CHECK(x) do { if (!(x)) { std::printf("line %d: %s\n", __LINE__, #x); return 1; } } while (0) +using namespace built_check; +static Probe asked(bool cold_answer) { Probe p; p.asked = true; p.answered_cold = cold_answer; p.made = !cold_answer; return p; } +static Probe timed(double first, double again) { Probe p; p.made = true; p.first_ms = first; p.again_ms = again; return p; } +static Probe refused() { Probe p; p.asked = true; return p; } +static std::vector sample(int cold_n, int warm_n, int refused_n = 0) { + std::vector v; + for (int i = 0; i < cold_n; i++) v.push_back(asked(true)); + for (int i = 0; i < warm_n; i++) v.push_back(asked(false)); + for (int i = 0; i < refused_n; i++) v.push_back(refused()); + return v; +} +static Do start(const std::string& rec, uint64_t stamp, Verdict v, bool always = false) { + const State had = state_read(rec); + int next = -1; + const Do d = decide(v, had, stamp, always, &next); + if (v == Verdict::Warm && had.found) remove(rec.c_str()); + if (d == Do::Again && !state_write(rec, stamp, next)) return Do::Nothing; // (as built_check_made, once built again) + return d; +} +int main(int argc, char** argv) { + if (argc < 2) return 2; + const std::string dir = argv[1]; + CHECK(mode(nullptr) == 1 && mode("") == 1 && mode("1") == 1 && mode("0") == 0 && mode("2") == 2 && mode("9") == 2 && mode("-1") == 0); + // one pipeline + CHECK(cold(asked(true)) && !cold(asked(false)) && !counts(refused()) && !cold(refused())); + CHECK(cold(timed(8.5, 1.0)) && !cold(timed(6.9, 1.0)) && !cold(timed(0.4, 0.3)) && cold(timed(3.2, 0.0))); + { + Probe p = timed(80, 0.5); + p.asked = true; // (asked, the answer alone counts: a slow hit is a hit) + CHECK(!cold(p)); + } + // the sample: 24 answers, 12 cold is cold, 11 warm; 15 answers say nothing, 16 do + CHECK(verdict(tally(sample(12, 12))) == Verdict::Cold && verdict(tally(sample(11, 13))) == Verdict::Warm); + CHECK(verdict(tally(sample(15, 0, 9))) == Verdict::Unknown && verdict(tally(sample(16, 0, 8))) == Verdict::Cold); + CHECK(verdict(tally(sample(0, 15))) == Verdict::Unknown && verdict(tally(sample(0, 16))) == Verdict::Warm); + CHECK(verdict(tally(sample(8, 8))) == Verdict::Cold && verdict(tally(sample(7, 9))) == Verdict::Warm); + CHECK(verdict(tally({})) == Verdict::Unknown); + { + const Tally t = tally(sample(5, 7, 3)); + CHECK(t.got == 12 && t.cold == 5 && t.answered == 5); + } + // the places: spread, distinct, the same for one seed + { + const std::vector a = spots(20000, kLook, 77), b = spots(20000, kLook, 77), c = spots(20000, kLook, 78); + CHECK(a == b && a != c && a.size() == (size_t)kLook && std::set(a.begin(), a.end()).size() == a.size()); + CHECK(spots(0, kLook, 1).empty() && spots(30, kLook, 5).size() == 30 && spots(1, kLook, 9) == std::vector{0}); + } + // the record + const std::string rec = dir + "/notes.built.check"; + CHECK(!state_read(rec).found); + CHECK(state_write(rec, 0x0123456789ABCDEFull, 2) && state_read(rec).found && state_read(rec).stamp == 0x0123456789ABCDEFull && + state_read(rec).strikes == 2); + { + FILE* f = fopen(rec.c_str(), "wb"); + fputs("predict-check 0000000000000001 0\n", f); // (another check's record is not this one) + fclose(f); + CHECK(!state_read(rec).found); + remove(rec.c_str()); + } + // starts in a row + const uint64_t A = 0xA, B = 0xB; + CHECK(start(rec, A, Verdict::Cold) == Do::Again && state_read(rec).strikes == 0); + CHECK(start(rec, A, Verdict::Cold) == Do::Stop && state_read(rec).strikes == 0); // (cold again right after: the draws) + CHECK(start(rec, A, Verdict::Unknown) == Do::Nothing && state_read(rec).found); + CHECK(start(rec, B, Verdict::Cold) == Do::Again && state_read(rec).stamp == B); // (another stamp begins anew) + CHECK(start(rec, B, Verdict::Warm) == Do::Nothing && !state_read(rec).found); // (warm forgets) + CHECK(start(rec, B, Verdict::Cold) == Do::Again); + CHECK(start(rec, B, Verdict::Warm, true) == Do::Again && start(rec, B, Verdict::Unknown, true) == Do::Again); // (mode 2) + // the counter + CHECK(!slow(39999) && slow(40000) && say(1, 0) && !say(10000, 1) && say(10001, 1)); + { + const std::vector s = sorted({5, 3, 5, 9, 1}); + CHECK(s == (std::vector{1, 3, 5, 9}) && has(s, 9) && !has(s, 4) && !has({}, 1)); + } + std::printf("ok\n"); + return 0; +} +""" + +FIXTURE = r""" +#include "built_check.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define CHECK(x) do { if (!(x)) { std::printf("line %d: %s (%s)\n", __LINE__, #x, last_log.c_str()); return 1; } } while (0) + +// ---- a stand-in driver: the caches hold graphics pipelines by note key and compute pipelines by program key +using VkResult = int; +constexpr VkResult VK_SUCCESS = 0, VK_PIPELINE_COMPILE_REQUIRED = 1000297000; +constexpr uint32_t VK_PIPELINE_CREATE_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT = 0x100; +using VkPipeline = uint64_t; +#define VK_NULL_HANDLE 0 +using VkPrimitiveTopology = int; +struct NvnState { uint64_t note; }; +struct PipeNote { uint64_t key, prog, variant; uint32_t topo; NvnState s; }; +struct Targets { int n; }; +struct Stage { int m; }; +struct Program { uint64_t key, variant; bool failed = false, compute = false; std::vector stages{Stage{}}; }; +struct VkComputePipelineCreateInfo { uint32_t flags; uint64_t key; }; +struct { struct { bool pipelineCreationCacheControl = true; } vk13; int device = 0; } dev; +std::set graphics_cache, compute_cache, refused; +std::map> programs; +std::map note_variant; +int compiles, asks, compute_asks, alive, wrong_variant, borrows, layouts, test_mode = 1, logs; +std::string dir, last_log; +VkPipeline build_graphics(const Program& p, const NvnState& s, const Targets&, VkPrimitiveTopology, int part, bool* cold) { + if (part != -1) return VK_NULL_HANDLE; + wrong_variant += note_variant[s.note] != p.variant; + if (refused.count(s.note)) return VK_NULL_HANDLE; + if (cold) { // (asked without compiling: the caches' answer) + asks++; + if (!graphics_cache.count(s.note)) return *cold = true, VK_NULL_HANDLE; + return alive++, s.note; + } + if (!graphics_cache.count(s.note)) { // (compiled into the caches: a compile's time) + std::this_thread::sleep_for(std::chrono::milliseconds(9)); + graphics_cache.insert(s.note), compiles++; + } + return alive++, s.note; +} +Targets note_targets(const PipeNote&) { return Targets{}; } +void ensure_layout(Program&) { layouts++; } +void program_cache_borrow(uint64_t key, const std::function& fn) { + borrows++; + auto it = programs.find(key); + if (it == programs.end()) return; + for (Program p : it->second) fn(p); // (a copy, let go after) +} +void vkDestroyPipeline(int, VkPipeline, const void*) { alive--; } +VkComputePipelineCreateInfo compute_create_info(const Program& p) { return {0, p.key}; } +struct PcacheUse { int cache = 1; }; +PcacheUse pcache_use() { return {}; } +VkResult cached_compute(int, const VkComputePipelineCreateInfo* ci, VkPipeline* pl) { + compute_asks++; + if (!(ci->flags & VK_PIPELINE_CREATE_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT)) return compiles++, *pl = ci->key, alive++, VK_SUCCESS; + if (!compute_cache.count(ci->key)) return VK_PIPELINE_COMPILE_REQUIRED; + return *pl = ci->key, alive++, VK_SUCCESS; +} +std::string pipe_notes_path() { return dir + "/notes.bin"; } +bool prepare_stopped() { return false; } +void logf(const char* fmt, ...) { + char line[2048]; + va_list a; + va_start(a, fmt); + vsnprintf(line, sizeof line, fmt, a); + va_end(a); + last_log = line, logs++; +} +int built_check_mode() { return test_mode; } +int built_check_next = 0; +std::atomic*> built_play_keys{nullptr}; +std::atomic st_built_slow{0}, st_built_slow_us{0}, st_built_missed{0}; +/* CUT */ + +uint64_t key_of(const PipeNote& n) { return n.key; } +std::vector> notes; +std::unordered_set built; +std::vector listed; +constexpr uint64_t kComputesKey = 0xC0FFEE, kStamp = 0x5157; +static bool run(bool computes_built = true) { return built_check_cold(notes, built, key_of, computes_built, listed, kStamp); } +static void fill_caches() { + graphics_cache.clear(), compute_cache.clear(); + for (const auto& n : notes) graphics_cache.insert(n->key); + compute_cache.insert(listed.begin(), listed.end()); +} +static bool has(const std::string& s) { return last_log.find(s) != std::string::npos; } + +int main(int argc, char** argv) { + if (argc < 2) return 2; + dir = argv[1]; + const std::string state = dir + "/notes.bin.built.check"; + for (uint64_t k = 1; k <= 40; k++) { + const uint64_t variant = k % 5 == 0 ? 2 : 1; // (some notes are of the program's second variant) + notes.push_back(std::make_unique(PipeNote{k, 1000 + k, variant, 3, NvnState{k}})); + note_variant[k] = variant; + programs[1000 + k] = {Program{1000 + k, 1}, Program{1000 + k, 2}}; + built.insert(k); + } + built.insert(kComputesKey); + for (uint64_t k = 500; k < 520; k++) { + listed.push_back(k); + Program c{k, 1}; + c.compute = true; + programs[k] = {c}; + } + // warm caches: nothing compiled, every pipeline let go, the caches as they were + fill_caches(); + const std::set had_graphics = graphics_cache; + CHECK(!run() && asks == 24 && compute_asks == 6 && compiles == 0 && alive == 0 && wrong_variant == 0 && has("have what it lists")); + CHECK(graphics_cache == had_graphics && !fopen(state.c_str(), "rb")); + // the caches lost them: cold, and still nothing compiled by the check + graphics_cache.clear(), compute_cache.clear(); + asks = compute_asks = 0; + CHECK(run() && asks == 24 && compute_asks == 6 && compiles == 0 && alive == 0 && graphics_cache.empty() && has("built again now")); + built_check_made(kStamp); // (precompile_run's, once everything was built and recorded again) + CHECK(built_check::state_read(state).found && built_check::state_read(state).stamp == kStamp); + // cold again right after: left to the draws, with the hint + CHECK(!run() && has("Shader Cache Size") && built_check::state_read(state).found); + // warm again: the strike forgotten + fill_caches(); + CHECK(!run() && !built_check::state_read(state).found); + // the threshold: the first 24 recorded notes are asked (40 notes, under kLook); computes left out here + graphics_cache.clear(); + for (uint64_t k = 13; k <= 40; k++) graphics_cache.insert(k); // (1-12 lost: 12 of 24) + CHECK(run(false) && has("12 of 24")); + graphics_cache.insert(12); // (11 of 24) + CHECK(!run(false) && has("11 of 24")); + // too few answers: 9 refused leave 15, 8 leave 16 + graphics_cache.clear(); + for (uint64_t k = 1; k <= 9; k++) refused.insert(k); + CHECK(!run(false) && has("too few")); + refused.erase(9); + CHECK(run(false)); + refused.clear(); + // without the flag: built twice and timed; the cold ones compiled, exactly the recorded notes asked, nothing else + dev.vk13.pipelineCreationCacheControl = false; + compute_asks = 0, compiles = 0; + CHECK(run() && compiles == 24 && compute_asks == 0 && alive == 0 && has("built twice")); + for (uint64_t k = 1; k <= 24; k++) CHECK(graphics_cache.count(k)); + CHECK(graphics_cache.size() == 24); + fill_caches(); + compiles = 0; + CHECK(!run() && compiles == 0 && alive == 0); + dev.vk13.pipelineCreationCacheControl = true; + // the switch at 0, or no record: nothing asked; at 2: cold with warm caches + asks = compute_asks = borrows = 0; + test_mode = 0; + CHECK(!run() && asks == 0 && compute_asks == 0 && borrows == 0); + test_mode = 1; + { + const std::unordered_set none; + CHECK(!built_check_cold(notes, none, key_of, true, listed, kStamp) && asks == 0 && borrows == 0); + } + test_mode = 2; + CHECK(run() && asks == 24 && has("RX_VK_BUILT_CHECK=2")); + remove(state.c_str()); + // the counter: nothing until published, nor with the switch at 0; then the record's keys and its compute programs + test_mode = 1; + built_play_build(3, 50000, false); + CHECK(st_built_slow == 0); + test_mode = 0; + built_play_publish(built, kComputesKey, listed); + CHECK(built_play_keys.load() == nullptr); + test_mode = 1; + built_play_publish(built, kComputesKey, listed); + CHECK(built_play_keys.load() != nullptr && built_check::has(*built_play_keys.load(), 507) && built_check::has(*built_play_keys.load(), 3)); + logs = 0; + built_play_build(3, 50000, false); + built_play_build(4, 60000, false); + built_play_build(999, 90000, false); // (not in the record) + built_play_build(507, 0, true); + built_play_build(888, 0, true); // (not listed) + CHECK(st_built_slow == 2 && st_built_slow_us == 110000 && st_built_missed == 1 && logs == 1); // (one line in 10 s) + { // (a record without the compute key lists no compute program) + std::unordered_set graphics_only(built); + graphics_only.erase(kComputesKey); + const std::vector* was = built_play_keys.exchange(nullptr); + built_play_publish(graphics_only, kComputesKey, listed); + CHECK(built_play_keys.load() && !built_check::has(*built_play_keys.load(), 507) && built_check::has(*built_play_keys.load(), 3)); + delete was; + } + std::printf("ok\n"); + return 0; +} +""" + + +def block(text, marker): + start = text.index(marker) + at = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[at] == "{") - (text[at] == "}") + at += 1 + return text[start:at] + + +def need(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def ordered(text, parts, why): + last = 0 + for p in parts: + k = text.find(p, last) + need(k >= 0, why + " (missing, or out of order: " + p + ")") + last = k + len(p) + + +def check_source(draw): + need('#include "built_check.h"' in draw, "vk_draw.cpp does not include built_check.h") + need('static const int mode = built_check::mode(getenv("RX_VK_BUILT_CHECK"));' in block(draw, "int built_check_mode() {"), + "the switch is not RX_VK_BUILT_CHECK, read once") + pre = block(draw, "void precompile_run(void (*progress)(size_t done, size_t total)) {") + ordered(pre, ["const std::vector computes_listed = computes_in_order();", "bool rebuilt = false;", "if (light) {", + "built = built_read(built_stamp);", "computes_built = built.count(computes_key) != 0;", + "if (!built.empty() && built_check_cold(notes, built, built_key, computes_built, computes_listed, built_stamp))", + "built.clear(), computes_built = false, rebuilt = true;", "std::erase_if(notes,", + "const std::unordered_set used = computes_built ?", "pipeline_cache_save(true);", + "built_write(built_stamp, built);", "if (rebuilt) built_check_made(built_stamp);", + "if (light && !prepare_stopped()) built_play_publish(built, computes_key, computes_listed);"], + "precompile_run does not ask before the erase, forget the record when cold, record the strikes after the record, " + "or publish it for the counter") + need(pre.count("built_check_cold(") == 1, "precompile_run asks more than once") + check = block(draw, "bool built_check_cold(") + ordered(check, ["if (!mode || built.empty()) return false;", "const bool ask = dev.vk13.pipelineCreationCacheControl;", + "built_check::spots(recorded.size(), (size_t)built_check::kLook, seed)", "program_cache_borrow(n.prog,", + "build_graphics(p, n.s, note_targets(n), (VkPrimitiveTopology)n.topo, -1, ask ? &cold : nullptr);", + "if (computes_built && ask)", "ci.flags |= VK_PIPELINE_CREATE_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT;", + "cached_compute(pcache_use().cache, &ci, &pl);", 'pipe_notes_path() + ".built.check"', + "built_check::decide(v, had, stamp, mode == 2, &built_check_next);", "return true;", "return false;"], + "built_check_cold does not ask our caches as a draw and a dispatch ask, or does not decide by the record of strikes") + need("driver_cache_only" not in check, "the check asks the driver's own cache, not ours (AMD's Windows driver keeps none)") + build = block(draw, "VkPipeline build_graphics(const Program& p, const NvnState& s, const Targets& at, VkPrimitiveTopology topo, int part, bool* cold) {") + ordered(build, ["const int64_t build_us =", "if (!driver_cache_only && pl && built_check::slow(build_us) && built_play_keys.load(std::memory_order_relaxed))", + "built_play_build(pipeline_key(p, s, at, topo), build_us, false);", "return pl;"], + "build_graphics does not count a slow play build of a recorded pipeline") + disp = block(draw, "void dispatch(VkCommandBuffer cmd, NvnState& s,") + ordered(disp, ["cached_compute(pcache_use().cache, &asked, &cached)", + "if (!cached && dev.vk13.pipelineCreationCacheControl && built_play_keys.load(std::memory_order_relaxed))", + "built_play_build(p.key, 0, true);", "auto build = [job, info] {"], + "the dispatch does not count a recorded compute program its ask missed") + docs = (ROOT / "docs/development.md").read_text(encoding="utf-8") + need("`RX_VK_BUILT_CHECK=0`, `=2`" in docs and "`gpu/vk/built_check.h`" in docs, + "docs/development.md: the switches table has no RX_VK_BUILT_CHECK row") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + draw = (ROOT / "gpu/vk/vk_draw.cpp").read_text(encoding="utf-8") + need("bool built_check_cold(" in draw, "vk_draw.cpp has no built_check_cold") + cut = "\n".join(block(draw, m) for m in ("bool built_check_cold(", "void built_check_made(uint64_t stamp) {", + "void built_play_publish(", "void built_play_build(uint64_t key, int64_t us, bool compute) {")) + env = {k: v for k, v in os.environ.items() if not k.startswith("RX_VK_")} + with tempfile.TemporaryDirectory(prefix="built-check-") as tmp: + tmp = Path(tmp) + for name, text in (("rules", RULES), ("cut", FIXTURE.replace("/* CUT */", cut))): + src, exe = tmp / (name + ".cpp"), tmp / (name + ".exe") + src.write_text(text, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-Wno-unused-parameter", "-I" + str(ROOT / "gpu/vk"), + str(src), "-o", str(exe)], check=True) + work = tmp / ("work-" + name) + work.mkdir() + r = subprocess.run([str(exe), str(work).replace("\\", "/")], capture_output=True, text=True, timeout=120, env=env) + need(r.returncode == 0 and r.stdout.strip() == "ok", "the built check (%s): %s" % (name, r.stdout.strip() or r.returncode)) + check_source(draw) + print("PASS: the light precompile's record is asked of the caches before it is trusted; lost, everything is built again " + "before the title, then left to the draws if lost again; RX_VK_BUILT_CHECK=0 as before") + + +if __name__ == "__main__": + main() diff --git a/tests/test_cap_free_async.py b/tests/test_cap_free_async.py new file mode 100644 index 0000000..c0341d2 --- /dev/null +++ b/tests/test_cap_free_async.py @@ -0,0 +1,241 @@ +"""Retired capture buffers the spares cannot keep are destroyed on a thread of their own (gpu/vk/vk_memory.cpp cap_buf_free, +destroy_buffer_later, RX_VK_CAP_FREE_ASYNC). + +take_capture retires old captures on the replay's thread with pools_mu held, and cap_buf_free destroyed each one past the +spares' limits right there, under cap_free_mu too: freeing a 64-256 MB host-cached buffer makes the driver unmap and release +every page. The code from cap_free_async_on to cap_buf_free and cap_exit_note is cut out unchanged and compiled with stand-ins +(a destroy_buffer that notes the thread and whether cap_free_mu is held, cap_free_mu as a mutex that knows its owner). +Checked: a spare is kept and not destroyed; one past the limits is destroyed once, on another thread, not under cap_free_mu; +RX_VK_CAP_FREE_ASYNC=0 (and off, no) destroys where it was retired, under the lock, as before; after cap_exit_note the thread +destroys nothing more; no thread to be had: destroyed inline after the lock; 4 threads retiring 400 buffers: each destroyed +exactly once; the 600-take line counts what was handed over, and the switch has its docs row. + + python -I -B tests/test_cap_free_async.py [--cxx clang++] +""" +import argparse +import os +from pathlib import Path +import subprocess +import tempfile + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] + + +def block(text, marker, tail=""): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + if tail: + end = text.index(tail, end) + len(tail) + return text[start:end] + + +def fail(message): + raise SystemExit("FAIL: " + message) + + +FIXTURE = r'''#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "cap_spare.h" // (cap_buf_free asks its rules: MEM-33) + +static void logf(const char*, ...) {} // (the switches' log lines: not checked here) +#include "linux_pressure.h" // (MEM-141's swap hold in cap_buf_free: off, as it always is on Windows) +static bool memory_swap_hold() { return false; } +static void memory_swap_valve(int, uint64_t) {} +typedef uint32_t VkBufferUsageFlags; // (cap_buf_make, MEM-39, in the cut: never called here) +VkBufferUsageFlags cap_usage() { return 0; } +typedef struct VkBuffer_T* VkBuffer; +typedef struct VmaAllocation_T* VmaAllocation; +struct OwnedMutex { // (cap_free_mu: knows which thread holds it) + std::mutex m; + std::atomic owner{}; + void lock() { m.lock(); owner = std::this_thread::get_id(); } + void unlock() { owner = std::thread::id(); m.unlock(); } + bool mine() const { return owner.load() == std::this_thread::get_id(); } +}; +OwnedMutex cap_free_mu; +std::atomic cap_shutting_down{false}; +uint64_t cap_free_bytes = 0; +uint64_t cap_spare_max() { return ~0ull; } +bool create_buffer_cached(uint64_t, VkBufferUsageFlags, VkBuffer&, VmaAllocation&, void**, bool) { return false; } +bool create_buffer_uncached(uint64_t, VkBufferUsageFlags, VkBuffer&, VmaAllocation&, void**, bool) { return false; } +struct Destroyed { uintptr_t id; std::thread::id by; bool under_free_mu; }; +std::mutex dmu; +std::vector destroyed; +void destroy_buffer(VkBuffer b, VmaAllocation) { + std::lock_guard g(dmu); + destroyed.push_back({(uintptr_t)b, std::this_thread::get_id(), cap_free_mu.mine()}); +} +struct test_thread { // (std::thread, or none to be had when TEST_NO_THREAD is set) + template explicit test_thread(F f) { + if (getenv("TEST_NO_THREAD")) throw std::system_error(std::make_error_code(std::errc::resource_unavailable_try_again)); + t = std::thread(f); + } + void detach() { t.detach(); } + std::thread t; +}; + +@FREEBUF@ +std::vector cap_free; +@ENVOFF@ +@BLOCK@ +@EXIT@ + +static VkBuffer buf(uintptr_t i) { return (VkBuffer)(i); } +static size_t count_of(uintptr_t id) { + std::lock_guard g(dmu); + size_t n = 0; + for (auto& d : destroyed) n += d.id == id; + return n; +} +static bool wait_for(uintptr_t id, size_t n = 1) { + for (int i = 0; i < 2000; i++) { + if (count_of(id) >= n) return true; + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + return false; +} +static void fill_spares() { // (12 spares: the next one is past the limits) + for (int i = 0; i < 12; i++) cap_buf_free(buf(1000 + i), nullptr, (void*)1, 1ull << 20); +} +static int bad(const char* why) { printf("BAD %s\n", why); return 1; } + +int main(int argc, char** argv) { + const std::string k = argc > 1 ? argv[1] : ""; + const auto me = std::this_thread::get_id(); + if (k == "keep") { + cap_buf_free(buf(1), nullptr, (void*)1, 32ull << 20); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + if (cap_free.size() != 1 || count_of(1)) return bad("a spare was not kept, or was destroyed"); + } else if (k == "past" || k == "nothread" || k == "off") { + fill_spares(); + cap_buf_free(buf(7), nullptr, (void*)1, 32ull << 20); + const size_t right_after = count_of(7); + if (!wait_for(7)) return bad("never destroyed"); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + if (count_of(7) != 1) return bad("destroyed more than once"); + Destroyed d{}; + { std::lock_guard g(dmu); for (auto& x : destroyed) if (x.id == 7) d = x; } + if (cap_free.size() != 12) return bad("the spares changed"); + if (k == "past") { + if (right_after) return bad("destroyed inside cap_buf_free"); + if (d.by == me) return bad("destroyed on the retiring thread"); + if (d.under_free_mu) return bad("destroyed under cap_free_mu"); + } else if (k == "nothread") { + if (d.by != me) return bad("no thread: not destroyed inline"); + if (d.under_free_mu) return bad("no thread: destroyed under cap_free_mu"); + } else { + if (d.by != me || !d.under_free_mu) return bad("switched off: not destroyed under the lock as before"); + } + } else if (k == "exit") { + fill_spares(); + cap_buf_free(buf(7), nullptr, (void*)1, 32ull << 20); // (the thread started) + if (!wait_for(7)) return bad("never destroyed"); + cap_exit_note(); + destroy_buffer_later(buf(8), nullptr, 1ull << 20); // (queued after the flag: the thread leaves it) + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + if (count_of(8)) return bad("destroyed after cap_exit_note"); + cap_buf_free(buf(9), nullptr, (void*)1, 32ull << 20, true); // (a worker after the flag: nothing, as before) + if (count_of(9) || cap_free.size() != 12) return bad("a worker's buffer touched after cap_exit_note"); + } else if (k == "many") { + fill_spares(); + std::vector ts; + for (int t = 0; t < 4; t++) + ts.emplace_back([t] { for (int i = 0; i < 100; i++) cap_buf_free(buf(100000 + t * 1000 + i), nullptr, (void*)1, 64ull << 20); }); + for (auto& t : ts) t.join(); + for (int t = 0; t < 4; t++) + for (int i = 0; i < 100; i++) + if (!wait_for(100000 + t * 1000 + i) || count_of(100000 + t * 1000 + i) != 1) return bad("not destroyed exactly once"); + std::lock_guard g(dmu); + for (auto& d : destroyed) if (d.under_free_mu) return bad("destroyed under cap_free_mu"); + if (cap_destroyed_later.load() != 400 || cap_destroyed_later_bytes.load() != 400ll * (64 << 20)) + return bad("the handed-over count is not 400 buffers of 64 MB"); + } else { + return bad("no such case"); + } + printf("OK %s\n", k.c_str()); + fflush(stdout); + _Exit(0); // (the detached thread is left, as at the process's exit) +} +''' + + +def fixture(mem): + start = mem.index("bool cap_free_async_on() {") + end_marker = "void cap_buf_free(VkBuffer buffer, VmaAllocation alloc, void* map, uint64_t size, bool from_worker = false) {" + code = mem[start:mem.index(end_marker)] + block(mem, end_marker) + code = code.replace("std::thread(", "test_thread(") + if "test_thread(" not in code: + raise ValueError("destroy_buffer_later starts no thread") + return (FIXTURE.replace("@FREEBUF@", block(mem, "struct FreeBuf {", ";")) + .replace("@ENVOFF@", block(mem, "bool env_value_off(const char* v) {")) + .replace("@BLOCK@", code).replace("@EXIT@", block(mem, "void cap_exit_note() {"))) + + +def check_source(mem, dev): + take = block(mem, "bool take_capture(") + if "cap_buf_retire(oldest);" not in take: + fail("take_capture no longer retires through cap_buf_retire") + line = mem[mem.index("vk: a capture taken (ms): all"):] + line = line[:line.index("logf(", 1) if "logf(" in line[1:] else len(line)] + if "destroyed " not in line or "cap_destroyed_later.exchange(0" not in line or "cap_destroyed_later_bytes.exchange(0" not in line: + fail("the 600-take line does not count the buffers handed to the destroying thread") + if "`RX_VK_CAP_FREE_ASYNC=0`" not in dev: + fail("docs/development.md has no RX_VK_CAP_FREE_ASYNC row") + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--cxx", default="g++") + a = p.parse_args() + mem = (ROOT / "gpu" / "vk" / "vk_memory.cpp").read_text(encoding="utf-8") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + try: + text = fixture(mem) + check_source(mem, dev) + except ValueError as e: + fail("a text this test reads moved, or is not there (%s)" % e) + flags = subprocess.BELOW_NORMAL_PRIORITY_CLASS if os.name == "nt" else 0 + problems = [] + with tempfile.TemporaryDirectory(prefix="cap-free-async-") as tmp: + cpp = Path(tmp) / "fixture.cpp" + exe = Path(tmp) / ("fixture" + (".exe" if os.name == "nt" else "")) + cpp.write_text(text, encoding="utf-8") + built = subprocess.run([a.cxx, "-std=c++20", "-O1", "-pthread", "-Wall", "-Wno-unused-function", "-I" + str(ROOT / "gpu" / "vk"), str(cpp), "-o", str(exe)], + capture_output=True, text=True, creationflags=flags) + if built.returncode: + print(built.stdout + built.stderr) + fail("the fixture did not build") + base = {k: v for k, v in os.environ.items() if k not in ("RX_VK_CAP_FREE_ASYNC", "TEST_NO_THREAD", "RX_VK_CAPTURE_KEEP_OLD")} + cases = [("keep", {}), ("past", {}), ("past", {"RX_VK_CAP_FREE_ASYNC": "1"}), ("off", {"RX_VK_CAP_FREE_ASYNC": "0"}), + ("off", {"RX_VK_CAP_FREE_ASYNC": "off"}), ("off", {"RX_VK_CAP_FREE_ASYNC": "No"}), ("exit", {}), + ("nothread", {"TEST_NO_THREAD": "1"}), ("many", {})] + for case, extra in cases: + done = subprocess.run([str(exe), case], env={**base, **extra}, capture_output=True, text=True, timeout=60, + creationflags=flags) + if done.returncode or done.stdout.strip() != "OK " + case: + problems.append("%s %s: %s" % (case, extra, (done.stdout + done.stderr).strip())) + if problems: + fail("\n ".join([""] + problems)) + print("PASS: retired capture buffers past the spares destroyed once on a thread of their own, not under cap_free_mu; " + "=0 as before; nothing after the exit note; inline without a thread (%d cases)" % len(cases)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_capture_behind.py b/tests/test_capture_behind.py index 864fff1..9ce5a1e 100644 --- a/tests/test_capture_behind.py +++ b/tests/test_capture_behind.py @@ -190,6 +190,7 @@ def production_fixture(mem): #include #include #include "capture_behind.h" +#include "cpu_count.h" // ((Triforce, 11-22, OS-49) copy_helper_main counts by cpu_count() namespace capture_behind = mkwvk::capture_behind; #define CHECK(x) do { if (!(x)) { std::fprintf(stderr, "FAIL:%d %s\n", __LINE__, #x); std::fflush(stderr); std::_Exit(1); } } while (0) typedef size_t SIZE_T; @@ -391,7 +392,7 @@ def check_source(mem, replay, texture, dev): check("if (behind && behind_on())" in make, "capture_make: the copies go behind without the caller's leave or the switch") for a, b in (("behind_now.store(b.get()", "behind_start(b);"), ("c->behind = b;", "behind_start(b);"), ("behind_start(b);", "copy_part(w);"), ("for (uint32_t i : c->late) na += !ahead[i], ahead[i] = 1;", "b->plan(ahead.data(), n);"), - ("g < std::prev(it)->second) ahead[(size_t)i] = 1", "b->plan(ahead.data(), n);")): + ("if (!ahead[(size_t)i] && in_texr.in(g)) ahead[(size_t)i] = 1", "b->plan(ahead.data(), n);")): # ((Triforce, 11-22, MEM-42) the loop's walk) before(make, a, b, "capture_make: %s before %s" % (a, b)) check("if (!ahead[(size_t)i] || b->ok[(size_t)i]) c->copied.push_back(pages[i])" in make, "capture_make: the pages copied ahead that failed are counted as copied, or those behind are not") diff --git a/tests/test_capture_copy_helpers.py b/tests/test_capture_copy_helpers.py index 64b65af..e069d2c 100644 --- a/tests/test_capture_copy_helpers.py +++ b/tests/test_capture_copy_helpers.py @@ -68,6 +68,7 @@ def main(): #include #include #include "capture_behind.h" +#include "cpu_count.h" // ((Triforce, 11-22, OS-49) copy_helper_main counts by cpu_count() namespace capture_behind = mkwvk::capture_behind; #define CHECK(x) do { if (!(x)) { std::fprintf(stderr, "FAIL:%d %s\n", __LINE__, #x); std::fflush(stderr); std::_Exit(1); } } while (0) typedef size_t SIZE_T; diff --git a/tests/test_capture_spare_reuse.py b/tests/test_capture_spare_reuse.py new file mode 100644 index 0000000..3037bba --- /dev/null +++ b/tests/test_capture_spare_reuse.py @@ -0,0 +1,440 @@ +"""A capture's buffer: made with the spares' lock let go, a load's big spare left alone, the spare a load took made again +(gpu/vk/cap_spare.h, vk_memory.cpp cap_buf_get, cap_prewarm_class, cap_spare_make and cap_buf_free; no game, no GPU). + +(Triforce, 11-22, the performance audit's MEM-33, after Source A's.) With no spare that fits, cap_buf_get made the buffer +on the game's submitting thread with cap_free_mu held, and the replay's take, holding pools_mu, waits on that mutex to +give a buffer back. Any spare that fit was taken, so a small capture could hold a load's 192 MB spare for the 24 +submissions it is kept; and a spare made ahead was made once. + +This compiles cap_spare.h and checks its rules against the code as it was on random inputs: pick() with any_size is the +old loop (the smallest that fits, the first of equals), keeps() with nothing reserved is the old limit, pick() with the +skip is the smallest fitting spare that is not a big one over twice the class, and stepped() stays between the need and +the class. + +Then it cuts cap_buf_free, cap_buf_size, cap_spare_make, cap_prewarm_class and cap_buf_get out of vk_memory.cpp and runs +them over stand-ins (the worker's thread is a list the test runs by hand): + - the first capture starts the workers and reserves their bytes; each worker's spare ends its reservation; + - a 1 MB capture leaves the 32, 64 and 128 MB spares alone and makes its own, with cap_free_mu NOT held; + - a capture of a 32 MB class takes the 32 MB spare, and no worker starts while a bigger spare is there; one of a 128 + MB class takes the 128 MB spare and a worker of 128 MB starts; + - the allocation fails: the big spare is taken after all; with no spare, the exact size without a memory object of + its own; a buffer made but not mapped is destroyed, not lost; + - what is being made counts: a buffer given back is not kept past the limit a worker's spare is about to fill, the + worker's spare is kept, no worker starts without room, and a worker that gets no buffer frees its reservation; + - spares capped at 32 MB: any spare is taken, as before; steps only with RX_VK_CAP_STEP_MB; + - RX_VK_CAP_SPARE_FIT=0: any fitting spare, no worker at a take, the buffer made with the lock held, nothing reserved. + +And reads the source: the switches, the 600-take count, the switches table. + + python -I -B tests/test_capture_spare_reuse.py [--cxx clang++] +""" +import argparse +import os +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + check(marker in text, "vk_memory.cpp has no " + marker) + start = text.index(marker) + end = text.index("{", start + len(marker) - 1) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def line(text, marker): + check(marker in text, "vk_memory.cpp has no " + marker) + start = text.index(marker) + return text[start:text.index("\n", start)] + + +HEAD = r""" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "cap_spare.h" +// ((Triforce, 11-22) MEM-141's swap hold, merged into cap_buf_free: off, as it always is on Windows +#include "linux_pressure.h" +static bool memory_swap_hold() { return false; } +static void memory_swap_valve(int, uint64_t) {} +typedef uint64_t VkBuffer; +typedef void* VmaAllocation; +typedef unsigned VkBufferUsageFlags; +#define VK_NULL_HANDLE 0 +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +static const uint64_t MB = 1ull << 20; + +struct FreeBuf { VkBuffer buffer; VmaAllocation alloc; void* map; uint64_t size; }; +std::vector cap_free; +std::atomic cap_shutting_down{false}; +std::mutex cap_free_mu; +uint64_t cap_free_bytes = 0; +uint64_t cap_reserved_bytes = 0; +size_t cap_reserved_n = 0; +std::atomic cap_made_n{0}, cap_made_us{0}; +std::atomic cap_prewarm_busy{0}; +std::atomic cap_prewarm_class_busy[4]{}; + +// the stand-ins +static uint64_t spare_max_v = ~0ull, step_v = 0; +static bool fit_v = true, prewarm_v = true; +uint64_t cap_spare_max() { return spare_max_v; } +bool cap_spare_fit_on() { return fit_v; } +uint64_t cap_step_mb() { return step_v; } +bool cap_prewarm_on() { return prewarm_v; } +bool cap_free_async_on() { return false; } +VkBufferUsageFlags cap_usage() { return 0; } +void cap_exit_note() {} +void logf(const char*, ...) {} +static std::vector destroyed; +void destroy_buffer(VkBuffer b, VmaAllocation) { destroyed.push_back(b); } +void destroy_buffer_later(VkBuffer b, VmaAllocation, uint64_t) { destroyed.push_back(b); } +struct Made { uint64_t size; bool own, lock_held; VkBuffer id; }; +static std::vector made; +static std::set fail_sizes, unmapped_sizes; +static bool fail_all = false; +static VkBuffer next_id = 100; +bool cap_buf_make(uint64_t size, VkBuffer& buffer, VmaAllocation& allocation, void** mapped, bool dedicated) { // (the capture buffers' maker) + const bool held = !cap_free_mu.try_lock(); // (is the spares' lock held by the caller, the only thread here) + if (!held) cap_free_mu.unlock(); + if (fail_all || fail_sizes.count(size)) { + made.push_back({size, dedicated, held, 0}); + buffer = VK_NULL_HANDLE, allocation = nullptr; + return false; + } + buffer = next_id++, allocation = (void*)&made; + made.push_back({size, dedicated, held, buffer}); + if (mapped) *mapped = unmapped_sizes.count(size) ? nullptr : (void*)&made; + return true; +} +struct Spawn { uint64_t bytes; int index; }; +static std::vector spawned; // (the workers' threads: run by hand) +""" + +TAIL = r""" +static void reset(uint64_t spare_max, bool fit, uint64_t step = 0) { + cap_free.clear(), cap_free_bytes = 0, cap_reserved_bytes = 0, cap_reserved_n = 0; + spawned.clear(), made.clear(), destroyed.clear(), fail_sizes.clear(), unmapped_sizes.clear(); + spare_max_v = spare_max, fit_v = fit, step_v = step, fail_all = false; + cap_prewarm_busy = 0; + for (auto& b : cap_prewarm_class_busy) b = 0; +} +static VkBuffer spare(uint64_t mb) { + cap_free.push_back({next_id, nullptr, (void*)&made, mb * MB}); + cap_free_bytes += mb * MB; + return next_id++; +} +static void run_workers() { + const std::vector s = spawned; + spawned.clear(); + for (const Spawn& w : s) cap_spare_make(w.bytes, w.index); +} +static bool has_spare(uint64_t mb) { + for (const FreeBuf& f : cap_free) + if (f.size == mb * MB) return true; + return false; +} +struct Got { bool ok; VkBuffer buffer; VmaAllocation alloc; void* map; uint64_t size; }; +static Got get(uint64_t need) { + Got g{false, 0, nullptr, nullptr, 0}; + g.ok = cap_buf_get(need, g.buffer, g.alloc, g.map, g.size); + return g; +} +static uint64_t rnd_state = 0x9E3779B97F4A7C15ull; +static uint64_t rnd() { + rnd_state ^= rnd_state << 13, rnd_state ^= rnd_state >> 7, rnd_state ^= rnd_state << 17; + return rnd_state; +} + +int main() { + // ---- the rules against the code as it was, on random inputs ---- + static const uint64_t kSizes[] = {1, 2, 4, 8, 16, 20, 32, 48, 64, 100, 128, 192, 256}; // (no spare is over 256 MB) + for (int t = 0; t < 20000; t++) { + std::vector sizes(rnd() % 13); + for (uint64_t& s : sizes) s = kSizes[rnd() % 13] * MB; + const uint64_t need = (rnd() % (260 * 256) + 1) * 4096; + uint64_t cls = 2 * MB; // (cap_buf_size without its switches) + while (cls < need && cls < 16 * MB) cls <<= 1; + if (cls < need) { + cls = (need + MB - 1) & ~(MB - 1); + for (uint64_t c : {32, 64, 128, 192, 256}) + if (c * MB >= need) { cls = c * MB; break; } + } + const auto size_of = [&](size_t i) { return sizes[i]; }; + size_t old_best = sizes.size(); // (the loop as it was) + for (size_t i = 0; i < sizes.size(); i++) + if (sizes[i] >= need && (old_best == sizes.size() || sizes[i] < sizes[old_best])) old_best = i; + CHECK(cap_spare::pick(sizes.size(), size_of, need, cls, true) == old_best); + size_t want = sizes.size(); // (with the skip: the smallest that fits and is not a big one over twice the class) + for (size_t i = 0; i < sizes.size(); i++) { + if (sizes[i] < need || (sizes[i] >= 32 * MB && sizes[i] > 2 * cls)) continue; + if (want == sizes.size() || sizes[i] < sizes[want]) want = i; + } + const size_t got = cap_spare::pick(sizes.size(), size_of, need, cls, false); + CHECK(got == want); + if (got != sizes.size()) CHECK(sizes[got] >= need && old_best != sizes.size() && sizes[got] >= sizes[old_best]); + if (cls >= 128 * MB) CHECK(got == old_best); // (nothing kept is over twice a class of 128 MB or more) + // the limit as it was, with nothing being made + const size_t n = rnd() % 14; + const uint64_t held = (rnd() % 900) * MB, size = kSizes[rnd() % 13] * MB; + static const uint64_t kMax[] = {~0ull, 32ull << 20, 12ull << 20, 512ull << 20, 2048ull << 20}; + const uint64_t spare_max = kMax[rnd() % 5]; + for (int keep_old = 0; keep_old < 2; keep_old++) { + const bool was = n < 12 && size <= (keep_old ? 64ull << 20 : 256ull << 20) && + held + size <= std::min(spare_max, keep_old ? ~0ull : 768ull << 20); + CHECK(cap_spare::keeps(n, 0, held, 0, size, spare_max, keep_old != 0) == was); + // and what is being made counts as if it were held already + const size_t workers = rnd() % 4; + const uint64_t reserved = (rnd() % 300) * MB; + const bool with = n + workers < 12 && size <= (keep_old ? 64ull << 20 : 256ull << 20) && + held + reserved + size <= std::min(spare_max, keep_old ? ~0ull : 768ull << 20); + CHECK(cap_spare::keeps(n, workers, held, reserved, size, spare_max, keep_old != 0) == with); + CHECK(cap_spare::room(held, reserved, size, spare_max) == (held + reserved + size <= std::min(spare_max, 768ull << 20))); + } + const uint64_t step = rnd() % 9; + for (int capped = 0; capped < 2; capped++) { + const uint64_t s = cap_spare::stepped(need, cls, step, capped != 0); + CHECK(s >= need && s <= cls); + if (!step || !capped || need <= 16 * MB || need > 32 * MB) CHECK(s == cls); + else CHECK(s == cls || (s % (step * MB) == 0 && s - need < step * MB)); + } + } + CHECK(cap_spare::skips(192 * MB, 32 * MB, false) && !cap_spare::skips(64 * MB, 32 * MB, false) && cap_spare::skips(32 * MB, 2 * MB, false)); + CHECK(!cap_spare::skips(192 * MB, 32 * MB, true) && !cap_spare::skips(16 * MB, 2 * MB, false) && !cap_spare::skips(256 * MB, 128 * MB, false)); + CHECK(cap_spare::stepped(17 * MB, 32 * MB, 4, true) == 20 * MB && cap_spare::stepped(17 * MB, 32 * MB, 0, true) == 32 * MB); + CHECK(cap_spare::stepped(17 * MB, 32 * MB, 4, false) == 32 * MB && cap_spare::stepped(31 * MB, 32 * MB, 4, true) == 32 * MB); + CHECK(cap_spare::stepped(16 * MB, 16 * MB, 4, true) == 16 * MB && cap_spare::stepped(40 * MB, 64 * MB, 4, true) == 64 * MB); + + // ---- the cut code ---- + { // the first capture: the workers start, their bytes reserved; the capture's own buffer is made with the lock let go + reset(~0ull, true); + const Got g = get(1 * MB); + CHECK(g.ok && g.size == 2 * MB && made.size() == 1 && made[0].size == 2 * MB && !made[0].own && !made[0].lock_held); + CHECK(spawned.size() == 3 && spawned[0].bytes == 32 * MB && spawned[1].bytes == 64 * MB && spawned[2].bytes == 128 * MB); + CHECK(cap_reserved_bytes == 224 * MB && cap_reserved_n == 3 && cap_made_n.load() == 1); + run_workers(); + CHECK(cap_reserved_bytes == 0 && cap_reserved_n == 0 && cap_free.size() == 3 && cap_free_bytes == 224 * MB); + CHECK(cap_prewarm_busy.load() == 0 && has_spare(32) && has_spare(64) && has_spare(128) && destroyed.empty()); + CHECK(made.size() == 4 && made[1].own && !made[1].lock_held); // (the workers' are not the submitting thread's count) + CHECK(cap_made_n.load() == 1); + + // a 1 MB capture leaves the three big spares alone + const Got small = get(1 * MB); + CHECK(small.ok && small.size == 2 * MB && cap_free.size() == 3 && made.size() == 5 && !made[4].lock_held && spawned.empty()); + // a capture of the 32 MB class takes the 32 MB spare; a bigger spare is still there: no worker + const Got mid = get(20 * MB); + CHECK(mid.ok && mid.size == 32 * MB && cap_free.size() == 2 && !has_spare(32) && made.size() == 5 && spawned.empty()); + // a capture of the 128 MB class takes the 128 MB spare: a worker of 128 MB starts, its bytes reserved + const Got big = get(100 * MB); + CHECK(big.ok && big.size == 128 * MB && cap_free.size() == 1 && made.size() == 5); + CHECK(spawned.size() == 1 && spawned[0].bytes == 128 * MB && spawned[0].index == 2 && cap_reserved_bytes == 128 * MB && cap_reserved_n == 1); + run_workers(); + CHECK(has_spare(128) && cap_reserved_bytes == 0 && cap_reserved_n == 0); + // a capture of the 64 MB class: the 64 MB spare, not the 128 MB one + const Got c64 = get(40 * MB); + CHECK(c64.ok && c64.size == 64 * MB && has_spare(128) && spawned.empty()); + } + { // the allocation fails: the big spare after all, and a worker for the one taken + reset(~0ull, true); + const VkBuffer s192 = spare(192); + fail_sizes.insert(32 * MB); + const Got g = get(20 * MB); + CHECK(g.ok && g.buffer == s192 && g.size == 192 * MB && cap_free.empty() && cap_free_bytes == 0); + CHECK(made.size() == 1 && made[0].size == 32 * MB && made[0].own && !made[0].lock_held); + CHECK(spawned.size() == 1 && spawned[0].bytes == 192 * MB && spawned[0].index == 3); + } + { // no spare either: the exact size, without a memory object of its own, the lock let go + reset(~0ull, true); + fail_sizes.insert(32 * MB); + const uint64_t need = 20 * MB + 4096; + const Got g = get(need); + CHECK(g.ok && g.size == need && made.size() == 2 && made[1].size == need && !made[1].own && !made[1].lock_held); + reset(~0ull, true); + fail_all = true; + const Got none = get(need); + CHECK(!none.ok && made.size() == 2); + } + { // made but not mapped: destroyed, then the exact size + reset(~0ull, true); + unmapped_sizes.insert(2 * MB); + const Got g = get(1 * MB + 4096); + CHECK(g.ok && g.size == 1 * MB + 4096 && made.size() == 2 && destroyed.size() == 1 && destroyed[0] == made[0].id && g.buffer == made[1].id); + } + { // what is being made counts with what is held + reset(256 * MB, true); + spare(64); + { + std::lock_guard lk(cap_free_mu); + cap_prewarm_class(128 * MB); + } + CHECK(spawned.size() == 1 && cap_reserved_bytes == 128 * MB && cap_reserved_n == 1); + cap_buf_free(900, nullptr, (void*)&made, 100 * MB); // (64 held + 128 being made + 100 > 256: not kept) + CHECK(destroyed.size() == 1 && destroyed[0] == 900 && cap_free.size() == 1); + cap_buf_free(901, nullptr, (void*)&made, 64 * MB); // (64 + 128 + 64 = 256: kept) + CHECK(destroyed.size() == 1 && cap_free.size() == 2 && cap_free_bytes == 128 * MB); + { + std::lock_guard lk(cap_free_mu); + cap_prewarm_class(32 * MB); // (no room: 128 held + 128 being made + 32 > 256. And a spare of 64 MB is there) + cap_prewarm_class(192 * MB); + } + CHECK(spawned.size() == 1); + run_workers(); // (the worker's 128 MB lands and is kept: its room was reserved) + CHECK(destroyed.size() == 1 && cap_free.size() == 3 && cap_free_bytes == 256 * MB && cap_reserved_bytes == 0 && cap_reserved_n == 0); + // twelve with the one being made + reset(~0ull, true); + for (int i = 0; i < 11; i++) spare(2); + { + std::lock_guard lk(cap_free_mu); + cap_prewarm_class(32 * MB); + } + CHECK(spawned.size() == 1 && cap_reserved_n == 1); + cap_buf_free(902, nullptr, (void*)&made, 2 * MB); + CHECK(destroyed.size() == 1 && cap_free.size() == 11); + run_workers(); + CHECK(destroyed.size() == 1 && cap_free.size() == 12 && has_spare(32)); + // a worker that gets no buffer frees what it reserved + reset(~0ull, true); + { + std::lock_guard lk(cap_free_mu); + cap_prewarm_class(64 * MB); + } + CHECK(cap_reserved_bytes == 64 * MB); + fail_all = true; + run_workers(); + CHECK(cap_reserved_bytes == 0 && cap_reserved_n == 0 && cap_free.empty() && cap_prewarm_busy.load() == 0); + } + { // spares capped at 32 MB: any spare is taken, as before; a 32 MB class capture's spare is made again + reset(32 * MB, true); + const VkBuffer s32 = spare(32); + const Got g = get(1 * MB); + CHECK(g.ok && g.buffer == s32 && made.empty() && spawned.empty()); + reset(32 * MB, true); + spare(32); + const Got h = get(20 * MB); + CHECK(h.ok && h.size == 32 * MB && made.empty() && spawned.size() == 1 && spawned[0].bytes == 32 * MB); + // steps only with the switch, and only where spares are capped + reset(32 * MB, true, 4); + const Got st = get(17 * MB); + CHECK(st.ok && st.size == 20 * MB && made.size() == 1 && made[0].own); + reset(32 * MB, true, 0); + CHECK(get(17 * MB).size == 32 * MB); + reset(~0ull, true, 4); + CHECK(get(17 * MB).size == 32 * MB); + reset(32 * MB, false, 4); + CHECK(get(17 * MB).size == 32 * MB); + } + { // RX_VK_CAP_SPARE_FIT=0: as before + reset(~0ull, false); + const VkBuffer s192 = spare(192); + const Got g = get(1 * MB); + CHECK(g.ok && g.buffer == s192 && made.empty() && spawned.empty() && cap_reserved_bytes == 0); + const Got n = get(1 * MB); + CHECK(n.ok && n.size == 2 * MB && made.size() == 1 && made[0].lock_held && !made[0].own); + const Got big = get(100 * MB); + CHECK(big.ok && big.size == 128 * MB && made.size() == 2 && made[1].lock_held && made[1].own && spawned.empty()); + reset(256 * MB, false); + spare(64); + { + std::lock_guard lk(cap_free_mu); + cap_prewarm_class(128 * MB); + } + CHECK(spawned.size() == 1 && cap_reserved_bytes == 0 && cap_reserved_n == 0); + cap_buf_free(903, nullptr, (void*)&made, 100 * MB); // (kept: nothing is counted for the worker) + CHECK(destroyed.empty() && cap_free.size() == 2); + run_workers(); // (and the worker's spare is destroyed as it lands: what the reservation is for) + CHECK(destroyed.size() == 1 && cap_free.size() == 2 && cap_reserved_bytes == 0); + } + if (fails) std::printf("capture spares: FAILED (%d checks)\n", fails); + return fails ? 1 : 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + mem = (VK / "vk_memory.cpp").read_text(encoding="utf-8") + check((VK / "cap_spare.h").exists(), "gpu/vk/cap_spare.h is missing") + check('#include "cap_spare.h"' in mem, "vk_memory.cpp does not include cap_spare.h") + + # the source + on = block(mem, "bool cap_spare_fit_on() {") + check('getenv("RX_VK_CAP_SPARE_FIT")' in on and "!(e && *e && atoi(e) == 0)" in on and "static const bool on" in on, + "RX_VK_CAP_SPARE_FIT is not on unless set to 0, or is read more than once") + step = block(mem, "uint64_t cap_step_mb() {") + check('getenv("RX_VK_CAP_STEP_MB")' in step and "e && atoi(e) > 0 ? (uint64_t)atoi(e) : 0" in step and "static const uint64_t v" in step, + "RX_VK_CAP_STEP_MB is not off unless set, or is read more than once") + for decl in ("uint64_t cap_reserved_bytes = 0;", "size_t cap_reserved_n = 0;", "std::atomic cap_made_n{0}, cap_made_us{0};"): + check(decl in mem, "vk_memory.cpp has no " + decl) + get = block(mem, "bool cap_buf_get(uint64_t need, VkBuffer& buffer, VmaAllocation& alloc, void*& map, uint64_t& size) {") + order = [get.find(s) for s in ( + "std::unique_lock lk(cap_free_mu);", + "if (take_spare(!fit || capped)) return map != nullptr;", + "size = fit ? cap_spare::stepped(need, cls, cap_step_mb(), capped) : cls;", + "if (!fit) {", + "lk.unlock();\n const bool made = cap_buf_make(size, buffer, alloc, &map, own);", + "if (made && map) return true;", + "lk.lock();\n if (take_spare(true)) return map != nullptr;\n lk.unlock();", + "return cap_buf_make(size, buffer, alloc, &map, false) && map;")] + check(min(order) >= 0 and order == sorted(order), + "cap_buf_get: the spare, then the buffer made with the lock let go, then the two fallbacks, are not there in that order") + check("if (fit && cls >= cap_spare::kBig) cap_prewarm_class(size);" in get, "a load's spare taken is not made again") + check("const bool capped = cap_spare_max() <= cap_spare::kBig;" in get, "the skip and the steps do not go by the spares' cap") + free = block(mem, "void cap_buf_free(VkBuffer buffer, VmaAllocation alloc, void* map, uint64_t size, bool from_worker = false) {") + # ((Triforce, 11-22) MEM-141 caps the spares while its swap hold is on (never on Windows): spare_cap(false, x) is x + check("cap_spare::keeps(cap_free.size(), cap_reserved_n, cap_free_bytes, cap_reserved_bytes, size," in free and + "linux_pressure::spare_cap(swap_hold_on, cap_spare_max()), keep_old)) {" in free, + "cap_buf_free does not count what is being made") + cls = block(mem, "void cap_prewarm_class(uint64_t size) {") + thread_line = "std::thread(cap_spare_make, bytes, index).detach();" + check(cls.count(thread_line) == 1, "cap_prewarm_class does not start its worker as the test expects") + check(cls.index("if (fit) cap_reserved_bytes += bytes, cap_reserved_n++;") < cls.index(thread_line), + "the worker's bytes are not reserved before its thread starts") + check("made on a submitting thread (600 takes)" in mem and "cap_made_n.load(std::memory_order_relaxed)" in mem, + "the 600-take line does not count the buffers made on a submitting thread") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`RX_VK_CAP_SPARE_FIT=0`" in dev and "`RX_VK_CAP_STEP_MB=`" in dev, "docs/development.md: the two switches are not in the switches table") + + cut = "\n".join([ + free, + block(mem, "uint64_t cap_buf_size(uint64_t need) {"), + line(mem, "int cap_prewarm_index(uint64_t size) {"), + block(mem, "void cap_spare_make(uint64_t size, int index) {"), + cls.replace(thread_line, "spawned.push_back({bytes, index});"), + get, + ]) + env = {k: v for k, v in os.environ.items() if not k.startswith("RX_VK_")} + with tempfile.TemporaryDirectory(prefix="cap-spare-") as tmp: + src, exe = Path(tmp) / "spare.cpp", Path(tmp) / "spare.exe" + src.write_text(HEAD + cut + TAIL, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-pthread", "-I" + str(VK), str(src), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True, timeout=120, env=env) + check(r.returncode == 0, "the capture spares as made:\n" + r.stdout + r.stderr) + print("PASS: a capture's buffer is made with the spares' lock let go, a load's spare is left alone and made again, and " + "what is being made counts with the spares") + + +if __name__ == "__main__": + main() diff --git a/tests/test_capture_tables.py b/tests/test_capture_tables.py new file mode 100644 index 0000000..7de4608 --- /dev/null +++ b/tests/test_capture_tables.py @@ -0,0 +1,203 @@ +"""The stored draws' tables keep their memory from frame to frame (gpu/vk/capture_tables.h, vk_temporal.cpp +capture_motion, temporal_capture_draw and take_captures; no game, no GPU). + +(Triforce, 11-21, the upscaler audit's UPS-02.) While DLSS, FSR 4 or FSR 3.1 upscales, every plain draw into the scene +stores its positions and is drawn again into the motion vectors. The count of a model's draws and the frame before's draws +by key were std::unordered_map, emptied and filled every frame: a node from the heap for every stored draw, about 1,400 of +each a frame in the owner's log, on the thread that records the frame. + +This compiles capture_tables.h alone and checks it: both ways (kept, and std::unordered_map with RX_VK_CAPTURE_TABLES=0) +answer the same over frames of draws with repeated keys, draws that come and go and a clear; the frame before's entries +are found until the next turn and no longer after it; and, with the heap's calls counted, a kept frame the size of the one +before takes nothing from the heap while the old way takes a node an entry. + +And reads the source: the chain's two tables are capture_tables' with the switch read once, capture_motion turns the table +before it fills it and looks the frame before's up through it, the draws are counted through next(), take_captures gives +the next frame's list its room, and the switch is in the switches table. + + python -I -B tests/test_capture_tables.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +PROGRAM = r""" +#include +#include +#include +#include +#include "capture_tables.h" +static long news = 0; +void* operator new(std::size_t n) { news++; if (void* p = std::malloc(n ? n : 1)) return p; throw std::bad_alloc(); } +void operator delete(void* p) noexcept { std::free(p); } +void operator delete(void* p, std::size_t) noexcept { std::free(p); } +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +using namespace capture_tables; +struct Draw { uint64_t address; uint32_t first, count; }; +static uint32_t rng = 12345; +static uint32_t next_random() { rng = rng * 1664525u + 1013904223u; return rng >> 8; } +int main() { + CHECK(kept(nullptr) && kept("") && kept("1") && kept("2") && !kept("0")); + CHECK(room(1444, true) == 1444 && room(1444, false) == 0 && room(0, true) == 0); + { // the n-th of a model, both ways the same; a clear starts every model at 0 again + Nth a(true), b(false); + for (int frame = 0; frame < 6; frame++) { + for (int i = 0; i < 3000; i++) { + const uint64_t model = (next_random() % 700) * 0x9E3779B97F4A7C15ull; + const int x = a.next(model), y = b.next(model); + CHECK(x == y); + } + CHECK(a.size() == b.size()); + a.clear(), b.clear(); + CHECK(a.size() == 0 && a.next(7) == 0 && b.next(7) == 0 && a.next(7) == 1); + a.clear(), b.clear(); + } + } + { // the frame before's entries, both ways the same: repeated keys (the last stands), draws that come and go + Before a(true), b(false); + for (int frame = 0; frame < 8; frame++) { + a.turn(), b.turn(); + const uint32_t n = 500 + next_random() % 1500; + for (uint32_t i = 0; i < n; i++) { + const uint64_t key = ((uint64_t)(next_random() % 2500) << 8) | (next_random() % 3); + const Draw d{0x1000ull * frame + i, i, n}; + a.put(key, d), b.put(key, d); + } + CHECK(a.size() == b.size()); + for (uint32_t k = 0; k < 2500; k++) + for (uint32_t nth = 0; nth < 3; nth++) { + const uint64_t key = ((uint64_t)k << 8) | nth; + const Draw* x = a.before(key); + const Draw* y = b.before(key); + CHECK(!x == !y); + if (x && y) CHECK(x->address == y->address && x->first == y->first && x->count == y->count); + if (frame == 0) CHECK(!x); // (nothing before the first frame) + } + } + // (a frame's entries are the frame before's only after the turn, and gone after the next one) + Before t(true); + t.turn(); + t.put(5, {50, 1, 2}); + t.put(5, {51, 1, 2}); // (the last with a key stands) + CHECK(t.size() == 1 && !t.before(5)); + t.turn(); + CHECK(t.before(5) && t.before(5)->address == 51 && !t.before(6)); + t.turn(); + CHECK(!t.before(5)); + t.put(9, {90, 0, 0}); + t.turn(); + t.clear(); + CHECK(!t.before(9) && t.size() == 0); + a.clear(), b.clear(); + a.turn(), b.turn(); + CHECK(!a.before(0) && !b.before(0)); + } + { // the heap: a kept frame the size of the one before takes nothing from it; the old way a node an entry + for (int keep = 1; keep >= 0; keep--) { + Nth nth(keep != 0); + Before before(keep != 0); + long steady = 0; + for (int frame = 0; frame < 12; frame++) { + const long from = news; + before.turn(); + for (uint32_t i = 0; i < 1444; i++) { + const uint64_t model = (uint64_t)(i / 2) * 0x100000001B3ull; + const int n = nth.next(model); + before.put(model << 8 | (uint64_t)n, {i, i, 1}); + if (frame) CHECK(before.before(model << 8 | (uint64_t)n) != nullptr); + } + nth.clear(); + if (frame >= 4) steady += news - from; + } + if (keep) CHECK(steady == 0); + else CHECK(steady >= 8 * (1444 + 722)); + } + } + { // an array far larger than what a frame puts in it is given back, so emptying it does not walk all of it + FlatMap m; + for (uint64_t i = 0; i < 40000; i++) m[i * 977] = 1; + const size_t large = m.capacity(); + empty_for_next(m); + CHECK(m.size() == 0 && m.capacity() == large); // (it held that many: kept for a frame as large) + for (uint64_t i = 0; i < 100; i++) m[i] = 1; + empty_for_next(m); // (100 entries in an array for 40,000: given back) + CHECK(m.size() == 0 && m.capacity() < large); + for (uint64_t i = 0; i < 1400; i++) m[i] = 1; + const size_t fits = m.capacity(); + empty_for_next(m); + CHECK(m.size() == 0 && m.capacity() == fits); + } + return fails ? 1 : 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + src = (VK / "vk_temporal.cpp").read_text(encoding="utf-8") + + check('#include "capture_tables.h"' in src, "vk_temporal.cpp does not use the tables") + check('const bool capture_tables_kept = capture_tables::kept(getenv("RX_VK_CAPTURE_TABLES"));' in src, + "the switch is not RX_VK_CAPTURE_TABLES, read once with the chain's state") + check("capture_tables::Nth captured_models{capture_tables_kept};" in src and + "capture_tables::Before captured_before{capture_tables_kept};" in src, + "the chain's two tables are not capture_tables' with the switch") + check("std::unordered_map captured_before" not in src and + "std::unordered_map captured_models" not in src, "a table is still a std::unordered_map of the chain's own") + motion = block(src, "int capture_motion(VkCommandBuffer cmd, const Camera& cam, const float* before, const float* reproject, uint32_t rw,") + at = [motion.index(part) for part in ( + "s.captured_before.turn();", + "const bool old_ok = history && s.captured_frame && s.captured_frame == to.last;", + "if (c.k.depth == cam.depth) s.captured_before.put(c.key, c);", + "if (caps.empty() || !make_capture_pipeline()) return 0;", + "const State::Captured* const was = old_ok ? s.captured_before.before(c.key) : nullptr;", + "const State::Captured& o = *was;")] + check(at == sorted(at), "capture_motion does not turn the table, fill it with this frame's draws and then look up the frame before's") + check("old.find(" not in motion and "std::unordered_map old" not in motion, + "capture_motion still moves the table into a map of its own") + draw = block(src, "void temporal_capture_draw(const CaptureDraw& k, CaptureSlot* out) {") + check("const int nth = s.captured_models.next(model >> 8);" in draw, "a stored draw is not counted through the kept table") + take = block(src, "std::vector take_captures() {") + check(take.index("std::vector c = std::move(s.captured);") < + take.index("s.captured.reserve(capture_tables::room(c.size(), s.capture_tables_kept));"), + "take_captures does not give the next frame's list its room") + drop = block(src, "void drop_all() {") + check("s.captured_models.clear(), s.captured_before.clear()" in drop, "the chain let go does not empty the tables") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`RX_VK_CAPTURE_TABLES=0`" in dev, "docs/development.md: RX_VK_CAPTURE_TABLES=0 is not in the switches table") + + with tempfile.TemporaryDirectory(prefix="capture-tables-") as tmp: + c, exe = Path(tmp) / "tables.cpp", Path(tmp) / "tables.exe" + c.write_text(PROGRAM, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(VK), str(c), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + check(r.returncode == 0, "the tables as made:\n" + r.stdout) + print("PASS: the stored draws' tables keep their memory from frame to frame, and answer as std::unordered_map did") + + +if __name__ == "__main__": + main() diff --git a/tests/test_chunk_default.py b/tests/test_chunk_default.py new file mode 100644 index 0000000..d8de6b6 --- /dev/null +++ b/tests/test_chunk_default.py @@ -0,0 +1,135 @@ +"""The size of a submission's parts: RX_VK_CHUNK, the lab's knob, the driver's default (gpu/vk/barrier_rules.h part_draws, +vk_replay.cpp part_draws and maybe_split; no game, no GPU). + +(Triforce, 11-22, the performance audit's SYNC-66, after Source B's.) A long submission goes to the GPU in parts of 120 draws +and dispatches on every card, read once into a static of maybe_split, so nothing could compare sizes inside a run. The size is +now RX_VK_CHUNK when set, else the RX_VK_LAB_AB knob chunk: in its phases, else the driver's default: 120 for every driver +until the lab picks one. + +Compiles a driver against barrier_rules.h and lab_ab.h and checks: + the default is 120 for NVIDIA, AMD, Intel and any other vendor, with RADV or not; + with no knob the size is what the old read gave, for 100,000 random texts of the variable (unset, 0, 120, negative, words, + any number) on random vendors: the default path splits where it split; + RX_VK_CHUNK beats the knob and the default; 0 stays 0 (never split); the knob beats the default only in its own phases + (lab_ab.h's value_of with the phase's knob set, another knob's phase, and no phase). +And reads vk_replay.cpp: maybe_split has no static size and asks part_draws() at each call, still returns for a size of 0 or +less and still counts ++part_work against it after the tail's test; part_draws reads the variable once and the knob at each +call; the switches table has the row and no longer says the in-pass split is on with NVIDIA's driver. + + python -I -B tests/test_chunk_default.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +DRIVER = r''' +#include "barrier_rules.h" +#include "lab_ab.h" +#include + +namespace br = mkwvk::barrier_rules; +namespace lab = mkwvk::lab_ab; +static int checks, fails; +#define CHECK(what, cond) do { checks++; if (!(cond)) fails++, printf("FAIL %s (line %d)\n", what, __LINE__); } while (0) + +int main() { + static const uint32_t kVendors[] = {0x10de, 0x1002, 0x8086, 0x5143, 0x13b5, 0, 0xffffffffu}; + for (const uint32_t v : kVendors) + for (int radv = 0; radv < 2; radv++) CHECK("120 for every driver until the lab picks", br::part_draws_default(v, radv != 0) == 120); + CHECK("the constant", br::kPartDraws == 120); + + // with no knob: the old read, whatever the variable says + uint64_t rng = 0xD1B54A32D192ED03ull; + const auto rnd = [&](uint64_t n) { rng ^= rng << 13, rng ^= rng >> 7, rng ^= rng << 17; return rng % n; }; + static const char* const kTexts[] = {nullptr, "", "0", "120", "480", "4096", "-5", "junk", "12x", " 64", "1", "999999999999"}; + for (int i = 0; i < 100000; i++) { + char num[32]; + const char* env = kTexts[rnd(sizeof kTexts / sizeof *kTexts)]; + if (rnd(3) == 0) snprintf(num, sizeof num, "%lld", (long long)rnd(5000) - 500), env = num; + const uint32_t vendor = kVendors[rnd(sizeof kVendors / sizeof *kVendors)]; + const int64_t now = br::part_draws(env != nullptr, env ? atoll(env) : 0, br::part_draws_default(vendor, rnd(2) != 0)); + CHECK("with no knob the size is the old read's", now == br::part_draws_before(env)); + if (fails) break; + } + CHECK("unset: 120", br::part_draws(false, 0, br::part_draws_default(0x10de, false)) == 120); + CHECK("RX_VK_CHUNK beats the knob", br::part_draws(true, 240, 480) == 240); + CHECK("RX_VK_CHUNK=0 stays 0 under a knob", br::part_draws(true, 0, 480) == 0); + CHECK("the knob beats the default", br::part_draws(false, 0, 480) == 480); + + // the knob through lab_ab.h, as vk_replay.cpp's part_draws asks it: only in its own phases + CHECK("no RX_VK_LAB_AB: the default", (lab::n_knobs = 0, lab::value_of("chunk", 120)) == 120); + lab::n_knobs = 2; + snprintf(lab::knobs[0].name, sizeof lab::knobs[0].name, "chunk"), lab::knobs[0].value = 480; + snprintf(lab::knobs[1].name, sizeof lab::knobs[1].name, "rblate"), lab::knobs[1].value = 1; + lab::active.store(0); + CHECK("a phase with no knob: the default", br::part_draws(false, 0, lab::value_of("chunk", 120)) == 120); + lab::active.store(1); + CHECK("the knob's phase: its number", br::part_draws(false, 0, lab::value_of("chunk", 120)) == 480); + CHECK("the knob's phase, the variable set: the variable", br::part_draws(true, 120, lab::value_of("chunk", 120)) == 120); + lab::active.store(2); + CHECK("another knob's phase: the default", br::part_draws(false, 0, lab::value_of("chunk", 120)) == 120); + + if (fails) printf("chunk default: FAILED (%d of %d checks)\n", fails, checks); + else printf("chunk default: all %d checks passed\n", checks); + return fails != 0; +} +''' + + +def check(ok, what): + if not ok: + raise SystemExit("FAIL: " + what) + + +def function(text, head): + at = text.index(head) + return text[at:text.index("\n}\n", at)] + + +def check_source(): + replay = (ROOT / "gpu/vk/vk_replay.cpp").read_text(encoding="utf-8") + split = function(replay, "void maybe_split(VkCommandBuffer& cmd) {") + check("static const int64_t every" not in replay, "maybe_split still reads the size once into a static") + check(split.startswith("void maybe_split(VkCommandBuffer& cmd) {\n const int64_t every = part_draws();\n"), + "maybe_split does not ask part_draws() at each call") + check('getenv("RX_VK_CHUNK")' not in split, "maybe_split reads RX_VK_CHUNK itself") + check("if (tail_on()) return;" in split and + "if (!rec_cur || every <= 0 || ++part_work < every || rec_parts + 1 >= kMaxParts || dump_on()) return;" in split and + split.index("if (tail_on()) return;") < split.index("++part_work < every"), + "maybe_split's test changed: a size of 0 or less must never split, and the count comes after the tail's test") + part = function(replay, "int64_t part_draws() {") + check('static const char* const env = getenv("RX_VK_CHUNK");' in part and "static const int64_t env_value = env ? atoll(env) : 0;" in part, + "part_draws does not read RX_VK_CHUNK once") + check("barrier_rules::part_draws_default(dev.props.vendorID, radv);" in part, "the default is not asked of the rule with the driver") + check(part.rstrip().endswith('return barrier_rules::part_draws(env != nullptr, env_value, lab_ab::value_of("chunk", by_driver));'), + "part_draws does not ask the knob at each call, behind the variable") + check(replay.count('getenv("RX_VK_CHUNK")') == 1, "RX_VK_CHUNK is read somewhere else") + dev = (ROOT / "docs/development.md").read_text(encoding="utf-8") + row = [line for line in dev.splitlines() if line.startswith("| `RX_VK_CHUNK=` |")] + check(len(row) == 1 and "**120 on every driver**" in row[0] and "`chunk:`" in row[0], "docs/development.md: no row for RX_VK_CHUNK with its default") + check("Unset: `pass` with NVIDIA's driver" not in dev, "docs/development.md still says the in-pass split is on with NVIDIA's driver") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + check_source() + with tempfile.TemporaryDirectory(prefix="chunk-default-") as tmp: + src = Path(tmp) / "driver.cpp" + src.write_text(DRIVER, encoding="utf-8") + exe = Path(tmp) / "driver.exe" + subprocess.run([args.cxx, "-std=c++20", "-Wall", "-Wextra", "-Werror", "-I", str(ROOT / "gpu/vk"), str(src), "-o", str(exe)], + check=True) + out = subprocess.run([str(exe)], capture_output=True, text=True) + print(out.stdout, end="") + check(out.returncode == 0, "the rule's checks failed") + print("PASS: a part's size: RX_VK_CHUNK, then the lab knob, then the driver's default, 120 for each") + + +if __name__ == "__main__": + main() diff --git a/tests/test_chunk_parts.py b/tests/test_chunk_parts.py new file mode 100644 index 0000000..fc17012 --- /dev/null +++ b/tests/test_chunk_parts.py @@ -0,0 +1,323 @@ +"""RX_COLD_OPT: the lifted chunks at -O2, each compiled as parts cut where a function begins (nx2/cold_parts.py), on +made-up chunks (game-free). + +(Triforce, 11-22, the performance audit's LIFT-17, after Source A's.) The build compiles the hot pieces at -O2 and every +other lifted chunk at -O1. RX_COLD_OPT=O2 (or a file that lists chunks) asks for -O2 on those too, and each such chunk is +then compiled as parts of RX_COLD_PART_MB, written under /split. + +The rules: + - RX_COLD_OPT unset, empty or O1 is nothing: the build's sources come back as the same list, no file takes other flags, + and nothing is written. O2 is every lifted chunk (code_NNN.c and the lift's own parts code_NNN_M.c), a file is the + chunks it lists; a hot piece, the hot file and the runtime's files never. Any other value, or a list with a name that + is not a chunk's, stops the build with a message. + - RX_COLD_PART_MB: 6 when unset, 0 for no cut; anything that is not a size stops the build. + - The flags of a -O2 chunk are the chunks' flags with -O2 where the level was and the part size as a define, so the size + is in every object's key; nothing else changes. + - The parts hold the chunk's functions once and in order; every part begins with the chunk's declarations and then + with a function's first line (or the note above it); no part is past the budget unless it is one function; a chunk + that fits the budget is one part, its own text. + - The key file: a second call with the same chunk and size writes no file; another size, another chunk text, a part + gone or a key gone writes them again, and parts the new cut does not make are removed. The key holds the size. + - The lift's files are not touched: the chunk's bytes and its folder's listing are the same after. + +The compile (clang; skipped without one): a made-up chunk of functions that call each other is built whole and as its +parts, both at -O2, and run on the same contexts: the same registers after every function. + +And reads the source: nx2/build.py asks cold_parts.expand for its sources and gives the parts cold_fl, the hot pieces +their own flags as before; nx2/make_game.py leaves nx2/cold_parts.py out of the lift key; the switches are in the table. + + python -I -B tests/test_chunk_parts.py [--cc clang] +""" +import argparse +import os +import re +import shutil +import subprocess +import sys +import tempfile + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(ROOT, "tests")) +sys.path.insert(0, os.path.join(ROOT, "nx2")) +import kit # noqa: E402,F401 (a64-kit on the path, for chunks) +import chunks # noqa: E402 +import cold_parts # noqa: E402 +import lifted_c # noqa: E402 + +RUN_ALL_ARGS = ["--cc", "clang"] + + +def check(what, ok): + if not ok: + raise SystemExit("FAIL: chunk parts: " + what) + + +def stops(fn): + try: + fn() + except SystemExit as e: + return bool(str(e)) + return False + + +def function(addr, size, callee=None, note=None): + head = "void f_%08x(Ctx *__restrict A64_NOESC c) {\n" % addr + line = "c->x[1] = c->x[1] * 3 + c->x[2] + %d;\n" % (addr & 0xff) + call = "f_%08x(c);\n" % callee if callee else "" + return ("/* %s */\n" % note if note else "") + head + line * max(1, (size - len(head) - len(call) - 2) // len(line)) + call + "}\n" + + +def made_up_chunk(n, giant=0): + """(the preamble, the functions): n small functions, each calling the one after it now and then; one giant among them""" + addrs = [0x80000000 + 16 * i for i in range(n)] + preamble = '#include "a64.h"\n' + "".join("void f_%08x(Ctx *A64_NOESC);\n" % a for a in addrs) + fns = [] + for i, a in enumerate(addrs): + size = giant if giant and i == n // 3 else 1500 + 37 * (i % 11) + fns.append(function(a, size, addrs[i + 1] if i % 4 == 0 and i + 1 < n else None, "a note" if i % 7 == 3 else None)) + return preamble, fns + + +def check_rules(tmp): + lvl = cold_parts.cold_level + check("unset, empty and O1 ask nothing", lvl({}) is None and lvl({"RX_COLD_OPT": ""}) is None and lvl({"RX_COLD_OPT": " O1 "}) is None) + check("O2 is every chunk", lvl({"RX_COLD_OPT": "O2"}) == "all") + check("another level stops the build", all(stops(lambda v=v: lvl({"RX_COLD_OPT": v})) for v in ("O3", "o2", "-O2", "1", "all"))) + listed = os.path.join(tmp, "cold.txt") + with open(listed, "w") as f: + f.write("# the next tier\ncode_003.c\n\ncode_012_1.c # the giant's part\n") + check("a file lists chunks", lvl({"RX_COLD_OPT": listed}) == frozenset(("code_003.c", "code_012_1.c"))) + for bad in ("code_hot_2.c", "code_hot.c", "a64_table.c", "code_3.c", "code_003.p0.c"): + with open(os.path.join(tmp, "bad.txt"), "w") as f: + f.write("code_001.c\n%s\n" % bad) + check("a list naming %s stops the build" % bad, stops(lambda: lvl({"RX_COLD_OPT": os.path.join(tmp, "bad.txt")}))) + o2 = cold_parts.chunk_opt_o2 + check("every chunk under O2", o2("code_000.c", "all") and o2("code_047.c", "all") and o2("code_012_3.c", "all")) + check("the hot pieces and the runtime's files never", not any(o2(n, "all") for n in ( + "code_hot.c", "code_hot_0.c", "code_hot_7.c", "a64_table.c", "a64_hooks.c", "hos_image.c", "a64_stubs.c", "code_000.p0.c", + "code_000.c.whole"))) + some = frozenset(("code_003.c",)) + check("only the listed chunks under a list", o2("code_003.c", some) and not o2("code_004.c", some) and not o2("code_hot_3.c", some)) + check("no chunk when nothing is asked", not o2("code_003.c", None)) + pb = cold_parts.part_bytes + check("6 MB when unset", pb({}) == 6 << 20 == pb({"RX_COLD_PART_MB": ""}) and cold_parts.PART_MB_DEFAULT == 6) + check("the size as given", pb({"RX_COLD_PART_MB": "8"}) == 8 << 20 and pb({"RX_COLD_PART_MB": "2.5"}) == int(2.5 * (1 << 20))) + check("0 is no cut", pb({"RX_COLD_PART_MB": "0"}) == 0) + check("what is not a size stops the build", all(stops(lambda v=v: pb({"RX_COLD_PART_MB": v})) for v in ("-1", "six", "1e9", "nan"))) + fl = ["-march=x86-64-v3", "-ffp-contract=off", "-Igen", "-O1", "-g0", "-mllvm", "-rotation-max-header-size=0", "-DA64_POLLING"] + cold = cold_parts.cold_flags(fl, "O1", {}) + check("the flags: -O2 where the level was, the part size, nothing else", + cold == [f if f != "-O1" else "-O2" for f in fl] + ["-DRX_COLD_PART_MB=6"] and fl.count("-O1") == 1) + check("the part size is in the flags", cold_parts.cold_flags(fl, "O1", {"RX_COLD_PART_MB": "8"}) != cold) + + +def check_cut(): + preamble, fns = made_up_chunk(400, giant=90000) + text = preamble + "".join(fns) + budget = 60000 + parts = cold_parts.chunk_parts(text, budget, chunks.fn_cost) + check("the chunk is cut", len(parts) > 5) + check("every part has the chunk's declarations", all(p.startswith(preamble) for p in parts)) + bodies = [p[len(preamble):] for p in parts] + check("the parts hold the chunk's functions once, in order", "".join(bodies) == "".join(fns)) + check("every part begins where a function begins", all(lifted_c.HEAD_NOTED.match(b) for b in bodies)) + check("a note stays with its function", not any(b.rstrip().endswith("*/") for b in bodies)) + for b in bodies: + marks = [m.start() for m in lifted_c.HEAD_NOTED.finditer(b)] + [len(b)] + cost = sum(chunks.fn_cost(y - x) for x, y in zip(marks, marks[1:])) + check("no part is past the budget unless it is one function", cost <= budget or len(marks) == 2) + giant = [b for b in bodies if len(b) > 80000] + check("the giant function is a part by itself", len(giant) == 1 and len(lifted_c.HEAD_NOTED.findall(giant[0])) == 1) + check("a chunk that fits is its own text", cold_parts.chunk_parts(text, 1 << 30, chunks.fn_cost) == [text]) + check("no cut at 0", cold_parts.chunk_parts(text, 0, chunks.fn_cost) == [text]) + one = preamble + fns[0] + check("a chunk of one function is its own text", cold_parts.chunk_parts(one, 10, chunks.fn_cost) == [one]) + check("the key holds the size and the text", len({cold_parts.parts_key(text, budget), cold_parts.parts_key(text, budget + 1), + cold_parts.parts_key(text + " ", budget)}) == 3) + return preamble, fns, text + + +def read(path): + with open(path, encoding="utf-8", newline="") as f: + return f.read() + + +def check_files(tmp, preamble, fns, text): + gen, split = os.path.join(tmp, "gen"), os.path.join(tmp, "build", "split") + os.makedirs(gen) + small_pre, small_fns = made_up_chunk(12) + names = {"code_000.c": text, "code_001.c": small_pre + "".join(small_fns), "code_002_1.c": text.replace("f_8", "f_9"), + "code_hot_0.c": text.replace("f_8", "f_a"), "a64_table.c": "int a64_table;\n"} + for name, body in names.items(): + with open(os.path.join(gen, name), "w", encoding="utf-8", newline="") as f: + f.write(body) + srcs = [os.path.join(gen, n) for n in sorted(names)] + listing = sorted(os.listdir(gen)) + + # nothing asked: the same list, nothing cold, nothing written + out, cold = cold_parts.expand(srcs, split, {}) + check("with RX_COLD_OPT unset the sources are the list given, with no cold file", out is srcs and cold == frozenset()) + check("with RX_COLD_OPT unset nothing is written", not os.path.exists(os.path.join(tmp, "build"))) + + env = {"RX_COLD_OPT": "O2", "RX_COLD_PART_MB": "0.05"} # (about 52 KB a part) + stats = {} + out, cold = cold_parts.expand(srcs, split, env, stats=stats) # (fn_cost: chunks.fn_cost, found by expand itself) + budget = cold_parts.part_bytes(env) + check("the hot piece and the runtime's file stay in the list, with their flags", all( + s in out and s not in cold for s in srcs if os.path.basename(s) in ("code_hot_0.c", "a64_table.c"))) + check("the chunks are replaced by their parts", not any(s in out for s in srcs if cold_parts.CHUNK.fullmatch(os.path.basename(s)))) + check("every part takes the cold flags, and nothing else does", cold == frozenset(s for s in out if os.path.dirname(s) == split)) + check("the order is the chunks' order, a chunk's parts together", + [os.path.basename(s).split(".p")[0] for s in out if s in cold] == sorted(os.path.basename(s).split(".p")[0] for s in out if s in cold)) + check("three chunks written", stats == {"written": 3}) + for chunk in ("code_000.c", "code_001.c", "code_002_1.c"): + mine = [s for s in out if os.path.basename(s).startswith(chunk[:-2] + ".p")] + check("%s: its parts are named after it, in order" % chunk, + [os.path.basename(s) for s in mine] == [cold_parts.part_name(chunk, i) for i in range(len(mine))]) + check("%s: its parts are the cut of its text" % chunk, + [read(s) for s in mine] == cold_parts.chunk_parts(names[chunk], budget, chunks.fn_cost)) + key = read(os.path.join(split, chunk + ".key")).split() + check("%s: the key file names the key and the parts" % chunk, + key == [cold_parts.parts_key(names[chunk], budget)] + [os.path.basename(s) for s in mine]) + check("the small chunk is one part, its own text", [read(s) for s in out if "code_001.p" in s] == [names["code_001.c"]]) + check("the lift's files are as they were", sorted(os.listdir(gen)) == listing and all( + read(os.path.join(gen, n)) == body for n, body in names.items())) + + # a second build: nothing written (every part given an old date first: a rewrite would change it) + for s in cold: + os.utime(s, (1000000000, 1000000000)) + stats = {} + out2, cold2 = cold_parts.expand(srcs, split, env, stats=stats) + check("a second build makes the same parts", out2 == out and cold2 == cold and stats == {"kept": 3}) + check("a second build writes no part", all(int(os.path.getmtime(s)) == 1000000000 for s in cold)) + + # another size: cut again, the parts the new cut does not make removed + before = len([s for s in out if "code_000.p" in s]) + stats = {} + out3, cold3 = cold_parts.expand(srcs, split, dict(env, RX_COLD_PART_MB="0.2"), stats=stats) + after = [s for s in out3 if "code_000.p" in s] + check("another part size cuts the chunks again", stats == {"written": 3} and 1 < len(after) < before) + check("the parts of the old cut are removed", sorted(f for f in os.listdir(split) if f.startswith("code_000.p")) == + sorted(os.path.basename(s) for s in after)) + check("the new parts hold the chunk once", "".join(read(s)[len(preamble):] for s in after) == "".join(fns)) + + # a changed chunk, a part gone, a key gone: that chunk alone is cut again + env = dict(env, RX_COLD_PART_MB="0.2") + with open(os.path.join(gen, "code_001.c"), "a", encoding="utf-8", newline="") as f: + f.write(function(0x80008000, 900)) + stats = {} + cold_parts.expand(srcs, split, env, stats=stats) + check("a changed chunk alone is cut again", stats == {"written": 1, "kept": 2}) + os.remove(after[1]) + stats = {} + cold_parts.expand(srcs, split, env, stats=stats) + check("a chunk with a part gone is cut again", stats == {"written": 1, "kept": 2} and os.path.isfile(after[1])) + os.remove(os.path.join(split, "code_002_1.c.key")) + stats = {} + cold_parts.expand(srcs, split, env, stats=stats) + check("a chunk with its key gone is cut again", stats == {"written": 1, "kept": 2}) + + # a list of chunks, and no cut + listed = os.path.join(tmp, "listed.txt") + with open(listed, "w") as f: + f.write("code_002_1.c\n") + out4, cold4 = cold_parts.expand(srcs, split, dict(env, RX_COLD_OPT=listed)) + check("a list: only its chunks are replaced", {os.path.basename(s).split(".p")[0] for s in cold4} == {"code_002_1"} and + os.path.join(gen, "code_000.c") in out4 and os.path.join(gen, "code_000.c") not in cold4) + out5, cold5 = cold_parts.expand(srcs, split, {"RX_COLD_OPT": "O2", "RX_COLD_PART_MB": "0"}) + check("no cut at 0: the chunks themselves take the cold flags", out5 == srcs and cold5 == frozenset( + s for s in srcs if cold_parts.CHUNK.fullmatch(os.path.basename(s)))) + return [s for s in after] + + +STAND_IN = """/* tests/test_chunk_parts.py: a stand-in for the guest model, enough for made-up functions */ +#include +typedef struct { uint64_t x[32]; } Ctx; +#define A64_NOESC +""" +MAIN = """#include +#include "a64.h" +%(decls)s +static void (*const fns[])(Ctx *) = {%(names)s}; +int main(void) { + Ctx c; + uint64_t sum = 0; + for (unsigned k = 0; k < sizeof fns / sizeof fns[0]; k++) { + for (int r = 0; r < 32; r++) c.x[r] = 0x9e3779b97f4a7c15ULL * (k + 1) + r; + fns[k](&c); + sum = sum * 31 + c.x[1]; + printf("%%u %%016llx\\n", k, (unsigned long long)c.x[1]); + } + printf("sum %%016llx\\n", (unsigned long long)sum); + return 0; +} +""" + + +def check_compile(tmp, cc, preamble, fns, text): + """The chunk whole and as parts, at -O2: the same results""" + work = os.path.join(tmp, "cc") + os.makedirs(work) + with open(os.path.join(work, "a64.h"), "w", newline="\n") as f: + f.write(STAND_IN) + names = lifted_c.HEAD.findall(text) + with open(os.path.join(work, "main.c"), "w", newline="\n") as f: + f.write(MAIN % {"decls": "".join("void %s(Ctx *);\n" % n for n in names), "names": ", ".join(names)}) + whole = os.path.join(work, "code_000.c") + with open(whole, "w", encoding="utf-8", newline="") as f: + f.write(text) + parts = cold_parts.write_parts(whole, os.path.join(work, "split"), 40000, chunks.fn_cost) + check("the chunk to compile is cut", len(parts) > 3) + import test_fpx_sticky as fs # (the build helpers the lifted-code tests use: the target and linker flags) + outs = [] + for label, srcs in (("whole", [whole]), ("parts", parts)): + exe = os.path.join(work, label + ".exe") + cmd = [cc] + fs.target_flags(cc, work) + ["-O2", "-g0", "-Wno-everything", "-I" + work, os.path.join(work, "main.c")] + srcs + ["-o", exe] + r = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + check("the chunk compiles %s:\n%s" % (label, r.stdout[-2000:]), r.returncode == 0) + outs.append(subprocess.run([exe], stdout=subprocess.PIPE, text=True, timeout=120).stdout) + check("the parts compute what the whole chunk computes", outs[0] == outs[1] and outs[0].count("\n") == len(names) + 1) + return len(names), len(parts) + + +def check_sources(): + with open(os.path.join(ROOT, "nx2", "build.py"), encoding="utf-8") as f: + build = f.read() + for line in (' srcs, cold = cold_parts.expand(srcs, os.path.join(out, "split"), os.environ, stats=cold_stats)\n', + " cold_fl = cold_parts.cold_flags(fl, a.opt, os.environ) if cold else fl", + ' hot_fl if os.path.basename(s).startswith("code_hot") else cold_fl if s in cold else fl,\n'): + check("nx2/build.py no longer has the line %r" % line.strip(), build.count(line) == 1) + at = [build.find(p) for p in (" srcs, cold = cold_parts.expand(", " srcs.sort(key=lambda s: -compile_cost(s, costs))", + " cold_fl = cold_parts.cold_flags(", " jobs = [(cc, s, os.path.join(obj_dir,")] + check("nx2/build.py expands its sources before it orders them and makes the jobs", min(at) >= 0 and at == sorted(at)) + with open(os.path.join(ROOT, "nx2", "make_game.py"), encoding="utf-8") as f: + make_game = f.read() + not_keyed = re.search(r"^LIFT_KEY_NOT = \((.*?)\) #", make_game, re.S | re.M) + check("nx2/make_game.py leaves nx2/cold_parts.py out of the lift key", bool(not_keyed) and '"cold_parts.py"' in not_keyed.group(1)) + for lift_file in ("lift.py", "split_hot.py", "chunks.py"): # (the lift reads none of it) + with open(os.path.join(ROOT, "nx2", lift_file), encoding="utf-8") as f: + check("nx2/%s names cold_parts" % lift_file, "cold_parts" not in f.read()) + with open(os.path.join(ROOT, "docs", "development.md"), encoding="utf-8") as f: + check("docs/development.md: RX_COLD_OPT and RX_COLD_PART_MB are not in the switches table", + "| `RX_COLD_OPT=O2` or `=`, `RX_COLD_PART_MB=` |" in f.read()) + + +def main(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--cc") + a = p.parse_args() + check_sources() + with tempfile.TemporaryDirectory(prefix="triforce-chunk-parts-") as tmp: + check_rules(tmp) + preamble, fns, text = check_cut() + check_files(tmp, preamble, fns, text) + cc = shutil.which(a.cc) if a.cc else shutil.which("clang") + if not cc: + print("PASS: the rules, the cut and the key files (the compile skipped: no clang)") + return + n, parts = check_compile(tmp, cc, preamble, fns, text) + print("PASS: the rules, the cut and the key files; %d made-up functions compute the same whole and in %d parts at -O2" % (n, parts)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_commit_limit.py b/tests/test_commit_limit.py new file mode 100644 index 0000000..b2eede2 --- /dev/null +++ b/tests/test_commit_limit.py @@ -0,0 +1,376 @@ +"""The waits at Windows' commit limit: the game's commits, the host's own allocations and the renderer's `new` +(host/host_alloc.h, host/mkw_glue.c commit_wait and hos_xmalloc, gpu/vk/new_wait.h, vk_main.cpp new_refused; no game). + +(Triforce, 11-22, the performance audit's OS-04, after Source A's and Source B's.) A refusal at the limit is tried again +every 50 ms for HOS_COMMIT_WAIT_MS (5000 unless set; 0: no wait). + +- host_alloc.h's numbers and loop; then commit_wait, cut out of mkw_glue.c over a VirtualAlloc that refuses as the case + says, a process a case (the switch is read once): refused twice more and then made (150 ms slept, one line); refused + for good (100 tries, 5 s, one line); another error (given up at once); the switch at 0 (no try, no sleep, no line), at + 200 (4 tries) and far too large (60 s at most); +- hos_xmalloc, hos_xcalloc and hos_xrealloc, cut out of mkw_glue.c over a C library that refuses as the case says: no + refusal costs nothing; refused three times, then made after 150 ms; refused for good: NULL after 5 s and the + out-of-memory notice, once however many fail; a size of 0 is not waited for; a refused realloc keeps its block and + its bytes; the switch at 0: NULL at once; +- new_wait.h's rule; then new_refused, cut out of vk_main.cpp and set as the new handler of the fixture, with the commit + left as the case says and real allocations that cannot be made: near the limit a `new` waits its 5 s and throws, the + same thread's next refusal throws at once (5 s a thread), another thread has its own; with 4 GiB or more left, or the + commit not known, it throws at once; a nothrow new in a NoWait scope answers null at once, and without the scope + waits like any other; the switch at 0: at once; +- the sources: the host's tables that end the game on NULL allocate through the three; every nothrow new of the + renderer is in a NoWait scope; mkwvk_init sets the handler; the switch is in the switches table. + + python -I -B tests/test_commit_limit.py [--cc clang] [--cxx clang++] +""" +import argparse +import os +import re +import subprocess +import tempfile +from pathlib import Path + +RUN_ALL_ARGS = ["--cc", "clang", "--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] +HOST = ROOT / "host" +VK = ROOT / "gpu" / "vk" + + +def fail(message): + raise SystemExit("FAIL: " + message) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +WINDOWS = r''' +#include +#include +#include +#include +#include +#include "host_alloc.h" +static int fails; +#define CHECK(x) do { if (!(x)) { printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +/* Windows, standing in */ +typedef size_t SIZE_T; +typedef long LONG; +typedef unsigned long DWORD; +typedef void *LPVOID; +typedef void *HANDLE; +#define WINAPI +#define MEM_COMMIT 0x1000 +#define PAGE_READWRITE 4 +#define ERROR_COMMITMENT_LIMIT 1455 +static int slept_ms, logs, boxes; +static char last_log[256]; +static void Sleep(DWORD ms) { slept_ms += (int)ms; } +static LONG InterlockedIncrement(volatile LONG *v) { return ++*v; } +static LONG InterlockedExchange(volatile LONG *v, LONG to) { const LONG was = *v; *v = to; return was; } +static HANDLE CreateThread(void *a, size_t b, DWORD (*fn)(LPVOID), void *c, DWORD d, void *e) { (void)a, (void)b, (void)fn, (void)c, (void)d, (void)e; boxes++; return NULL; } +static void CloseHandle(HANDLE h) { (void)h; } +static void hos_log(const char *fmt, ...) { + va_list ap; + va_start(ap, fmt); + vsnprintf(last_log, sizeof last_log, fmt, ap); + va_end(ap); + logs++; +} +''' + +COMMIT = WINDOWS + r''' +static int refuse, tries; +static DWORD error = ERROR_COMMITMENT_LIMIT; +static void *VirtualAlloc(void *p, SIZE_T n, int a, int b) { + (void)n, (void)a, (void)b; + tries++; + if (refuse > 0) { refuse--; return NULL; } + return p; +} +static DWORD GetLastError(void) { return error; } +@WAIT_MS@ +@COMMIT_WAIT@ +int main(int argc, char **argv) { + const char *c = argc > 1 ? argv[1] : ""; + static char page[4096]; + if (!strcmp(c, "made")) { /* the caller's own try was refused; two more are, the third is made */ + refuse = 2; + CHECK(commit_wait(page, 4096) == 1 && slept_ms == 150 && tries == 3 && logs == 1 && strstr(last_log, "made after 150 ms")); + refuse = 0; + CHECK(commit_wait(page, 4096) == 1 && slept_ms == 200 && tries == 4); /* (made at the first try again) */ + } else if (!strcmp(c, "never")) { /* refused for good */ + refuse = 1 << 30; + CHECK(commit_wait(page, 4096) == 0 && slept_ms == atoi(argv[2]) && tries == atoi(argv[2]) / 50 && logs == 1); + CHECK(strstr(last_log, "still refused after") != NULL); + } else if (!strcmp(c, "other")) { /* not the limit's refusal: given up at once */ + refuse = 1 << 30, error = 8; + CHECK(commit_wait(page, 4096) == 0 && slept_ms == 50 && tries == 1 && logs == 0); + } else if (!strcmp(c, "off")) { /* HOS_COMMIT_WAIT_MS=0 */ + refuse = 0; + CHECK(commit_wait(page, 4096) == 0 && slept_ms == 0 && tries == 0 && logs == 0); + } else { /* the numbers */ + CHECK(host_alloc_wait_ms(NULL) == 5000 && host_alloc_wait_ms("") == 5000); + CHECK(host_alloc_wait_ms("0") == 0 && host_alloc_wait_ms("-5") == 0 && host_alloc_wait_ms("x") == 0); + CHECK(host_alloc_wait_ms("200") == 200 && host_alloc_wait_ms("60000") == 60000 && host_alloc_wait_ms("999999") == 60000); + CHECK(host_alloc_tries(5000) == 100 && host_alloc_tries(200) == 4 && host_alloc_tries(49) == 0 && host_alloc_tries(0) == 0 && host_alloc_tries(-1) == 0); + CHECK(hos_commit_wait_ms() == 5000); + } + if (!fails) printf("PASS %s\n", c); + return fails != 0; +} +''' + +ALLOC = WINDOWS + r''' +/* the C library, refusing as the case says */ +static int refuse, asked; +static void *lib_malloc(size_t n) { asked++; if (refuse > 0 && n) { refuse--; return NULL; } return malloc(n); } +static void *lib_calloc(size_t c, size_t n) { asked++; if (refuse > 0 && c && n) { refuse--; return NULL; } return calloc(c, n); } +static void *lib_realloc(void *p, size_t n) { asked++; if (refuse > 0 && n) { refuse--; return NULL; } return realloc(p, n); } +#define malloc lib_malloc +#define calloc lib_calloc +#define realloc lib_realloc +@WAIT_MS@ +@ALLOC@ +#undef malloc +#undef calloc +#undef realloc +static void *count_try(void *ctx) { int *n = (int *)ctx; return ++*n >= 3 ? ctx : NULL; } +static int stepped; +static void count_sleep(int ms) { stepped += ms; } +int main(int argc, char **argv) { + const char *c = argc > 1 ? argv[1] : ""; + if (!strcmp(c, "off")) { /* HOS_COMMIT_WAIT_MS=0: NULL at once, and the notice */ + refuse = 1; + CHECK(hos_xmalloc(64) == NULL && slept_ms == 0 && asked == 1 && boxes == 1); + } else { + /* the loop alone: the third try gives it */ + int n = 0, waited = -1; + CHECK(host_alloc_retry(count_try, &n, 100, count_sleep, &waited) == &n && n == 3 && waited == 150 && stepped == 150); + n = -1000, stepped = 0; + CHECK(host_alloc_retry(count_try, &n, 4, count_sleep, &waited) == NULL && waited == 200 && stepped == 200); + CHECK(host_alloc_retry(count_try, &n, 0, count_sleep, &waited) == NULL && waited == 0); + /* nothing refused: nothing slept, nothing said */ + void *a = hos_xmalloc(100), *b = hos_xcalloc(10, 10), *r = hos_xrealloc(NULL, 50); + CHECK(a && b && r && slept_ms == 0 && logs == 0 && asked == 3); + memset(r, 0x5a, 50); + for (int i = 0; i < 100; i++) CHECK(((unsigned char *)b)[i] == 0); + /* refused three times (the first try and two more), then made */ + refuse = 3; + void *m = hos_xmalloc(4096); + CHECK(m && slept_ms == 150 && logs == 1 && strstr(last_log, "made after 150 ms") && boxes == 0); + refuse = 2, slept_ms = 0; + unsigned char *z = (unsigned char *)hos_xcalloc(8, 32); + CHECK(z && slept_ms == 100 && z[0] == 0 && z[255] == 0); + /* a refused realloc keeps its block and gives its bytes on once it is made */ + refuse = 4, slept_ms = 0; + unsigned char *g = (unsigned char *)hos_xrealloc(r, 5000); + CHECK(g && slept_ms == 200 && g[0] == 0x5a && g[49] == 0x5a); + /* a size of 0 is the C library's own answer, never waited for */ + slept_ms = 0, refuse = 0; + free(hos_xmalloc(0)), free(hos_xcalloc(0, 16)), free(hos_xcalloc(16, 0)); + CHECK(slept_ms == 0); + /* refused for good: NULL after the 5 s, the notice once however many fail */ + refuse = 1 << 30, logs = 0; + CHECK(hos_xmalloc(1 << 20) == NULL && slept_ms == 5000 && boxes == 1 && logs == 2 && strstr(last_log, "out of memory")); + slept_ms = 0; + CHECK(hos_xcalloc(4, 4) == NULL && hos_xrealloc(g, 9000) == NULL && slept_ms == 10000 && boxes == 1); + CHECK(g[0] == 0x5a); /* (the block a refused realloc was given is still its own) */ + free(a), free(b), free(m), free(z), free(g); + } + if (!fails) printf("PASS %s\n", c); + return fails != 0; +} +''' + +NEW = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include "new_wait.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +namespace mkwvk { +static std::atomic slept_ms{0}, logs{0}; +static void Sleep(int ms) { slept_ms += ms; } +static void logf(const char*, ...) { logs++; } +static bool known = true; +static uint64_t left = 1ull << 30; // (1 GiB of commit left: at the limit) +static bool new_commit_left(uint64_t* avail) { *avail = left; return known; } +/* REFUSED */ +} // namespace mkwvk +using namespace mkwvk; +static volatile size_t huge = ~size_t(0) / 4; // (no memory can hold it: every such `new` is refused) +// 1: bad_alloc thrown; the sleeps it made are in slept_ms +static int refused_new() { + try { + ::operator delete(::operator new(huge)); // (the function itself: a new-expression's allocation may be left out by the compiler) + return 0; + } catch (const std::bad_alloc&) { + return 1; + } +} +int main(int argc, char** argv) { + const std::string c = argc > 1 ? argv[1] : ""; + using namespace new_wait; + std::set_new_handler(new_refused); + if (c == "off") { // HOS_COMMIT_WAIT_MS=0 + CHECK(refused_new() == 1 && slept_ms.load() == 0); + } else { + // the rule + CHECK(kNear == (4ull << 30) && kStepMs == 50); + CHECK(wait_ms(nullptr) == 5000 && wait_ms("") == 5000 && wait_ms("0") == 0 && wait_ms("-1") == 0 && wait_ms("300") == 300 && wait_ms("999999") == 60000); + CHECK(waits(true, 1ull << 30, 0, 5000, 0) && waits(true, kNear - 1, 4950, 5000, 0)); + CHECK(!waits(true, kNear, 0, 5000, 0) && !waits(true, 64ull << 30, 0, 5000, 0)); // (4 GiB or more left: not the limit's) + CHECK(!waits(false, 0, 0, 5000, 0)); // (the commit not known) + CHECK(!waits(true, 0, 5000, 5000, 0) && !waits(true, 0, 0, 0, 0)); // (its time used up; no wait asked) + CHECK(!waits(true, 0, 0, 5000, 1)); // (a NoWait scope) + // the handler as the runtime calls it: one step slept and back for another try, while it waits + new_refused(), new_refused(), new_refused(); + CHECK(slept_ms.load() == 150 && t_waited_ms == 150 && logs.load() == 3); + t_waited_ms = 0, slept_ms = 0; + // a `new` that cannot be made, at the limit: 5 s of tries, then it throws; this thread's next one throws at once + CHECK(refused_new() == 1 && slept_ms.load() == 5000 && t_waited_ms == 5000); + CHECK(refused_new() == 1 && slept_ms.load() == 5000); + // another thread has its own 5 s + std::thread([] { CHECK(t_waited_ms == 0 && refused_new() == 1 && t_waited_ms == 5000); }).join(); + CHECK(slept_ms.load() == 10000 && logs.load() == 8 + 0); // (the first 8 waits are said) + // far from the limit, and the commit not known: at once + t_waited_ms = 0, slept_ms = 0, left = 8ull << 30; + CHECK(refused_new() == 1 && slept_ms.load() == 0); + left = 1ull << 30, known = false; + CHECK(refused_new() == 1 && slept_ms.load() == 0); + known = true; + // a nothrow new: null at once in a NoWait scope; like any other outside one + { + NoWait at_once; + CHECK(t_no_wait == 1); + { NoWait inner; CHECK(t_no_wait == 2); } + CHECK(::operator new(huge, std::nothrow) == nullptr && slept_ms.load() == 0); + } + CHECK(t_no_wait == 0); + CHECK(::operator new(huge, std::nothrow) == nullptr && slept_ms.load() == 5000); + } + if (!fails) std::printf("PASS %s\n", c.c_str()); + return fails ? 1 : 0; +} +''' + +# (file, what must be there once: the host's tables that end the game on NULL) +HOST_SITES = [ + ("nvn_native.c", "if (c > POOL_MAX) return hos_xmalloc(n);"), + ("nvn_native.c", "return b ? b : hos_xmalloc((size_t)1 << c);"), + ("nvn_native.c", "cmd_rec *k = (cmd_rec *)hos_xcalloc((size_t)1 << CMD_CHUNK_BITS, sizeof *k);"), + ("nvn_native.c", "cmd_free = (uint32_t *)hos_xrealloc(cmd_free, cap_cmd_free * sizeof *cmd_free);"), + ("nvn_native.c", "uint64_t *t = (uint64_t *)hos_xcalloc(cap, sizeof *t);"), + ("nvn_native.c", "live_table *pub = (live_table *)hos_xmalloc(sizeof *pub);"), + ("nvn_native.c", "dead = (dead_copy *)hos_xrealloc(dead, cap_dead * sizeof *dead);"), + ("nvn_native.c", "nvn_tex **t = (nvn_tex **)hos_xcalloc(n, sizeof *t);"), + ("nn_video.c", "host_objs = (HostObj *)hos_xrealloc(host_objs, cap_host_objs * sizeof *host_objs);"), + ("nn_video.c", "Pool *o = (Pool *)hos_xcalloc(1, sizeof *o);"), +] + + +def read_sources(): + glue = (HOST / "mkw_glue.c").read_text(encoding="utf-8") + for name, needle in HOST_SITES: + text = (HOST / name).read_text(encoding="utf-8") + if text.count(needle) != 1 or '#include "host_alloc.h"' not in text: + fail("host/%s: %r is not there once: that table no longer waits at the commit limit" % (name, needle)) + for path in sorted(VK.glob("*.cpp")) + sorted(VK.glob("*.h")): + if path.name == "new_wait.h": + continue + lines = path.read_text(encoding="utf-8").split("\n") + for i, line in enumerate(lines): + if "std::nothrow" in line and not line.lstrip().startswith("//"): + if not any("new_wait::NoWait at_once;" in before for before in lines[max(0, i - 2):i]): + fail("gpu/vk/%s:%d: a nothrow new outside a NoWait scope: at the commit limit it waits 5 s for a null it has " + "its own answer to" % (path.name, i + 1)) + main = (VK / "vk_main.cpp").read_text(encoding="utf-8") + init = block(main, "const MkwVkRenderer* mkwvk_init(const MkwVkHost* h) {") + if "std::set_new_handler(new_refused);" not in init: + fail("vk_main.cpp mkwvk_init does not set the renderer's new handler") + refused = block(main, "void new_refused() {") + for need in ('new_wait::wait_ms(getenv("HOS_COMMIT_WAIT_MS"))', "throw std::bad_alloc();", "Sleep(new_wait::kStepMs);", + "new_wait::waits(known, avail, new_wait::t_waited_ms, max_ms, new_wait::t_no_wait)"): + if need not in refused: + fail("vk_main.cpp new_refused: %r is not there" % need) + if "ms.ullAvailPageFile" not in block(main, "bool new_commit_left(uint64_t* avail) {"): + fail("vk_main.cpp new_commit_left does not read the system's commit left") + commit = block(glue, "int hos_commit(uint64_t addr, uint64_t size) {") + if "commit_err == ERROR_COMMITMENT_LIMIT && commit_wait(GP(lo), (SIZE_T)(hi - lo))" not in commit: + fail("mkw_glue.c hos_commit no longer waits out a refusal at the commit limit") + if "| `HOS_COMMIT_WAIT_MS=` |" not in (ROOT / "docs/development.md").read_text(encoding="utf-8"): + fail("docs/development.md: HOS_COMMIT_WAIT_MS is not in the switches table") + last = block(glue, "void *hos_xrealloc(void *p, size_t n) {") + alloc = glue[glue.index("typedef struct { int kind; void *p; size_t count, n; } HostAllocAsk;"):glue.index(last) + len(last)] + return {"@WAIT_MS@": block(glue, "int hos_commit_wait_ms(void) {"), "@COMMIT_WAIT@": block(glue, "static int commit_wait(void *p, SIZE_T size) {"), + "@ALLOC@": re.sub(r"^DWORD WINAPI hos_out_of_memory_box\(LPVOID\);$", "static DWORD hos_out_of_memory_box(LPVOID p) { (void)p; return 0; }", + alloc, flags=re.M)}, refused + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cc", default="clang") + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + try: + pieces, refused = read_sources() + except (ValueError, OSError) as e: + fail("a text this test reads is not there (%s)" % e) + env = {k: v for k, v in os.environ.items() if k != "HOS_COMMIT_WAIT_MS"} + with tempfile.TemporaryDirectory(prefix="commit-limit-") as tmp: + def build(name, text, cxx=False): + for mark, piece in pieces.items(): + text = text.replace(mark, piece) + src, exe = Path(tmp) / (name + (".cpp" if cxx else ".c")), Path(tmp) / (name + ".exe") + src.write_text(text, encoding="utf-8") + command = [args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-Wno-unused-function", "-pthread", "-I" + str(VK)] if cxx else \ + [args.cc, "-std=gnu17", "-O1", "-Wall", "-Wextra", "-Werror", "-Wno-unused-function", "-Wno-unused-variable", "-Wno-unknown-warning-option", + "-Wno-unused-but-set-variable", "-Wno-unused-but-set-global", "-I" + str(HOST)] + built = subprocess.run(command + [str(src), "-o", str(exe)], capture_output=True, text=True) + if built.returncode: + print(built.stdout + built.stderr) + fail("the %s fixture did not build" % name) + return exe + + def run(exe, *case, switch=None): + case_env = dict(env) if switch is None else dict(env, HOS_COMMIT_WAIT_MS=switch) + done = subprocess.run([str(exe), *case], env=case_env, capture_output=True, text=True, timeout=120) + if done.returncode or "PASS" not in done.stdout: + print(done.stdout[-3000:] + done.stderr[-1000:]) + fail("%s %s, HOS_COMMIT_WAIT_MS %s" % (exe.stem, " ".join(case), "not set" if switch is None else switch)) + + commit = build("commit", COMMIT) + run(commit, "numbers") + run(commit, "made") + run(commit, "never", "5000") + run(commit, "other") + run(commit, "off", switch="0") + run(commit, "never", "200", switch="200") + run(commit, "never", "60000", switch="999999") + print("PASS: the game's commits: tried again every 50 ms for HOS_COMMIT_WAIT_MS, not at 0, not for another error") + alloc = build("alloc", ALLOC) + run(alloc, "wait") + run(alloc, "off", switch="0") + print("PASS: the host's allocations: the same wait, then NULL and the notice once; a size of 0 and the switch at 0 never wait") + new = build("new", NEW.replace("/* REFUSED */", refused), cxx=True) + run(new, "wait") + run(new, "off", switch="0") + print("PASS: the renderer's new: 5 s a thread at the limit, at once with 4 GiB left or in a NoWait scope, and with the switch at 0") + print("PASS: a refusal at Windows' commit limit is waited out in all three places, for HOS_COMMIT_WAIT_MS") + + +if __name__ == "__main__": + main() diff --git a/tests/test_commit_room.py b/tests/test_commit_room.py new file mode 100644 index 0000000..35524a1 --- /dev/null +++ b/tests/test_commit_room.py @@ -0,0 +1,498 @@ +"""The textures near Windows' commit limit, and the page files' room to grow counted as free commit (gpu/vk/commit_room.h, +vk_texture.cpp commit_now, page_file_growth and texture_frame's cut; no game, no GPU). + +(Triforce, 11-22, the performance audit's OS-05, after Source A's.) Off unless RX_VK_TEX_COMMIT_RESERVE_MB is set (or, for the +growth alone, RX_VK_TEX_COMMIT_GROWTH=1): with neither, nothing changes. + +- the rules: the switches' values; commit_low's line, which must agree with tex_pressure::commit_low over a grid of free + commit, limits and its switch; when the growth is read; the cap (the shortfall under the reserve off what the textures + hold, 1 GB at least, none with no reserve or the reserve free); +- the page files' records walked over made-up buffers laid out as 64-bit Windows gives them: a page file that may grow, + one of fixed size, one shrunk, the growth capped by its drive's free space, two on one drive capped together, two + drives apart, a name with no drive, a lower-case drive, a buffer too short; a record whose next one lies outside the + buffer, overlaps it, or whose name lies outside, ends the walk with what came before counted; a drive's free space is + asked only for drives with growth; +- commit_now, cut out of vk_texture.cpp over a scripted GlobalMemoryStatusEx and a stand-in growth, a process a case (the + switches are read once): with no switch the growth is never asked and the answer is Windows' as before; with a reserve + it is asked only while less is free than the reserve or commit_low's line, and added to the free commit and the limit; + RX_VK_TEX_COMMIT_GROWTH=0 never asks, =1 asks under commit_low's line without a reserve, and not with that line off; +- page_file_growth compiled against Windows and run on this PC (NtQuerySystemInformation and the drives' free space, no + network): it answers, and a second ask within the second gives the same answer without asking again; +- texture_frame's cut, cut out and run against stand-ins: no reserve, no look at the commit; the reserve free, nothing + cut; short of it, the budget and the rules' budget held to what the textures hold less the shortfall, 1 GB at least, + a lower budget kept; Linux (commit not known) nothing; the "commit limit near" line on a change and every 600 presents; +- mutations of the cut, of commit_now and of the header must each fail; the sources: the cut after the 97% rule and + before rule_to, the growth in commit_now after Windows' numbers, the switches table. + + python -I -B tests/test_commit_room.py [--cxx clang++] +""" +import argparse +import os +import subprocess +import tempfile +from pathlib import Path + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +CUT = "if (const uint64_t reserve = commit_reserve()) {" + + +def fail(message): + raise SystemExit("FAIL: " + message) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def in_order(where, text, needles): + at = [text.find(n) for n in needles] + if -1 in at or at != sorted(at): + fail(where + ": not in this order: " + " / ".join(needles)) + + +RULES = r''' +#include +#include +#include "commit_room.h" +#include "tex_pressure.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +using namespace commit_room; +static const uint64_t MB = 1ull << 20, GB = 1ull << 30; + +// a buffer of records as 64-bit Windows lays them out, the names after them +struct Buf { + alignas(8) uint8_t b[2048] = {}; + size_t len = 0, names = 1024; + const uint16_t* name(const char* s) { + uint16_t* at = (uint16_t*)(b + names); + size_t n = std::strlen(s); + for (size_t i = 0; i < n; i++) at[i] = (uint16_t)s[i]; + names += n * 2 + 8; + return at; + } + // a record at `at`: its next, its size and the size it may grow to (pages), its name + void put(size_t at, uint32_t next, uint32_t total, uint32_t most, const char* file, const uint16_t* where = nullptr) { + PageFileRecord r{}; + r.next = next, r.total_pages = total, r.max_pages = most, r.min_pages = total; + if (file) { + r.name = where ? where : name(file); + r.name_bytes = r.name_room = (uint16_t)(std::strlen(file) * 2); + } + std::memcpy(b + at, &r, sizeof r); + len = names; + } +}; +static uint64_t free_c, free_d; +static int asked_c, asked_d, asked_other; +static uint64_t free_of(char d) { + if (d == 'C') return asked_c++, free_c; + if (d == 'D') return asked_d++, free_d; + return asked_other++, 0; +} +static uint64_t walk(const Buf& B) { return growth(B.b, B.len, 4096, free_of); } +static void drives(uint64_t c, uint64_t d) { free_c = c, free_d = d, asked_c = asked_d = asked_other = 0; } + +int main() { + // ---- the switches ---- + CHECK(reserve_of(nullptr) == 0 && reserve_of("") == 0 && reserve_of("0") == 0 && reserve_of("x") == 0 && reserve_of("-5") == 0); + CHECK(reserve_of("2560") == 2560 * MB && reserve_of("1") == MB && reserve_of("999999") == 65536 * MB); + CHECK(!growth_on(nullptr, 0) && growth_on(nullptr, 2560 * MB) && !growth_on("", 0) && growth_on("", MB)); // (unset: with a reserve) + CHECK(!growth_on("0", 2560 * MB) && !growth_on("0", 0) && growth_on("1", 0) && growth_on("1", MB) && !growth_on("x", MB)); + // ---- commit_low's line, as tex_pressure::commit_low has it ---- + CHECK(low_line(0, -1) == 0 && low_line(32 * GB, 0) == 0); + CHECK(low_line(32 * GB, -1) == 4096 * MB && low_line(64 * GB, -1) == 64 * GB / 10 && low_line(32 * GB, 6000) == 6000 * MB); + for (uint64_t limit : {0ull, 8 * GB, 16 * GB, 40 * GB, 41 * GB, 64 * GB, 128 * GB}) + for (int64_t want : {-1ll, 0ll, 1ll, 3000ll, 4096ll, 9000ll}) + for (uint64_t avail = 0; avail <= 20 * GB; avail += 256 * MB) { + const uint64_t low = low_line(limit, want); + if (tex_pressure::commit_low(avail, limit, want) != (low && avail < low)) { + std::printf("FAIL: low_line(%llu MB, %lld) disagrees with tex_pressure::commit_low at %llu MB free\n", + (unsigned long long)(limit / MB), (long long)want, (unsigned long long)(avail / MB)); + fails++; + limit = 1ull << 62, avail = 1ull << 62; + } + } + // ---- when the growth is read: counted, and under the higher of the reserve and the line ---- + CHECK(!read_growth(false, 0, 2560 * MB, 4096 * MB)); + CHECK(read_growth(true, 2000 * MB, 2560 * MB, 0) && !read_growth(true, 2560 * MB, 2560 * MB, 0)); + CHECK(read_growth(true, 3000 * MB, 2560 * MB, 4096 * MB) && !read_growth(true, 4096 * MB, 2560 * MB, 4096 * MB)); + CHECK(!read_growth(true, 0 + 1, 0, 0)); // (no reserve, the line off: never) + // ---- the cap ---- + CHECK(cap(4 * GB, 1024 * MB, 0) == UINT64_MAX); // (no reserve) + CHECK(cap(4 * GB, 2560 * MB, 2560 * MB) == UINT64_MAX && cap(4 * GB, 9 * GB, 2560 * MB) == UINT64_MAX); + CHECK(cap(4 * GB, 1024 * MB, 2560 * MB) == 4 * GB - 1536 * MB); // (the shortfall off what they hold) + CHECK(cap(1500 * MB, 1024 * MB, 2560 * MB) == GB); // (1 GB at least) + CHECK(cap(4 * GB, 0, 2560 * MB) == 4 * GB - 2560 * MB && cap(2 * GB, 0, 2560 * MB) == GB && cap(0, 0, 2560 * MB) == GB); + // ---- the drive of a name ---- + { + const uint16_t a[] = {'\\', '?', '?', '\\', 'C', ':', '\\', 'p'}, b[] = {'\\', '?', '?', '\\', 'd', ':', '\\'}, c[] = {'\\', 'D', 'e', 'v'}; + CHECK(drive_of(a, 8) == 'C' && drive_of(b, 7) == 'D' && drive_of(c, 4) == 0 && drive_of(a, 5) == 0); + } + // ---- the records ---- + { // one that may grow by 2000 pages, on a drive with room + Buf B; + B.put(0, 0, 1000, 3000, "\\??\\C:\\pagefile.sys"); + drives(100 * GB, 0); + CHECK(walk(B) == 2000ull * 4096 && asked_c == 1 && asked_d == 0 && asked_other == 0); + drives(MB, 0); + CHECK(walk(B) == MB); // (capped by the drive's free space) + } + { // fixed, and shrunk below what it may be: nothing + Buf B; + B.put(0, 40, 3000, 3000, "\\??\\C:\\pagefile.sys"); + B.put(40, 0, 3000, 2000, "\\??\\D:\\pagefile.sys"); + drives(100 * GB, 100 * GB); + CHECK(walk(B) == 0 && asked_c == 0 && asked_d == 0); // (no drive asked without growth) + } + { // two on C capped together, one on D apart + Buf B; + B.put(0, 40, 1000, 3000, "\\??\\C:\\pagefile.sys"); + B.put(40, 40, 0, 1000, "\\??\\C:\\swap\\second.sys"); + B.put(80, 0, 0, 500, "\\??\\d:\\pagefile.sys"); + drives(10 * MB, 100 * GB); + CHECK(walk(B) == 10 * MB + 500ull * 4096 && asked_c == 1 && asked_d == 1); + drives(100 * GB, MB); + CHECK(walk(B) == 3000ull * 4096 + MB); + } + { // a name with no drive counts nothing, and the walk goes on + Buf B; + B.put(0, 40, 0, 1000, "\\Device\\HarddiskVolume3\\pagefile.sys"); + B.put(40, 0, 0, 1000, "\\??\\C:\\pagefile.sys"); + drives(100 * GB, 0); + CHECK(walk(B) == 1000ull * 4096 && asked_other == 0); + } + { // the next record outside the buffer: the walk ends, the first counted + Buf B; + B.put(0, 4000, 0, 1000, "\\??\\C:\\pagefile.sys"); + drives(100 * GB, 0); + CHECK(walk(B) == 1000ull * 4096); + B.put(0, 8, 0, 1000, "\\??\\C:\\pagefile.sys"); // (one that would overlap it) + B.put(40, 0, 0, 7000, "\\??\\C:\\pagefile.sys"); + CHECK(walk(B) == 1000ull * 4096); + } + { // a name outside the buffer: that record and those after it count nothing + Buf B; + static const uint16_t elsewhere[] = {'\\', '?', '?', '\\', 'C', ':', '\\', 'p', 'f'}; + B.put(0, 40, 0, 1000, "\\??\\C:\\pagefile.sys"); + B.put(40, 40, 0, 2000, "\\??\\C:\\pf", elsewhere); + B.put(80, 0, 0, 4000, "\\??\\C:\\pagefile.sys"); + drives(100 * GB, 0); + CHECK(walk(B) == 1000ull * 4096); + uint16_t* near_end = (uint16_t*)(B.b + B.len - 4); // (a name that starts inside and runs past the end) + near_end[0] = 'C', near_end[1] = ':'; + B.put(40, 40, 0, 2000, "\\??\\C:\\pf", near_end); + B.len = B.names; + CHECK(walk(B) == 1000ull * 4096); + } + { // too short for a record, or empty + Buf B; + B.put(0, 0, 0, 1000, "\\??\\C:\\pagefile.sys"); + drives(100 * GB, 0); + CHECK(growth(B.b, 39, 4096, free_of) == 0 && growth(B.b, 0, 4096, free_of) == 0 && asked_c == 0); + } + if (!fails) std::printf("PASS rules\n"); + return fails ? 1 : 0; +} +''' + +NOW = r''' +#include +#include +#include +#include +#include +#include +#include +#include "commit_room.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +static const uint64_t MB = 1ull << 20, GB = 1ull << 30; +// (Windows' numbers as the case sets them, and the page files' growth as a stand-in that counts its asks) +static uint64_t win_avail, win_limit; +static BOOL fake_status(LPMEMORYSTATUSEX m) { + m->ullAvailPageFile = win_avail, m->ullTotalPageFile = win_limit; + return TRUE; +} +#define GlobalMemoryStatusEx fake_status +static int growth_asks; +static uint64_t page_file_growth_fake() { return growth_asks++, 3 * GB; } +/* CUT */ +static bool look(uint64_t avail_mb, uint64_t limit_mb, uint64_t* avail, uint64_t* limit) { + win_avail = avail_mb * MB, win_limit = limit_mb * MB, growth_asks = 0; + return commit_now(avail, limit); +} +int main(int argc, char** argv) { + const std::string c = argc > 1 ? argv[1] : ""; + uint64_t a = 0, l = 0; + if (c == "unset") { // (no switch: Windows' numbers, the growth never asked, however little is free) + for (uint64_t free_mb : {0ull, 100ull, 2000ull, 4000ull, 9000ull}) { + CHECK(look(free_mb, 40000, &a, &l) && a == free_mb * MB && l == 40000 * MB && growth_asks == 0 && commit_grown == 0); + } + } else if (c == "reserve") { // (RX_VK_TEX_COMMIT_RESERVE_MB=2560: under the higher of it and commit_low's line, 4 GB here) + CHECK(look(9000, 40000, &a, &l) && a == 9000 * MB && l == 40000 * MB && growth_asks == 0); + CHECK(look(4096, 40000, &a, &l) && growth_asks == 0); + CHECK(look(4000, 40000, &a, &l) && a == 4000 * MB + 3 * GB && l == 40000 * MB + 3 * GB && growth_asks == 1 && commit_grown == 3 * GB); + CHECK(look(1000, 40000, &a, &l) && a == 1000 * MB + 3 * GB && growth_asks == 1); + CHECK(look(9000, 40000, &a, &l) && a == 9000 * MB && commit_grown == 0); // (the last growth not carried over) + } else if (c == "reserve-big") { // (a reserve above commit_low's line: the reserve's) + CHECK(look(5000, 40000, &a, &l) && growth_asks == 1 && a == 5000 * MB + 3 * GB); + CHECK(look(6000, 40000, &a, &l) && growth_asks == 1); + CHECK(look(8000, 40000, &a, &l) && growth_asks == 0 && a == 8000 * MB); + } else if (c == "never") { // (RX_VK_TEX_COMMIT_GROWTH=0, a reserve set: never) + CHECK(look(100, 40000, &a, &l) && a == 100 * MB && l == 40000 * MB && growth_asks == 0); + } else if (c == "growth-alone") { // (RX_VK_TEX_COMMIT_GROWTH=1, no reserve: under commit_low's line) + CHECK(look(5000, 40000, &a, &l) && growth_asks == 0); + CHECK(look(4000, 40000, &a, &l) && growth_asks == 1 && a == 4000 * MB + 3 * GB); + } else if (c == "line-off") { // (RX_VK_TEX_COMMIT_GROWTH=1, RX_VK_TEX_COMMIT_FREE_MB=0, no reserve: nothing to read it for) + CHECK(look(100, 40000, &a, &l) && growth_asks == 0 && a == 100 * MB); + } else { + std::printf("FAIL: unknown case\n"); + fails++; + } + if (!fails) std::printf("PASS %s\n", c.c_str()); + return fails ? 1 : 0; +} +''' + +NOW_CASES = [ + ("unset", {}), ("reserve", {"RX_VK_TEX_COMMIT_RESERVE_MB": "2560"}), ("reserve-big", {"RX_VK_TEX_COMMIT_RESERVE_MB": "8000"}), + ("never", {"RX_VK_TEX_COMMIT_RESERVE_MB": "2560", "RX_VK_TEX_COMMIT_GROWTH": "0"}), + ("growth-alone", {"RX_VK_TEX_COMMIT_GROWTH": "1"}), + ("line-off", {"RX_VK_TEX_COMMIT_GROWTH": "1", "RX_VK_TEX_COMMIT_FREE_MB": "0"}), +] + +REAL = r''' +#include +#include +#include +#include +#include "commit_room.h" +/* GROWTH */ +int main() { + const auto t0 = std::chrono::steady_clock::now(); + const uint64_t g = page_file_growth(); + const auto t1 = std::chrono::steady_clock::now(); + const uint64_t again = page_file_growth(); + const auto t2 = std::chrono::steady_clock::now(); + const double first_ms = std::chrono::duration(t1 - t0).count(), again_ms = std::chrono::duration(t2 - t1).count(); + std::printf("page files' room to grow on this PC: %llu MB (asked in %.3f ms, again in %.4f ms)\n", (unsigned long long)(g >> 20), first_ms, again_ms); + if (again != g) { std::printf("FAIL: a second ask within the second gave another answer\n"); return 1; } + if (g > (1ull << 50)) { std::printf("FAIL: an answer past any drive\n"); return 1; } + std::printf("PASS real\n"); + return 0; +} +''' + +FRAME = r''' +#include +#include +#include +#include +#include +#include "commit_room.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +static const uint64_t MB = 1ull << 20, GB = 1ull << 30; +static uint64_t reserve_now, now_avail, now_limit = 40 * GB, commit_grown, frame_no; +static bool now_known = true; +static int asks, lines; +static char last_line[600]; +static uint64_t commit_reserve() { return reserve_now; } +static bool commit_now(uint64_t* avail, uint64_t* limit) { + asks++; + if (!now_known) return false; + *avail = now_avail, *limit = now_limit; + return true; +} +static void logf(const char* f, ...) { + va_list ap; + va_start(ap, f); + vsnprintf(last_line, sizeof last_line, f, ap); + va_end(ap); + lines++; +} +// one look's cut: budget and rule_budget as texture_frame has them after the 97% rule +static void look(uint64_t& budget, uint64_t& rule_budget, uint64_t total_bytes) { +/* CUT */ +} +static uint64_t b, r; +static void at(uint64_t frame, uint64_t free_mb, uint64_t total_mb, uint64_t budget_mb = 0, uint64_t rule_mb = 6000) { + frame_no = frame, now_avail = free_mb * MB, asks = 0; + b = budget_mb ? budget_mb * MB : UINT64_MAX, r = rule_mb * MB; + look(b, r, total_mb * MB); +} +int main() { + // no reserve: no look at the commit, nothing cut, no line + at(10, 100, 4096); + CHECK(asks == 0 && b == UINT64_MAX && r == 6000 * MB && lines == 0); + reserve_now = 2560 * MB; + // the reserve free: nothing cut + at(20, 3000, 4096); + CHECK(asks == 1 && b == UINT64_MAX && r == 6000 * MB && lines == 0); + // short by 1536 MB: what the textures hold less that, on both budgets, one line + at(30, 1024, 4096); + CHECK(b == 2560 * MB && r == 2560 * MB && lines == 1 && std::strstr(last_line, "vk: commit limit near: 1024 MB of 40960 free")); + // still short: no line until a 600th present, then one + at(40, 1024, 4096); + CHECK(b == 2560 * MB && lines == 1); + at(600, 1024, 4096); + CHECK(lines == 2 && std::strstr(last_line, "near:")); + // a budget already lower is kept; the floor + at(610, 1024, 4096, 1000, 900); + CHECK(b == 1000 * MB && r == 900 * MB); + at(620, 1024, 1500); + CHECK(b == GB && r == GB); + at(630, 0, 900); + CHECK(b == GB && r == GB); // (1 GB at least: more than they hold, nothing goes) + // free again: nothing cut, one line + at(640, 3000, 4096); + CHECK(b == UINT64_MAX && r == 6000 * MB && lines == 3 && std::strstr(last_line, "no longer near")); + // the commit not known (Linux): nothing + now_known = false; + at(650, 0, 4096); + CHECK(b == UINT64_MAX && r == 6000 * MB && lines == 3); + if (!fails) std::printf("PASS frame\n"); + return fails ? 1 : 0; +} +''' + +# (each: where, what is taken out, the text, what replaces it) +MUTATIONS = [ + ("frame", "the budget capped", " budget = std::min(budget, cap);\n", ""), + ("frame", "the rules' budget capped", " rule_budget = std::min(rule_budget, cap);\n", ""), + ("frame", "the commit not known: no cut", "commit_now(&avail, &limit) ? commit_room::cap(total_bytes, avail, reserve) : UINT64_MAX", + "(commit_now(&avail, &limit), commit_room::cap(total_bytes, avail, reserve))"), + ("now", "the growth added", "*avail += commit_grown, *limit += commit_grown;", "(void)0;"), + ("now", "the growth only where it can matter", "if (commit_room::read_growth(", "if (true || commit_room::read_growth("), + ("header", "the floor", "return std::max(kFloor, total > shortfall ? total - shortfall : 0);", "return total > shortfall ? total - shortfall : 0;"), + ("header", "the drive's cap", "sum += std::min(by_drive[d], free_of((char)('A' + d)));", "sum += by_drive[d];"), + ("header", "a name outside the buffer ends the walk", "if (n < base || n - base > len || r.name_bytes > len - (n - base)) break;", + "if (n < base || n - base > len) break;"), + ("header", "no growth unset without a reserve", "if (!env || !*env) return reserve != 0;", "if (!env || !*env) return true;"), +] + + +def read_sources(): + tex = (VK / "vk_texture.cpp").read_text(encoding="utf-8") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + if '#include "commit_room.h"' not in tex: + fail("vk_texture.cpp does not include commit_room.h") + now = block(tex, "bool commit_now(uint64_t* avail, uint64_t* limit) {") + in_order("vk_texture.cpp commit_now (Windows' numbers, then the growth where it can matter, added to both)", now, + ["*avail = ms.ullAvailPageFile, *limit = ms.ullTotalPageFile;", + "if (commit_room::read_growth(commit_growth_on(), *avail, commit_reserve(), commit_room::low_line(*limit, commit_free_mb()))) {", + "commit_grown = page_file_growth();", "*avail += commit_grown, *limit += commit_grown;", "return true;"]) + switches = {name: block(tex, sig) for name, sig in (("reserve", "uint64_t commit_reserve() {"), ("growth", "bool commit_growth_on() {"), + ("free", "int64_t commit_free_mb() {"))} + for name, text, need in (("reserve", switches["reserve"], 'static const uint64_t v = commit_room::reserve_of(getenv("RX_VK_TEX_COMMIT_RESERVE_MB"));'), + ("growth", switches["growth"], 'static const bool v = commit_room::growth_on(getenv("RX_VK_TEX_COMMIT_GROWTH"), commit_reserve());'), + ("free", switches["free"], 'getenv("RX_VK_TEX_COMMIT_FREE_MB") ? atoll(getenv("RX_VK_TEX_COMMIT_FREE_MB")) : -1;')): + if need not in text: + fail("vk_texture.cpp: the %s switch is not read once as the rule says" % name) + growth = block(tex, "uint64_t page_file_growth() {") + for need in ("commit_room::kPageFileClass", "commit_room::growth(", "GetDiskFreeSpaceExW", "std::chrono::seconds(1)"): + if need not in growth: + fail("vk_texture.cpp page_file_growth: %r is not there" % need) + frame = block(tex, "void texture_frame(uint64_t completed) {") + if "static const int64_t want_mb = commit_free_mb();" not in frame: + fail("vk_texture.cpp texture_frame: commit_low's switch is not the one commit_now reads") + if frame.count(CUT) != 1: + fail("vk_texture.cpp texture_frame: the reserve's cut is not there once") + in_order("vk_texture.cpp texture_frame (the cut after the 97% rule, before rule_to and the pass)", frame, + ["if (known && !by_blocks && used > allowed / 100 * 97 && !room_in_blocks) {", + "rule_budget = std::min(rule_budget, total_bytes > over ? total_bytes - over : 0);", CUT, + "const uint64_t rule_to =", "if (budget < evict_to) evict_to = hold_to = 0;", + "texture_sweep(completed, total_bytes <= budget, budget,"]) + in_order("docs/development.md (the rows beside RX_VK_TEX_COMMIT_FREE_MB's)", dev, + ["| `RX_VK_TEX_COMMIT_FREE_MB=` |", "| `RX_VK_TEX_COMMIT_RESERVE_MB=` |", "| `RX_VK_TEX_COMMIT_GROWTH=0` or `=1` |"]) + now_cut = "\n".join([switches["reserve"], switches["growth"], switches["free"], "uint64_t commit_grown = 0;", now]) + if now_cut.count("commit_grown = page_file_growth();") != 1: + fail("vk_texture.cpp commit_now: the growth is not asked once") + return block(frame, CUT), now_cut.replace("commit_grown = page_file_growth();", "commit_grown = page_file_growth_fake();"), growth + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + try: + cut, now, growth = read_sources() + header = (VK / "commit_room.h").read_text(encoding="utf-8") + except (ValueError, OSError) as e: + fail("a text this test reads is not there (%s)" % e) + env = {k: v for k, v in os.environ.items() if not k.startswith("RX_VK_TEX_COMMIT_")} + with tempfile.TemporaryDirectory(prefix="commit-room-") as tmp: + tmpdir = Path(tmp) + + def build(name, text, head=None): + inc = tmpdir / ("inc-" + name) + inc.mkdir() + (inc / "commit_room.h").write_text(head if head is not None else header, encoding="utf-8") + src, exe = tmpdir / (name + ".cpp"), tmpdir / (name + ".exe") + src.write_text(text, encoding="utf-8") + built = subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-Wno-unused-function", + "-Wno-unused-parameter", "-I" + str(inc), "-I" + str(VK), str(src), "-o", str(exe)], capture_output=True, text=True) + if built.returncode: + print(built.stdout + built.stderr) + fail("the fixture did not build (%s)" % name) + return exe + + def run(exe, case=None, **switches): + try: + return subprocess.run([str(exe)] + ([case] if case else []), env=dict(env, **switches), capture_output=True, text=True, timeout=120) + except subprocess.TimeoutExpired: + return subprocess.CompletedProcess([], 1, "no end in 120 s", "") + + def passes(exe, cases): + for case, switches in cases: + done = run(exe, case, **switches) + if done.returncode or "PASS" not in done.stdout: + return False, (done.stdout + done.stderr)[-2000:] + return True, "" + + fixtures = {"rules": (RULES, None), "now": (NOW, now), "frame": (FRAME, cut)} + cases = {"rules": [(None, {})], "now": NOW_CASES, "frame": [(None, {})]} + + def text_of(which, part): + body, cut_text = fixtures[which] + return body.replace("/* CUT */", part if part is not None else (cut_text or "")) + + for which in ("rules", "now", "frame"): + ok, out = passes(build(which, text_of(which, None)), cases[which]) + if not ok: + print(out) + fail(which) + print("PASS " + which + ("" if which != "now" else " (%d cases, each its own process)" % len(NOW_CASES))) + done = run(build("real", REAL.replace("/* GROWTH */", growth))) + print(done.stdout.strip()) + if done.returncode or "PASS real" not in done.stdout: + fail("page_file_growth on this PC") + for i, (where, what, old, new) in enumerate(MUTATIONS): + if where == "header": + if header.count(old) != 1: + fail("mutation %r: not in commit_room.h once (update the test with the code)" % what) + ok = all(passes(build("m%d%s" % (i, w), text_of(w, None), header.replace(old, new)), cases[w])[0] for w in ("rules", "now", "frame")) + else: + part = fixtures[where][1] + if part.count(old) != 1: + fail("mutation %r: not in the cut code once (update the test with the code)" % what) + ok, _ = passes(build("m%d" % i, text_of(where, part.replace(old, new))), cases[where]) + if ok: + fail("mutation %r: the fixtures still passed without it" % what) + print("PASS mutation: without %s a case fails" % what) + print("PASS: off unless set; the growth read only where it can matter and the walk bounded by the buffer; short of the " + "reserve, the textures held to what they hold less the shortfall, 1 GB at least") + + +if __name__ == "__main__": + main() diff --git a/tests/test_compact_start.py b/tests/test_compact_start.py new file mode 100644 index 0000000..34fae3c --- /dev/null +++ b/tests/test_compact_start.py @@ -0,0 +1,147 @@ +"""The free space a VRAM compaction run starts from, by the card (gpu/vk/vram_compact.h default_min_free; the performance +audit's TEX-38, after Source A's; no game, no GPU). + +The rule, compiled and run: a sixteenth of what the driver allows, 64 MB at least and 512 at most, 512 while nothing is +known; for cards from 2 to 32 GB as drivers report them (a little under the card's size) it is 512 from 8 GB up, exactly as +before, and less only under that; 192 MB still while the count is pressed. Each part taken out (a mutation) fails it. Then +the source: compact_step takes RX_VK_COMPACT_MB first and the rule otherwise, with the allowance of texture_frame's last +look; and the docs row. + + python -I -B tests/test_compact_start.py [--cxx clang++] +""" +import argparse +import os +import subprocess +import tempfile +from pathlib import Path + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def in_order(text, parts, where): + at = 0 + for part in parts: + found = text.find(part, at) + if found < 0: + raise SystemExit("%s: %r is missing, or not after the part before it (update the test with the code)" % (where, part)) + at = found + len(part) + + +RULE_TEST = r''' +#include +#include +#include "vram_compact.h" +#define CHECK(x) do { if (!(x)) { std::fprintf(stderr, "FAIL:%d %s\n", __LINE__, #x); std::exit(1); } } while (0) +using namespace vram_compact; +int main() { + const uint64_t MB = 1ull << 20, GB = 1ull << 30; + // (the entry's four: not known, 512 MB, a 6 GB laptop's 5,233 MB, 16 GB) + CHECK(default_min_free(0) == 512 * MB); + CHECK(default_min_free(512 * MB) == 64 * MB); + CHECK(default_min_free(5233 * MB) >> 20 == 327); + CHECK(default_min_free(16 * GB) == 512 * MB); + // (cards from 2 to 32 GB, the allowance a driver gives a little under the card: 512 from 8 GB up, as before) + const struct { uint64_t card_gb, allowed_mb, want_mb; } cards[] = { + {2, 1843, 115}, {3, 2867, 179}, {4, 3686, 230}, {6, 5233, 327}, {6, 5734, 358}, {8, 7300, 456}, + {8, 7782, 486}, {10, 9420, 512}, {11, 10900, 512}, {12, 11468, 512}, {16, 15564, 512}, {20, 19456, 512}, + {24, 23552, 512}, {32, 31784, 512}, {32, 32607, 512}, + }; + for (const auto& c : cards) { + const uint64_t v = default_min_free(c.allowed_mb * MB); + if ((v >> 20) != c.want_mb) { + std::fprintf(stderr, "FAIL: a %llu GB card allowing %llu MB: %llu MB, want %llu\n", (unsigned long long)c.card_gb, + (unsigned long long)c.allowed_mb, (unsigned long long)(v >> 20), (unsigned long long)c.want_mb); + return 1; + } + CHECK(v >= 64 * MB && v <= 512 * MB); + if (c.allowed_mb >= 8192) CHECK(v == 512 * MB); // (this PC's 32 GB card and every card from 8 GB: today's 512) + CHECK(run_from(v, true) == (v < 192 * MB ? v : 192 * MB) && run_from(v, false) == v); + } + // (the bounds) + CHECK(default_min_free(1) == 64 * MB && default_min_free(1024 * MB) == 64 * MB && default_min_free(1040 * MB) == 65 * MB); + CHECK(default_min_free(8 * GB) == 512 * MB && default_min_free(8 * GB - 16 * MB) == 511 * MB && default_min_free(~0ull) == 512 * MB); + std::puts("PASS rule: a sixteenth of the allowance, 64 to 512 MB, 512 unknown; 512 on every card from 8 GB"); + return 0; +} +''' + +MUTATIONS = [ + ("512 while not known", " if (!allowed) return 512ull << 20;\n", ""), + ("a sixteenth", "return std::clamp(allowed / 16, 64ull << 20, 512ull << 20);", + "return std::clamp(allowed / 8, 64ull << 20, 512ull << 20);"), + ("64 MB at least", "return std::clamp(allowed / 16, 64ull << 20, 512ull << 20);", + "return std::clamp(allowed / 16, 1, 512ull << 20);"), + ("512 MB at most", "return std::clamp(allowed / 16, 64ull << 20, 512ull << 20);", + "return std::clamp(allowed / 16, 64ull << 20, 4096ull << 20);"), +] + + +def check_source(tex, dev): + step = block(tex, "void compact_step(uint64_t completed) {") + in_order(step, ['static const uint64_t asked = (uint64_t)(getenv("RX_VK_COMPACT_MB") ? std::max(1, atoi(getenv("RX_VK_COMPACT_MB"))) : 0) << 20;', + "const uint64_t min_free = asked ? asked : vram_compact::default_min_free(compact.allowed);", + "free_bytes < vram_compact::run_from(min_free, compact.pressed))"], "vk_texture.cpp compact_step") + in_order(tex, ["const bool known = vram_usage(&used, &allowed, &block_free);", + "compact.pressed = known && vram_compact::pressed(used, allowed);", "compact.allowed = known ? allowed : 0;"], + "vk_texture.cpp texture_frame: the allowance of the last look") + if "uint64_t allowed = 0;" not in block(tex, "struct Compact {"): + raise SystemExit("vk_texture.cpp: struct Compact has no allowance") + if "default_min_free" not in dev or "RX_VK_COMPACT_MB=512" not in dev: + raise SystemExit("docs/development.md: the compaction's row does not say the start by the card, nor the switch back") + + +def build(cxx, d, name, src, include): + cpp, exe = d / (name + ".cpp"), d / (name + ".exe") + cpp.write_text(src, encoding="utf-8") + r = subprocess.run([cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(include), str(cpp), "-o", str(exe)], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + if r.returncode: + return None, r.stdout + return exe, "" + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--cxx", default=os.environ.get("CXX", "g++")) + a = ap.parse_args() + check_source((VK / "vk_texture.cpp").read_text(encoding="utf-8"), (ROOT / "docs" / "development.md").read_text(encoding="utf-8")) + print("PASS sources: compact_step takes RX_VK_COMPACT_MB first, the rule with the last look's allowance otherwise; the row") + header = (VK / "vram_compact.h").read_text(encoding="utf-8") + with tempfile.TemporaryDirectory(prefix="compact-start-") as tmp: + d = Path(tmp) + exe, out = build(a.cxx, d, "rule", RULE_TEST, VK) + if not exe: + raise SystemExit("rule: the build failed:\n" + out) + r = subprocess.run([str(exe)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=60) + if r.returncode: + raise SystemExit("the rule's test failed:\n" + r.stdout) + print(r.stdout.strip()) + for i, (what, old, new) in enumerate(MUTATIONS): + if header.count(old) != 1: + raise SystemExit("mutation %r: its text is not once in vram_compact.h (update the test with the header)" % what) + inc = d / ("mutant%d" % i) + inc.mkdir() + (inc / "vram_compact.h").write_text(header.replace(old, new), encoding="utf-8") + exe, out = build(a.cxx, d, "mutant%d" % i, RULE_TEST, inc) + if exe: + r = subprocess.run([str(exe)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=60) + if r.returncode == 0: + raise SystemExit("mutation %r: the rule's test still passed without it" % what) + print("PASS mutations: %d parts of the rule, each taken out, fail its test" % len(MUTATIONS)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_compile_stops_on_exception.py b/tests/test_compile_stops_on_exception.py new file mode 100644 index 0000000..1a86eb9 --- /dev/null +++ b/tests/test_compile_stops_on_exception.py @@ -0,0 +1,166 @@ +"""The compile stops when a compile raises (nx2/build.py main's compile loop, MemoryGate.stop; no compiler, no game). + +(Triforce, 11-21, the performance audit's LIFT-31, after Source A's.) A compiler's error comes back as a value and is +tried again alone. An exception does not (a missing or blocked compiler, an object that cannot be written, a full disk, +Ctrl-C): it left the pool's block, and the block's exit ran every compile still queued first, the whole compile with +nothing said, before the build stopped. + +Runs main's own statements, from "t0, failed =" to "if still", cut out through the AST as tests/test_compile_retry_jobs.py +does, and the real MemoryGate, with stand-in compiles: +1. one worker, 12 jobs, the first raises: the exception reaches the caller, and at most one more compile starts; +2. the real memory gate holding three workers back (no memory free): the first compile raises, the gate is stopped, at + most one of the three starts (the one that took the raising compile's lease first), and none is left holding a lease; +3. the gate alone: a waiter and a late comer both raise CompileStopped once it is stopped, and a gate not stopped gives + leases as before. + + python -I -B tests/test_compile_stops_on_exception.py +""" +import argparse +import ast +import concurrent.futures +from contextlib import redirect_stdout +import io +import os +from pathlib import Path +import threading +import time +from types import SimpleNamespace +import unittest + +ROOT = Path(__file__).resolve().parents[1] +SOURCE = ROOT / "nx2/build.py" + + +def actual(source): + """main's compile statements as run_compile_jobs(), and the real MemoryGate and CompileStopped, in one scope""" + tree = ast.parse(source.read_text(encoding="utf-8")) + classes = [node for node in tree.body if isinstance(node, ast.ClassDef) and node.name in ("MemoryGate", "CompileStopped")] + main, = [node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == "main"] + start, = [i for i, node in enumerate(main.body) if isinstance(node, ast.Assign) + and any(isinstance(t, ast.Tuple) and [getattr(n, "id", None) for n in t.elts] == ["t0", "failed"] + for t in node.targets)] + end, = [i for i, node in enumerate(main.body) if i > start and isinstance(node, ast.If) + and isinstance(node.test, ast.Name) and node.test.id == "still"] + wrapper = ast.parse("def run_compile_jobs():\n return 0\n").body[0] + wrapper.body = main.body[start:end + 1] + wrapper.body + module = ast.fix_missing_locations(ast.Module(body=classes + [wrapper], type_ignores=[])) + scope = dict(os=os, time=time, concurrent=concurrent, threading=threading, GATE=None, COMPILER_MB=1000, + counted=False, expected={}, pending=[], eta_jobs=1, eta_left=lambda pending, jobs: 0) + exec(compile(module, str(source), "exec"), scope) + return scope + + +class CompileStopsOnException(unittest.TestCase): + def setUp(self): + self.scope = actual(SOURCE) + self.started, self.lock = [], threading.Lock() + + def run_jobs(self): + with redirect_stdout(io.StringIO()): + return self.scope["run_compile_jobs"]() + + def test_one_worker_stops_after_the_compile_that_raised(self): + jobs = [("cc", __file__, "obj%d.o" % i, [], b"") for i in range(12)] + + def compile_one(job): + with self.lock: + self.started.append(job[2]) + if job[2] == "obj0.o": + raise FileNotFoundError("authored: the compiler is gone") + time.sleep(0.05) + return job[1], 0.05, "" + self.scope.update(compile_one=compile_one, jobs=jobs, a=SimpleNamespace(jobs=1)) + with self.assertRaisesRegex(FileNotFoundError, "the compiler is gone"): + self.run_jobs() + # (the worker may have taken the next job before the queue was cancelled: that one, no more. Before: all 12) + self.assertLessEqual(len(self.started), 2, self.started) + self.assertEqual(self.started[0], "obj0.o") + + def test_workers_waiting_at_the_memory_gate_start_nothing(self): + gate = self.scope["MemoryGate"]() + gate.floor = 1 + # (nothing free: with one compile running, every other waits at the gate) + self.scope.update(mem_need=lambda src: 400, free_mb=lambda: 0, rss_mb=lambda p: 0, GATE=gate) + waiting = threading.Event() + jobs = [("cc", __file__, "obj%d.o" % i, [], b"") for i in range(4)] + + def compile_one(job): # (as build.py's: the lease first, then the compiler) + first = job[2] == "obj0.o" + if first: + lease = gate.enter(job[1]) + for _ in range(400): # (until the other three wait at the gate) + if len(self.started) == 3: + break + time.sleep(0.005) + gate.leave(lease) + raise OSError("authored: the object cannot be written") + with self.lock: + self.started.append(job[2]) # (asked for its lease) + lease = gate.enter(job[1]) + with self.lock: + self.started.append("compiling " + job[2]) + time.sleep(0.3) # (a compile takes its time: the gate is stopped long before this one's lease is free) + gate.leave(lease) + return job[1], 0.3, "" + # (obj0 holds the one lease the floor gives before the others ask) + real_submit = concurrent.futures.ThreadPoolExecutor.submit + + def submit(ex, fn, job): + future = real_submit(ex, fn, job) + if job[2] == "obj0.o": + for _ in range(400): + if gate.running: + break + time.sleep(0.005) + return future + concurrent.futures.ThreadPoolExecutor.submit = submit + try: + self.scope.update(compile_one=compile_one, jobs=jobs, a=SimpleNamespace(jobs=4)) + with self.assertRaisesRegex(OSError, "cannot be written"): + self.run_jobs() + finally: + concurrent.futures.ThreadPoolExecutor.submit = real_submit + self.assertTrue(gate.stopped, "the gate was not told the build is stopping") + # (the raising compile's own lease is free before the exception reaches the loop: the waiter that takes it in + # between is the one compile that may still start. Before: all three) + self.assertLessEqual(len([s for s in self.started if s.startswith("compiling")]), 1, self.started) + self.assertEqual(len(gate.running), 0, "a stopped compile was left holding a lease") + waiting.set() + + def test_a_stopped_gate_gives_no_lease(self): + Stopped = self.scope["CompileStopped"] + gate = self.scope["MemoryGate"]() + gate.floor = 1 + self.scope.update(mem_need=lambda src: 400, free_mb=lambda: 0, rss_mb=lambda p: 0) + first = gate.enter("a.c") # (under the floor: at once, as before) + self.assertEqual(len(gate.running), 1) + raised = [] + + def waiter(): + try: + gate.enter("b.c") + except Stopped: + raised.append(1) + t = threading.Thread(target=waiter) + t.start() + time.sleep(0.1) + self.assertTrue(t.is_alive(), "the waiter did not wait at the gate") + t0 = time.time() + gate.stop() + t.join(5) + self.assertFalse(t.is_alive()) + self.assertLess(time.time() - t0, 0.9, "the waiter was not woken: it waited out its second") + self.assertEqual(raised, [1]) + with self.assertRaises(Stopped): # (and one that comes after) + gate.enter("c.c") + self.assertEqual(len(gate.running), 1) + gate.leave(first) + self.assertEqual(len(gate.running), 0) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", type=Path, default=SOURCE) + args, remaining = parser.parse_known_args() + SOURCE = args.source.resolve() + unittest.main(argv=[__file__, *remaining]) diff --git a/tests/test_compute_pin.py b/tests/test_compute_pin.py new file mode 100644 index 0000000..cbb45e7 --- /dev/null +++ b/tests/test_compute_pin.py @@ -0,0 +1,214 @@ +"""Compute programs held to 32 lanes only where they can tell (gpu/vk/compute_pin.h; no game, no GPU). + +(Triforce, 11-22, the performance audit's PIPE-52, after Source A's and Source B's.) Every compute pipeline asked for +subgroups of 32 lanes. Mesa's Intel driver compiles such a program for 32 lanes only, minutes and gigabytes on the large +ones; most programs cannot tell how many lanes run beside them. On Mesa's Intel driver only the ones that can are pinned +now; every other driver keeps "all", its pipelines exactly as they were. + +Compiles a small driver against the header and checks: + the mode unset: lanes on Mesa's Intel driver, all on any other; all / lanes / shared as written; an unknown value + is the driver's default; + all every program pinned, whatever it is; + lanes each of the six flags pins by itself (vote, lane mask, lane index, shuffles, FSWZADD, matrices); a + barrier of Subgroup scope pins, and so does a module nobody looked at (the default); shared memory pins at + 1, 32 invocations, not at 33 or 64; a program with none of these is free; a program without a stage is + pinned; + shared as lanes, and shared memory pins at 64 and 4096 invocations too; + the scan a made-up module with OpControlBarrier at Subgroup scope: found; at Workgroup scope: not; none: not; + a scope that is no constant, a cut-off module, a wrong magic word: counted as one (a doubt pins); + compute_entry: a GLCompute entry point yes, a Fragment one no, no entry point no. + +And reads the source: the four places that make a compute program's pipeline call pin_compute and no longer +pin_subgroup; build_graphics and the mesh fallback still call pin_subgroup; pin_compute with "all" is pin_subgroup with +the workgroup's width, before anything is asked or counted; the note is made in make_module between vk_shader.cpp's +"Outside the translation stamp" marks; and the switches table has the row. + + python -I -B tests/test_compute_pin.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +DRIVER = r''' +#include "compute_pin.h" +#include +#include + +using namespace compute_pin; +static int checks, fails; +static void want(const char* what, bool got, bool wanted) { + checks++; + if (got != wanted) fails++, printf("FAIL %s: %d, want %d\n", what, (int)got, (int)wanted); +} +// a program that sees nothing: its module looked at, no barrier, no shared memory +static Facts plain() { + Facts f; + f.has_stage = true, f.subgroup_barrier = false, f.invocations = 64; + return f; +} +static std::vector module(uint32_t model, int barrier_scope, bool constant_scope = true) { + std::vector m{0x07230203, 0x00010600, 0, 100, 0}; + const auto op = [&m](uint32_t code, std::initializer_list words) { + m.push_back((uint32_t)((words.size() + 1) << 16) | code); + for (uint32_t w : words) m.push_back(w); + }; + op(17, {1}); // OpCapability Shader + if (model != ~0u) op(15, {model, 4, 0x6e69616d, 0}); // OpEntryPoint %4 "main" + op(21, {2, 32, 0}); // %2 = OpTypeInt 32 0 + op(43, {2, 10, 2}); // %10 = OpConstant %2 2 (Workgroup) + op(43, {2, 11, 3}); // %11 = OpConstant %2 3 (Subgroup) + op(43, {2, 12, 0x100}); // %12 = semantics + if (!constant_scope) op(50, {2, 13, 0}); // %13 = OpSpecConstant: no plain constant + op(19, {3}); // %3 = OpTypeVoid + op(33, {5, 3}); // %5 = OpTypeFunction %3 + op(54, {3, 4, 0, 5}); // %4 = OpFunction + op(248, {6}); // OpLabel + if (barrier_scope) op(224, {(uint32_t)(constant_scope ? (barrier_scope == 3 ? 11 : 10) : 13), 10, 12}); // OpControlBarrier + op(253, {}); // OpReturn + op(56, {}); // OpFunctionEnd + return m; +} + +int main() { + // the mode + want("unset on Mesa's Intel driver: lanes", mode_for(nullptr, true) == Mode::Lanes, true); + want("unset on any other driver: all", mode_for(nullptr, false) == Mode::All, true); + for (int intel = 0; intel < 2; intel++) { + want("all as written", mode_for("all", intel) == Mode::All, true); + want("lanes as written", mode_for("lanes", intel) == Mode::Lanes, true); + want("shared as written", mode_for("shared", intel) == Mode::Shared, true); + want("an unknown value is the driver's default", mode_for("1", intel) == mode_for(nullptr, intel), true); + want("an empty value is the driver's default", mode_for("", intel) == mode_for(nullptr, intel), true); + } + // all: everything pinned + want("all: a plain program", needed(Mode::All, plain()), true); + want("all: no stage", needed(Mode::All, Facts{}), true); + // lanes and shared + for (const Mode m : {Mode::Lanes, Mode::Shared}) { + want("a plain program is free", needed(m, plain()), false); + want("no stage: pinned", needed(m, Facts{}), true); + Facts unknown; // (a stage, but its module nobody looked at) + unknown.has_stage = true, unknown.invocations = 64; + want("a module nobody looked at: pinned", needed(m, unknown), true); + bool Facts::* const flags[] = {&Facts::vote, &Facts::mask, &Facts::lane_index, &Facts::shuffles, &Facts::fswzadd, + &Facts::matrices, &Facts::subgroup_barrier}; + for (bool Facts::* flag : flags) { + Facts f = plain(); + f.*flag = true; + want("each flag pins by itself", needed(m, f) && sees_lanes(f), true); + } + want("a plain program sees no lanes", sees_lanes(plain()), false); + for (const uint64_t n : {1ull, 32ull, 33ull, 64ull, 4096ull}) { + Facts f = plain(); + f.shared_memory = 256, f.invocations = n; + want("shared memory by the workgroup's invocations", needed(m, f), m == Mode::Shared || n <= 32); + f.shared_memory = 0; + want("no shared memory: the workgroup's size alone pins nothing", needed(m, f), false); + } + } + // the scan of a module + const auto barrier = [](const std::vector& m) { return subgroup_barrier(m.data(), m.size()); }; + want("a barrier of Subgroup scope", barrier(module(5, 3)), true); + want("a barrier of Workgroup scope", barrier(module(5, 2)), false); + want("no barrier", barrier(module(5, 0)), false); + want("a scope that is no constant: a doubt", barrier(module(5, 3, false)), true); + std::vector cut = module(5, 0); + cut.pop_back(); + cut.back() = (9u << 16) | 253; // (an instruction longer than what is left) + want("a cut-off module: a doubt", barrier(cut), true); + std::vector wrong = module(5, 0); + wrong[0] = 0; + want("a wrong magic word: a doubt", barrier(wrong), true); + want("no module: a doubt", subgroup_barrier(nullptr, 0), true); + const auto compute = [](const std::vector& m) { return compute_entry(m.data(), m.size()); }; + want("a GLCompute entry point", compute(module(5, 0)), true); + want("a Fragment entry point", compute(module(4, 0)), false); + want("no entry point", compute(module(~0u, 0)), false); + want("a wrong magic word is no compute module", compute(wrong), false); + if (fails) printf("compute pin: FAILED (%d of %d checks)\n", fails, checks); + else printf("compute pin: all %d checks passed\n", checks); + return fails != 0; +} +''' + + +def check_source(): + draw = (ROOT / "gpu/vk/vk_draw.cpp").read_text(encoding="utf-8") + at = draw.find("void pin_compute(VkPipelineShaderStageCreateInfo& st, const Program& p) {") + if at < 0: + raise SystemExit("vk_draw.cpp has no pin_compute") + pin = draw[at:draw.index("\n}\n", at)] + first = "if (mode == compute_pin::Mode::All) return pin_subgroup(st, p.workgroup[0]);" + if first not in pin or min(pin.find(n) for n in ("compute_pin::Facts f;", "st_pin_free++", "st_pin_pinned++")) < pin.index(first): + raise SystemExit("pin_compute: with RX_VK_COMPUTE_PIN=all it is not pin_subgroup as it was, before anything is asked or counted") + for needle in ("f.vote = i.uses_subgroup_vote", "f.mask = i.uses_subgroup_mask", "f.lane_index = i.uses_subgroup_invocation_id", + "f.shuffles = i.uses_subgroup_shuffles", "f.fswzadd = i.uses_fswzadd", "f.matrices = i.uses_cooperative_matrix", + "f.subgroup_barrier = !compute_lanes_free(st.module);", "f.shared_memory = p.shared_memory;", + "f.invocations = (uint64_t)p.workgroup[0] * p.workgroup[1] * p.workgroup[2];", + "if (!compute_pin::needed(mode, f)) {"): + if needle not in pin: + raise SystemExit("pin_compute does not fill the rule's facts: " + needle) + if pin.index("if (!compute_pin::needed(mode, f)) {") > pin.rindex("pin_subgroup(st, p.workgroup[0]);"): + raise SystemExit("pin_compute pins before it asks the rule") + # the four makers of a compute program's pipeline, and the two that are not asked + makers = (("bool prepare_compute_pipeline(Program& p) {", "pin_compute(ci.stage, p);"), + ("VkComputePipelineCreateInfo compute_create_info(const Program& q) {", "pin_compute(ci.stage, q);")) + for start, call in makers: + body = draw[draw.index(start):] + if call not in body[:body.index("\n}\n")]: + raise SystemExit(start + " does not call pin_compute") + if draw.count("pin_compute(ci.stage, p);") != 3 or draw.count("pin_compute(ci.stage, q);") != 1: + raise SystemExit("wanted pin_compute at the four makers of a compute program's pipeline") + if "pin_subgroup(ci.stage," in draw: + raise SystemExit("a compute program's pipeline is still pinned by pin_subgroup directly") + if draw.count(" pin_subgroup(si);\n") != 2: + raise SystemExit("build_graphics and the mesh fallback's stages no longer call pin_subgroup") + if "compute_pin_mode() != compute_pin::Mode::All)\n logf(\"vk: precompile: compute pipelines held to 32 lanes" not in draw: + raise SystemExit("the precompile does not log the pinned and free counts, only where the rule is asked") + + shader = (ROOT / "gpu/vk/vk_shader.cpp").read_text(encoding="utf-8") + out_mark, in_mark = "// ---- (Triforce) Outside the translation stamp", "// ---- (Triforce) Inside the translation stamp again" + lo = shader.index(out_mark) + hi = shader.index(in_mark, lo) + for needle in ("void lanes_note(VkShaderModule m, const std::vector& code) {", + "if (!m || compute_pin_mode() == compute_pin::Mode::All) return;", + "lanes_note(m, code);", "compute_pin::Mode compute_pin_mode() {", "bool compute_lanes_free(VkShaderModule m) {", + 'compute_pin::mode_for(env, dev.driver_id == VK_DRIVER_ID_INTEL_OPEN_SOURCE_MESA)'): + found = shader.find(needle) + if not lo < found < hi: + raise SystemExit("vk_shader.cpp: not between the first pair of translation stamp marks: " + needle) + if shader.count("compute_pin") != shader[lo:hi].count("compute_pin") or shader.count("lanes_") != shader[lo:hi].count("lanes_"): + raise SystemExit("vk_shader.cpp names the rule outside the marks: the translation stamp would move") + make = shader[shader.index("VkShaderModule make_module("):hi] + if make.count("lanes_note(m, code);") != 2 or make.index("exact_note(m, code);") > make.index("lanes_note(m, code);"): + raise SystemExit("make_module does not note each module it makes") + header = (ROOT / "gpu/vk/vk_shader.h").read_text(encoding="utf-8") + if '#include "compute_pin.h"' not in header or "compute_pin::Mode compute_pin_mode();" not in header: + raise SystemExit("vk_shader.h does not declare compute_pin_mode") + docs = (ROOT / "docs/development.md").read_text(encoding="utf-8") + if "| `RX_VK_COMPUTE_PIN=all`, `lanes` or `shared` |" not in docs or "`gpu/vk/compute_pin.h`" not in docs: + raise SystemExit("docs/development.md: the switches table has no row for RX_VK_COMPUTE_PIN") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="g++") + args = parser.parse_args() + check_source() + with tempfile.TemporaryDirectory(prefix="compute-pin-") as tmp: + cpp, exe = Path(tmp) / "compute_pin_test.cpp", Path(tmp) / "compute-pin.exe" + cpp.write_text(DRIVER, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-Wall", "-Wextra", "-Werror", "-I" + str(ROOT / "gpu/vk"), str(cpp), "-o", str(exe)], + check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + print(r.stdout.rstrip()) + if r.returncode: + raise SystemExit("FAIL: the compute pin's checks failed") + + +if __name__ == "__main__": + main() diff --git a/tests/test_dead_count.py b/tests/test_dead_count.py new file mode 100644 index 0000000..892de47 --- /dev/null +++ b/tests/test_dead_count.py @@ -0,0 +1,91 @@ +"""What destroying the dead images costs a present is counted (gpu/vk/dead_count.h, vk_texture.cpp texture_frame; no game, +no GPU). + +(Triforce, 11-21, the performance audit's THREAD-21, its first step.) texture_frame destroys every dead image the GPU is +done with in the middle of the present being recorded. The audit's port bounds that to a share a present, and its own +proof asks for the cost first: under 3 ms in every present, the bounded step is not built. This is the count. + +This compiles dead_count.h alone: a present's loop adds to the totals, the worst present is kept with what it destroyed +and left, a present of 2 ms or more gets a line and one of 3 ms or more is counted apart, the totals' line is due every +600 presents and only when a present was counted, and reset starts afresh. + +And reads the source: texture_frame times its loop, counts a destroyed image and a pooled one apart, and the loop itself +is as it was (the compaction's break, the GPU's progress, the pool, the four destroys). + + python -I -B tests/test_dead_count.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +PROGRAM = r""" +#include +#include +#include "dead_count.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +int main() { + dead_count::Counts c; + CHECK(!c.due(600)); // (nothing counted yet: no line) + CHECK(!c.note(3, 5, 0.4, 7)); + CHECK(c.presents == 1 && c.destroyed == 3 && c.pooled == 5 && c.worst_ms == 0.4 && c.worst_destroyed == 3 && c.worst_left == 7); + CHECK(!c.note(0, 0, 0.0, 0) && c.presents == 2 && c.worst_ms == 0.4); + CHECK(!c.note(40, 0, 1.99, 2) && c.over_say == 0 && c.worst_ms == 1.99 && c.worst_destroyed == 40); + CHECK(c.note(300, 12, 2.0, 9) && c.over_say == 1 && c.over_worth == 0); // (2 ms: a line of its own) + CHECK(c.note(700, 0, 18.6, 0) && c.over_say == 2 && c.over_worth == 1); // (3 ms or more: the audit's line) + CHECK(c.worst_ms == 18.6 && c.worst_destroyed == 700 && c.worst_left == 0); + CHECK(!c.note(1, 1, 0.1, 0) && c.worst_ms == 18.6); // (a smaller one after it does not replace it) + CHECK(c.presents == 6 && c.destroyed == 1044 && c.pooled == 18); + CHECK(c.ms > 23.08 && c.ms < 23.10); + CHECK(c.due(600) && c.due(1200) && !c.due(601) && !c.due(599)); + c.reset(); + CHECK(c.presents == 0 && c.destroyed == 0 && c.worst_ms == 0 && c.over_say == 0 && !c.due(600)); + CHECK(dead_count::kSayMs == 2.0 && dead_count::kWorthMs == 3.0 && dead_count::kReport == 600); + return fails ? 1 : 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + tex = (VK / "vk_texture.cpp").read_text(encoding="utf-8").replace("\r\n", "\n") + start = tex.index("void texture_frame(uint64_t completed) {") + frame = tex[start:tex.index("\n}\n", start)] + check('#include "dead_count.h"' in tex, "vk_texture.cpp does not use dead_count.h") + at = 0 + for part in ("static dead_count::Counts dead_ct;", "const auto dead_t0 = std::chrono::steady_clock::now();", + "for (auto it = dead.begin(); it != dead.end();) {", "if (compact.in_pass) break;", + "if (it->value > completed) {", "pool_put(*it);", "dead_pooled++;", + "for (VkImageView v : it->views) vkDestroyImageView(dev.device, v, nullptr);", + "destroy_image(it->image, it->allocation);", "if (it->buffer) destroy_buffer(it->buffer, it->buffer_alloc);", + "for (VkDeviceMemory m : it->memory) vkFreeMemory(dev.device, m, nullptr);", "dead_destroyed++;", + "dead_ct.note(dead_destroyed, dead_pooled, dead_ms, dead.size())", "if (dead_ct.due(frame_no)) {", + "dead_ct.reset();", "pool_trim();"): + at = frame.find(part, at) + check(at >= 0, "texture_frame: the dead images' loop is not timed and counted as it was (missing, or out of order: " + part + ")") + at += len(part) + + with tempfile.TemporaryDirectory(prefix="dead-count-") as tmp: + c, exe = Path(tmp) / "dead.cpp", Path(tmp) / "dead.exe" + c.write_text(PROGRAM, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(VK), str(c), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + check(r.returncode == 0, "the counts as made:\n" + r.stdout) + print("PASS: what destroying the dead images costs a present is counted") + + +if __name__ == "__main__": + main() diff --git a/tests/test_dead_pool_now.py b/tests/test_dead_pool_now.py new file mode 100644 index 0000000..2a40c33 --- /dev/null +++ b/tests/test_dead_pool_now.py @@ -0,0 +1,55 @@ +"""A dead render target's image goes to the pool in the pass that lets it go (gpu/vk/vk_texture.cpp texture_sweep; no game, +no GPU). + +(Triforce, 11-21, the performance audit's TEX-30, after Source A's.) A render target let go waited in the dead list until +the next present's texture_frame put its image in the pool: a target made in this present's recording made a new image +meanwhile. texture_sweep now pools the ones the GPU is done with right after it lets them go. + +Reads the source: after the pass has erased the textures it let go, the dead list's entries with a pool key whose value +the GPU has completed go to pool_put, never while a compaction pass is open, and only with the switch on; texture_frame's +own loop still pools and destroys the rest, and the pool is still trimmed each present; the switches are in the table. + + python -I -B tests/test_dead_pool_now.py +""" +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def in_order(text, parts, why): + at = 0 + for p in parts: + at = text.find(p, at) + check(at >= 0, why + " (missing, or out of order: " + p + ")") + at += len(p) + + +def main(): + tex = (VK / "vk_texture.cpp").read_text(encoding="utf-8").replace("\r\n", "\n") + sweep = tex[tex.rindex("void texture_sweep("):] + sweep = sweep[:sweep.index("\n}\n")] + in_order(sweep, ["destroy_later(*tex);", "it = textures.erase(it);", + 'static const bool dead_pool_now = !(getenv("RX_VK_DEAD_POOL_NOW") && !atoi(getenv("RX_VK_DEAD_POOL_NOW")));', + "if (dead_pool_now && !compact.in_pass)", "for (auto it = dead.begin(); it != dead.end();) {", + "if (it->pool_key && it->value <= completed) {", "pool_put(*it);", "it = dead.erase(it);", "++it;"], + "texture_sweep does not pool the dead targets' images the GPU is done with after it lets them go, or does so in a " + "compaction pass, for an image with no pool key, or for one the GPU still uses") + frame = tex[tex.index("void texture_frame(uint64_t completed) {"):] + frame = frame[:frame.index("\n}\n")] + in_order(frame, ["for (auto it = dead.begin(); it != dead.end();) {", "if (compact.in_pass) break;", "if (it->value > completed) {", + "if (it->pool_key) {", "pool_put(*it);", "destroy_image(it->image, it->allocation);", "pool_trim();"], + "texture_frame no longer pools and destroys what is left of the dead list, or no longer trims the pool each present") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`RX_VK_DEAD_SUM=0`, `RX_VK_DEAD_POOL_NOW=0`, `RX_VK_DEAD_MIN_UNUSED=3`" in dev, + "docs/development.md: TEX-30's switches are not in the switches table") + print("PASS: a dead render target's image goes to the pool in the pass that lets it go") + + +if __name__ == "__main__": + main() diff --git a/tests/test_desc_counts.py b/tests/test_desc_counts.py new file mode 100644 index 0000000..e622c4d --- /dev/null +++ b/tests/test_desc_counts.py @@ -0,0 +1,217 @@ +"""A program's descriptor counts kept with the program (gpu/vk/desc_counts.h; vk_shader.h Program::desc_n, vk_draw.cpp +build_descriptors; no game, no GPU). + +(Triforce, 11-22, the performance audit's DESC-22, after Source A's.) build_descriptors summed the program's buffers, +images and texel views at every build, six lists a stage, to size three reserve() calls. The sums are fixed once the +program is translated, so they are kept packed in one 32-bit word on the program and read at every later build. + +Compiles a driver against the header: the round trip of every count up to 1023; more than 1023 of a kind packs to the +unset word; the unset and the never words hold no counts and leave the outputs alone; a build stores only with the +rule on, the program ready and the word still unset; the guard fires only for a build that did not walk and had a +vector move. + +Runs a model of the build over the real rule with a std::vector as build_descriptors uses it: by kept counts the +vector never moves; with a count kept short it does, the guard sees it, the word becomes never and the next build walks. + +Reads the source: + - vk_shader.h: desc_n right after ready and before stages, mutable, ~0u, and the static_assert that keeps it in the + padding (sizeof(Program) is in the program cache's stamp; this change must not move it); + - build_descriptors: the word read only with the switch on, the walk only when it holds no counts, the store after + the walk and by the rule (ready, acquire), the three data() pointers taken after the reserve and compared at the + end, the rebuild; RX_VK_DESC_COUNTS read once and the lab's knob; + - vk_shader.cpp, a stamp file, does not name the field; the switches table has the row. + + python -I -B tests/test_desc_counts.py [--cxx clang++] +""" +import argparse +import re +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +DRIVER = r''' +#include "desc_counts.h" +#include +#include + +using namespace mkwvk::desc_counts; +static int checks, fails; +static void want(const char* what, bool ok) { + checks++; + if (!ok) fails++, printf("FAIL %s\n", what); +} + +// build_descriptors' use of the rule, on one vector of each kind: reserve by the word or by the walk, push what the +// program really has, store, guard +struct Model { + uint32_t word = kUnset; + size_t bufs, imgs, views; // (what the program has) + std::vector b, i, v; + int walks = 0, rebuilds = 0; + bool build(bool on, bool ready) { + b.clear(), i.clear(), v.clear(); + size_t nb = 0, ni = 0, nv = 0; + const uint32_t counted = on ? word : kUnset; + const bool walked = !unpack(counted, &nb, &ni, &nv); + if (walked) { + walks++, nb = bufs, ni = imgs, nv = views; + if (stores(on, ready, word)) word = pack(nb, ni, nv); + } + b.reserve(nb), i.reserve(ni), v.reserve(nv); + const void *b0 = b.data(), *i0 = i.data(), *v0 = v.data(); + for (size_t k = 0; k < bufs; k++) b.push_back(1); + for (size_t k = 0; k < imgs; k++) i.push_back(1); + for (size_t k = 0; k < views; k++) v.push_back(1); + if (moved(walked, b0, b.data(), i0, i.data(), v0, v.data())) { + word = kNever, rebuilds++; + return build(on, ready); + } + return walked; + } +}; + +int main() { + want("on when unset", on(nullptr) && on("1") && !on("0")); + size_t a = 7, b = 8, c = 9; + for (size_t n = 0; n <= kMost; n++) { + const size_t x = n, y = (n * 7) % (kMost + 1), z = kMost - n; + a = b = c = 5000; + if (!unpack(pack(x, y, z), &a, &b, &c) || a != x || b != y || c != z) { + want("round trip", false); + break; + } + } + want("round trip of every count up to 1023", true); + want("all three at the most", holds(pack(1023, 1023, 1023)) && pack(1023, 1023, 1023) == 0x3FFFFFFFu); + want("none at all", holds(pack(0, 0, 0)) && pack(0, 0, 0) == 0); + want("over 1023 buffers: unset", pack(1024, 1, 1) == kUnset); + want("over 1023 images: unset", pack(1, 1024, 1) == kUnset); + want("over 1023 texel views: unset", pack(1, 1, 1024) == kUnset && pack(1, 1, (size_t)1 << 40) == kUnset); + a = 7, b = 8, c = 9; + want("the unset word holds nothing", !holds(kUnset) && !unpack(kUnset, &a, &b, &c) && a == 7 && b == 8 && c == 9); + want("nor the never word", !holds(kNever) && !unpack(kNever, &a, &b, &c) && a == 7 && b == 8 && c == 9 && kNever != kUnset); + want("stored: on, ready, unset", stores(true, true, kUnset)); + want("not while it is translated", !stores(true, false, kUnset)); + want("not with the switch off", !stores(false, true, kUnset)); + want("not over the never word", !stores(true, true, kNever)); + want("not over counts", !stores(true, true, pack(1, 2, 3))); + int p0 = 0, p1 = 0; + want("the guard: nothing moved", !moved(false, &p0, &p0, &p0, &p0, &p0, &p0)); + want("the guard: buffers moved", moved(false, &p0, &p1, &p0, &p0, &p0, &p0)); + want("the guard: images moved", moved(false, &p0, &p0, &p0, &p1, &p0, &p0)); + want("the guard: texel views moved", moved(false, &p0, &p0, &p0, &p0, &p0, &p1)); + want("the guard: a walked build is not asked", !moved(true, &p0, &p1, &p0, &p1, &p0, &p1)); + + Model m; + m.bufs = 12, m.imgs = 30, m.views = 2; + want("a program still translated walks and stores nothing", m.build(true, false) && m.word == kUnset && m.walks == 1); + want("ready: its first build walks and stores", m.build(true, true) && m.word == pack(12, 30, 2) && m.walks == 2); + for (int k = 0; k < 50; k++) m.build(true, true); + want("then no build walks and none moves", m.walks == 2 && m.rebuilds == 0 && m.b.size() == 12 && m.i.size() == 30 && m.v.size() == 2); + want("the switch off: every build walks", m.build(false, true) && m.build(false, true) && m.walks == 4 && m.word == pack(12, 30, 2)); + Model big; + big.bufs = 3, big.imgs = 2000, big.views = 0; + for (int k = 0; k < 5; k++) big.build(true, true); + want("more than 1023 of a kind: every build walks", big.walks == 5 && big.word == kUnset && big.rebuilds == 0); + Model bad; // (a word kept short: what the guard is for. Fresh vectors, so the short reserve is all the room there is) + bad.bufs = 400, bad.imgs = 300, bad.views = 200; + bad.word = pack(4, 300, 200); + want("a count kept short: the build moves, is made again walked", bad.build(true, true) && bad.rebuilds == 1 && bad.word == kNever); + for (int k = 0; k < 5; k++) bad.build(true, true); + want("and walks from then on, the word never set again", bad.walks == 6 && bad.rebuilds == 1 && bad.word == kNever && bad.b.size() == 400); + if (fails) printf("desc counts: FAILED (%d of %d checks)\n", fails, checks); + else printf("desc counts: all %d checks passed\n", checks); + return fails != 0; +} +''' + + +def block(text, marker): + start = text.index(marker) + at = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[at] == "{") - (text[at] == "}") + at += 1 + return text[start:at] + + +def need(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def ordered(text, parts, why): + last = 0 + for p in parts: + k = text.find(p, last) + need(k >= 0, why + " (missing, or out of order: " + p + ")") + last = k + len(p) + + +def check_source(): + header = (ROOT / "gpu/vk/vk_shader.h").read_text(encoding="utf-8") + prog = block(header, "struct Program {") + ordered(prog, ["std::atomic ready{false};", "mutable std::atomic desc_n{~0u};", "std::vector stages;"], + "vk_shader.h: Program::desc_n is not right after ready and before stages") + between = prog[prog.index("std::atomic ready{false};"):prog.index("mutable std::atomic desc_n{~0u};")].splitlines()[1:] + need(all(not line.strip() or line.strip().startswith("//") for line in between), + "vk_shader.h: a field came between Program::ready and Program::desc_n") + need("MUST NOT GROW THE STRUCT" in prog, "vk_shader.h: the field's comment no longer says it must not grow the struct") + after = header[header.index(prog) + len(prog):][:700] + need(re.search(r"static_assert\(sizeof\(std::atomic\) == 4 && offsetof\(Program, desc_n\) \+ 4 <= offsetof\(Program, stages\) &&\s*" + r"offsetof\(Program, stages\) == 16,", after), + "vk_shader.h: the static_assert that keeps desc_n in the padding before stages is gone") + need("#include " in header, "vk_shader.h uses offsetof without ") + draw = (ROOT / "gpu/vk/vk_draw.cpp").read_text(encoding="utf-8") + need('#include "desc_counts.h"' in draw, "vk_draw.cpp does not include desc_counts.h") + on = block(draw, "bool desc_counts_on() {") + need('static const bool env = desc_counts::on(getenv("RX_VK_DESC_COUNTS"));' in on and 'return env && !lab_ab::on("descwalk");' in on, + "the switch is not RX_VK_DESC_COUNTS read once, with the lab's descwalk knob") + build = block(draw, "void build_descriptors(") + ordered(build, ["size_t nbuf = 0, nimg = 0, nview = 0;", "const bool counts_on = desc_counts_on();", + "const uint32_t counted = counts_on ? p.desc_n.load(std::memory_order_relaxed) : desc_counts::kUnset;", + "const bool walked = !desc_counts::unpack(counted, &nbuf, &nimg, &nview);", "if (walked) {", + "for (const Stage& st : p.stages) {", "st.info.constant_buffer_descriptors) nbuf += d.count;", + "st.info.image_descriptors) nimg += d.count;", + "if (desc_counts::stores(counts_on, p.ready.load(std::memory_order_acquire), p.desc_n.load(std::memory_order_relaxed)))", + "p.desc_n.store(desc_counts::pack(nbuf, nimg, nview), std::memory_order_relaxed);", + "dwr.bufs.reserve(nbuf), dwr.imgs.reserve(nimg), dwr.views.reserve(nview);", + "const void* const bufs_at = dwr.bufs.data();", "const void* const imgs_at = dwr.imgs.data();", + "const void* const views_at = dwr.views.data();", "desc_keep::prog_of(kept_descs(), &p, nbuf + nimg + nview)", + "if (desc_counts::moved(walked, bufs_at, dwr.bufs.data(), imgs_at, dwr.imgs.data(), views_at, dwr.views.data())) {", + "p.desc_n.store(desc_counts::kNever, std::memory_order_relaxed);", "build_descriptors(s, p, keep, scene);"], + "build_descriptors does not read the kept counts, walk only without them, store by the rule, and guard its vectors") + need(build.count("for (const auto& d : st.info.") == 6, "build_descriptors sums other than its six lists a stage") + need(build.rstrip().endswith("build_descriptors(s, p, keep, scene);\n }\n}"), "the guard is not the last thing build_descriptors does") + named = lambda text: len(re.findall(r"(? [--cxx clang++] +""" +import argparse +import subprocess +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +RUN_ALL_ARGS = ["--cxx", "{cxx}", "--headers", "{vulkan_headers}"] + +DRIVER = r''' +#include "desc_pool_grow.h" +#include + +using namespace mkwvk::desc_pool_grow; +static int checks, fails; +static void want(const char* what, bool ok) { + checks++; + if (!ok) fails++, printf("FAIL %s\n", what); +} +struct Write { int descriptorType; unsigned descriptorCount; }; + +int main() { + want("on when unset", on(nullptr)); + want("on at 1", on("1")); + want("off at 0", !on("0")); + want("a new slot: half with the rule", first_shift(true) == 1); + want("a new slot: whole without it", first_shift(false) == 0); + // the sizes: whole as before, half of each at shift 1, an n-th under the divider, never none + want("whole sets", count(kSets, 0, 1) == 8192); + want("half sets", count(kSets, 1, 1) == 4096); + static const unsigned whole[kKinds] = {32768, 16384, 32768, 4096, 2048, 2048}; + for (int k = 0; k < kKinds; k++) { + want("a whole pool's descriptors are as they were", count(kWhole[k], 0, 1) == whole[k]); + want("half of each type", count(kWhole[k], 1, 1) == whole[k] / 2); + want("divided as before when whole", count(kWhole[k], 0, 4) == whole[k] / 4); + want("the divider and the half together", count(kWhole[k], 1, 32) == whole[k] / 64); + want("never none", count(kWhole[k], 8, 32) >= 1 && count(kWhole[k], 40, 32) >= 1 && count(kWhole[k], 1, 0) == whole[k] / 2); + } + // made again whole: only a pool that is not whole, of a slot whose last recording had extra pools + want("a half pool that ran out", remake(1, 1) && remake(1, 3)); + want("a half pool that did not", !remake(1, 0)); + want("a whole pool that ran out", !remake(0, 1)); + want("a whole pool that did not", !remake(0, 0)); + // which type counts where (VkDescriptorType's numbers) + want("kinds", kind(6) == kUniform && kind(7) == kStorage && kind(1) == kSampled && kind(3) == kStorageImage && + kind(4) == kUniformTexel && kind(5) == kStorageTexel && kind(0) == kKinds && kind(2) == kKinds && kind(1000) == kKinds); + // a recording's counts and the peaks + Took a, b, most; + const Write w1[] = {{6, 4}, {1, 9}, {1, 2}, {0, 5}}; + const Write w2[] = {{7, 3}, {5, 1}}; + took(a, w1, 4), took(a, w2, 2), took(b, w1, 4); + want("sets counted", a.sets == 2 && b.sets == 1); + want("descriptors by type", a.n[kUniform] == 4 && a.n[kSampled] == 11 && a.n[kStorage] == 3 && a.n[kStorageTexel] == 1 && + a.n[kStorageImage] == 0 && a.n[kUniformTexel] == 0); + peak(most, b), peak(most, a), peak(most, b); + want("the peaks are the most of each", most.sets == 2 && most.n[kSampled] == 11 && most.n[kUniform] == 4 && most.n[kStorage] == 3); + Took big; + big.sets = 289, big.n[kSampled] = 17251; // (Source A's busiest recording: fits a whole pool, not half of the sampled images) + want("a busy recording outgrows the half", fits(big, 0, 1) && !fits(big, 1, 1)); + Took small; + small.sets = 289, small.n[kSampled] = 9000; + want("an ordinary one fits it", fits(small, 1, 1) && !fits(small, 1, 4)); + if (fails) printf("desc pool grow: FAILED (%d of %d checks)\n", fails, checks); + else printf("desc pool grow: all %d checks passed\n", checks); + return fails != 0; +} +''' + + +def block(text, marker): + start = text.index(marker) + at = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[at] == "{") - (text[at] == "}") + at += 1 + return text[start:at] + + +def need(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def ordered(text, parts, why): + at = [text.find(p) for p in parts] + need(min(at) >= 0 and at == sorted(at), why) + + +def check_source(): + draw = (ROOT / "gpu/vk/vk_draw.cpp").read_text(encoding="utf-8") + need('#include "desc_pool_grow.h"' in draw, "vk_draw.cpp does not include desc_pool_grow.h") + need('static const bool on = desc_pool_grow::on(getenv("RX_VK_DESC_POOL_GROW"));' in block(draw, "bool pool_grow_on() {"), + "the switch is not RX_VK_DESC_POOL_GROW, read once") + res = block(draw, "struct RecordingRes {") + need("uint32_t shift = 0;" in res and "desc_pool_grow::Took took;" in res, "RecordingRes does not keep its pool's shift and its counts") + pool = block(draw, "VkDescriptorPool make_pool(uint32_t shift = 0) {") + need("ci.maxSets = grow::count(grow::kSets, shift, div);" in pool and "grow::count(grow::kWhole[kind], shift, div)" in pool, + "make_pool does not size the pool by the header with its shift") + need('getenv("RX_VK_DESC_POOL_DIV")' in pool, "RX_VK_DESC_POOL_DIV no longer divides") + for gone in ("32768", "8192 / div"): + need(gone not in pool.split("static const uint32_t div")[1], "make_pool still has a size of its own: " + gone) + bind = block(draw, "bool bind_descriptors(") + ordered(bind, ["if (vkAllocateDescriptorSets(dev.device, &ai, &set) != VK_SUCCESS) {", "const VkDescriptorPool pool = make_pool();", + "cur->extra.push_back(pool);", "pool_took(dwr.writes.data(), dwr.writes.size());", "set_write_template(set, p)"], + "bind_descriptors: the extra pool is not a whole one, or the set is not counted once it is allocated") + need("desc_pool_grow::took(cur->took, writes, n);" in block(draw, "void pool_took("), "pool_took does not count into the recording's own") + reset = block(draw, "void slot_reset_locked(RecordingRes& r) {") + ordered(reset, ["if (desc_pool_grow::remake(r.shift, r.extra.size())) {", "if (const VkDescriptorPool made = make_pool(0)) {", + "vkDestroyDescriptorPool(dev.device, r.pool, nullptr);", "r.pool = made, r.shift = 0, whole = true;", "pools_whole++;", + "if (!whole) vkResetDescriptorPool(dev.device, r.pool, 0);", "pools_added += r.extra.size();", + "for (VkDescriptorPool p : r.extra) vkDestroyDescriptorPool(dev.device, p, nullptr);", "r.extra.clear();"], + "slot_reset_locked does not make the whole pool before it lets the half one go, reset otherwise, and count") + begin = block(draw, "void draw_begin_recording(int slot, VkCommandBuffer cmd, uint64_t value) {") + ordered(begin, ["if (!r.pool) {", "r.shift = desc_pool_grow::first_shift(pool_grow_on());", "r.pool = make_pool(r.shift);", + "} else if (r.reset_ahead) {", "desc_pool_grow::peak(pool_most, r.took);", "r.took = {};", + 'logf("vk: the recording streams: ', "logf(\"vk: the recording streams' descriptor pools: ", "pools_whole", + "pools_added", "pool_most.sets", "pool_most.n[desc_pool_grow::kSampled]"], + "draw_begin_recording does not make a new slot's pool at the first shift, or its line lost the counts") + docs = (ROOT / "docs/development.md").read_text(encoding="utf-8") + for name in ("`RX_VK_DESC_POOL_GROW=0`", "`RX_VK_DESC_POOL_DIV=<1..32>`", "`RX_VK_STREAM_MB=`", "`gpu/vk/desc_pool_grow.h`"): + need(name in docs, "docs/development.md: the switches table has no " + name) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="clang++") + parser.add_argument("--headers", type=Path, required=True) + args = parser.parse_args() + with tempfile.TemporaryDirectory(prefix="desc-pool-grow-") as tmp: + cpp, exe = Path(tmp) / "desc_pool_grow_test.cpp", Path(tmp) / "desc-pool-grow.exe" + cpp.write_text(DRIVER, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-Wall", "-Wextra", "-Werror", "-I" + str(ROOT / "gpu/vk"), str(cpp), "-o", str(exe)], + check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + print(r.stdout.rstrip()) + if r.returncode: + raise SystemExit("FAIL: the half pool rule's checks failed") + check_source() + for case in ("pool-grow", "pool-grow-off"): + subprocess.run([sys.executable, "-I", "-B", str(ROOT / "tests/test_recording_admission.py"), "--cxx", args.cxx, + "--headers", str(args.headers), "--case", case], check=True) + print("PASS: the slots' descriptor pools start at half and are whole after a recording ran out; RX_VK_DESC_POOL_GROW=0 as before") + + +if __name__ == "__main__": + main() diff --git a/tests/test_desc_sets.py b/tests/test_desc_sets.py index 990a6f6..13161c6 100644 --- a/tests/test_desc_sets.py +++ b/tests/test_desc_sets.py @@ -88,7 +88,7 @@ def fixture_of(source): # ((Triforce) a whole push tries the push template first, the write structs when it gives none; the changed bindings by theirs) full = bind[:bind.index("static thread_local std::vector changed")] if not re.search(r"if \(!push_write_template\(cmd, bp, p\)\)[^\n]*\n\s*vkCmdPushDescriptorSetKHR\(cmd, bp, p\.layout, 0, " - r"\(uint32_t\)dw\(\)\.writes\.size\(\), dw\(\)\.writes\.data\(\)\);", full): + r"\(uint32_t\)dwr\.writes\.size\(\), dwr\.writes\.data\(\)\);", full): raise SystemExit("a whole push does not try the push template first, with the write structs when it gives none") if "push_write_template(" in bind[len(full):]: raise SystemExit("a push of the changed bindings only goes through the push template") diff --git a/tests/test_dispatch_quiet.py b/tests/test_dispatch_quiet.py new file mode 100644 index 0000000..eac743a --- /dev/null +++ b/tests/test_dispatch_quiet.py @@ -0,0 +1,188 @@ +"""The leftover per-dispatch diagnostics quieted (gpu/vk/vk_draw.cpp dispatch, taau_key; vk_texture.cpp texture_frame; +no game, no GPU). + +(Triforce, 11-22, the performance audit's DESC-30, after Source A's and Source B's.) Every compute dispatch still paid for +diagnostics nobody had asked for: RX_VK_LOG_PUSH's key written out as text, taau_key's key written out and its list +walked as text (several times a dispatch), two clock reads and a sequentially consistent add around the program pick, and +a clock read for the first-pipeline timer when there was nothing to time. Every 600 presents texture_frame filled a list +only RX_VK_LOG_BAKED reads and sorted every texture to print eight. + +Cuts taau_key out of vk_draw.cpp and runs it both ways, the parsed prefixes and the text compare it was (kept for the +lab's "diagold" phases), over 16 lists and some 300,000 keys each: the same answer for every key, and the answers a +list must give (a prefix, a whole key, several entries, an entry that begins with '-', upper case, 17 digits, an empty +list, empty entries: none of those last ones ever match). + +Reads the source: + - taau_key: no snprintf before the parsed prefixes answer; the list's expression as it was (its reads of the + environment are counted by tests/test_taau_env_lookup.py); + - dispatch: no clock around program_for outside prof_full and the lab's knob, the pick's time taken from prof_t.exact + with a relaxed add; RX_VK_LOG_PUSH's key formatted only under its switch (or the knob); PipeClock's clock read only + when it times something (or the knob); + - program_pick, the picks' slow path, is still under PROF_EXACT: what the pick's time now stands on; + - texture_frame: RX_VK_LOG_BAKED read once, its list filled only for it, no whole sort outside the knob; the VRAM and + textures lines keep their text. + + python -I -B tests/test_dispatch_quiet.py [--cxx clang++] +""" +import argparse +import re +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +FIXTURE = r''' +#define _CRT_SECURE_NO_WARNINGS +#include +#include +#include +#include +#include +static const char *configured_keys, *no_f16; +static const char* fixture_getenv(const char* name) { + if (!strcmp(name, "RX_VK_TAAU_KEYS")) return configured_keys; + if (!strcmp(name, "RX_VK_NO_F16")) return no_f16; + return nullptr; +} +#define getenv fixture_getenv +namespace Shader { namespace Backend { namespace SPIRV { static bool mma_software = true; } } } +static bool diag_old_now; +static bool dispatch_diag_old() { return diag_old_now; } +/* CUT */ + +int main(int argc, char** argv) { + // (argv[1]: the list, "(unset)" for none set; argv[2]: "hardware" for a card that runs the network itself) + if (argc > 1 && strcmp(argv[1], "(unset)")) configured_keys = argv[1]; + if (argc > 2 && !strcmp(argv[2], "hardware")) Shader::Backend::SPIRV::mma_software = false; + std::vector keys = {0xfd029a317a9a4b94ull, 0x9dc98871eb7fc101ull, 0, ~0ull, 0xfd029a317a9a4b95ull, 0xfd029a317a9a4b84ull, + 0x0d029a317a9a4b94ull, 0xfd02000000000000ull, 0xfd01ffffffffffffull, 0x00000000000000fdull, + 0xabcdef0123456789ull, 0x0123456789abcdefull, 0xa000000000000000ull, 0x0a00000000000000ull}; + uint64_t x = 0x9E3779B97F4A7C15ull; + for (int i = 0; i < 300000; i++) { + x ^= x << 13, x ^= x >> 7, x ^= x << 17; + keys.push_back(x); + if (i % 3 == 0) keys.push_back((x & 0x0000ffffffffffffull) | 0xfd02000000000000ull); // (many under the usual prefix) + if (i % 7 == 0) keys.push_back(x >> (i % 61)); // (leading zeros) + } + size_t matched = 0, differ = 0; + for (uint64_t k : keys) { + diag_old_now = false; + const bool now = taau_key(k); + diag_old_now = true; + const bool was = taau_key(k); + matched += now, differ += now != was; + if (now != was && differ <= 4) printf("DIFFERS %016llx: parsed %d, text %d\n", (unsigned long long)k, (int)now, (int)was); + } + diag_old_now = false; + printf("%zu %zu %zu %d %d %d\n", keys.size(), matched, differ, (int)taau_key(0xfd029a317a9a4b94ull), (int)taau_key(0x9dc98871eb7fc101ull), + (int)taau_key(0)); + return differ != 0; +} +''' + +# the list, the card, and the answers for fd029a317a9a4b94, 9dc98871eb7fc101 and 0 +LISTS = [("(unset)", "software", (1, 0, 0)), ("(unset)", "hardware", (0, 0, 0)), ("-", "software", (0, 0, 0)), ("", "software", (0, 0, 0)), + ("fd029a317a9a4b94", "hardware", (1, 0, 0)), ("fd02", "software", (1, 0, 0)), ("f", "software", (1, 0, 0)), + ("fd029a317a9a4b94,9dc98871eb7fc101", "software", (1, 1, 0)), ("9dc9,-fd02", "software", (0, 1, 0)), + ("FD029A317A9A4B94", "software", (0, 0, 0)), ("fd029a317a9a4b940", "software", (0, 0, 0)), (",,fd,,0000,", "software", (1, 0, 1)), + ("0", "software", (0, 0, 1)), ("g,fd0x,9d c9, fd", "software", (0, 0, 0)), ("0000000000000000", "software", (0, 0, 1)), + ("-,fd029a317a9a4b9", "software", (1, 0, 0))] + + +def block(text, marker): + start = text.index(marker) + at = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[at] == "{") - (text[at] == "}") + at += 1 + return text[start:at] + + +def need(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def ordered(text, parts, why): + last = 0 + for p in parts: + k = text.find(p, last) + need(k >= 0, why + " (missing, or out of order: " + p + ")") + last = k + len(p) + + +def check_source(draw): + key = block(draw, "bool taau_key(uint64_t key) {") + ordered(key, ['static const char* keys = getenv("RX_VK_TAAU_KEYS") ? getenv("RX_VK_TAAU_KEYS")', + "static const std::vector prefixes = [] {", "if (!dispatch_diag_old()) {", + "if ((key & p.mask) == p.value) return true;", "return false;", "snprintf(hex, sizeof hex,"], + "taau_key does not answer from its list parsed once before any text is made") + need(key.count("snprintf(") == 1 and key.count("getenv(") == 3, "taau_key formats or reads the environment in another place") + need('return lab_ab::on("diagold");' in block(draw, "bool dispatch_diag_old() {"), "the lab's knob is not diagold") + disp = block(draw, "void dispatch(VkCommandBuffer cmd, NvnState& s, uint32_t x, uint32_t y, uint32_t z, uint64_t indirect) {") + ordered(disp, ["const bool pick_clocked = prof_full.load(std::memory_order_relaxed) != 0 || dispatch_diag_old();", + "const auto t_pick = pick_clocked ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};", + "const int64_t x_pick = prof_t.exact;", "const Program* cp = program_for(s, ProgramKind::Compute, types);", + "if (pick_clocked)", "st_disp_pick_ns +=", "else if (const int64_t dx = prof_t.exact - x_pick)", + "st_disp_pick_ns.fetch_add(dx, std::memory_order_relaxed);"], + "dispatch still reads a clock around every program pick, or lost the pick's time") + pick = disp[disp.index("const uint8_t types[8]{};"):disp.index("dclock.t_prog =")] + need(pick.count("steady_clock::now()") == 2 and "pick_clocked ?" in pick and pick.index("if (pick_clocked)") < pick.rindex("steady_clock::now()"), + "dispatch reads the clock around the pick outside prof_full") + at = disp.index('static const char* want = getenv("RX_VK_LOG_PUSH");') + push = disp[at:disp.index("Program& p = const_cast(*cp);", at)] + ordered(push, ["if ((want && !done) || dispatch_diag_old()) {", "snprintf(hex, sizeof hex,", + "if (want && !done && !strncmp(hex, want, strlen(want))) {", "done = true;"], + "dispatch formats RX_VK_LOG_PUSH's key without its switch") + ordered(disp, ["struct PipeClock {", "const bool pipe_timed = !p.compute_pipeline || !p.layout;", + "PipeClock pipe_clock{pipe_timed || dispatch_diag_old() ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{},", + "pipe_timed};", "ensure_layout(p);", "pipe_clock.stop();"], + "dispatch reads PipeClock's clock when it times nothing") + need("} pipe_clock{std::chrono::steady_clock::now()," not in disp, "PipeClock is made with a clock read at every dispatch") + shader = (ROOT / "gpu/vk/vk_shader.cpp").read_text(encoding="utf-8") + slow = block(shader, "const Program* program_pick(const NvnState& s, ProgramKind kind, const uint8_t color_types[8]) {") + need(slow.split("\n")[1].strip().startswith("PROF_EXACT();"), "program_pick is no longer timed by PROF_EXACT: the pick's time stood on it") + internal = (ROOT / "gpu/vk/vk_internal.h").read_text(encoding="utf-8") + need("prof_t.exact = x0 + dt;" in internal and "inline thread_local ProfThread prof_t;" in internal, + "PROF_EXACT no longer adds its time to the thread's prof_t.exact") + tex = (ROOT / "gpu/vk/vk_texture.cpp").read_text(encoding="utf-8") + frame = block(tex, "void texture_frame(uint64_t completed) {") + need(frame.count('getenv("RX_VK_LOG_BAKED")') == 1 and 'static const bool log_baked = getenv("RX_VK_LOG_BAKED") != nullptr;' in frame, + "texture_frame does not read RX_VK_LOG_BAKED once") + need("if (log_baked || diag_old) baked.push_back(t);" in frame and "if (log_baked) {" in frame and frame.count("baked.push_back(") == 1, + "texture_frame fills the list of RX_VK_LOG_BAKED without the switch") + need(frame.count("std::sort(") == 1 and "if (diag_old) std::sort(list.begin(), list.end(), larger);" in frame and + "else std::partial_sort(list.begin(), list.begin() + (ptrdiff_t)std::min(n, list.size()), list.end(), larger);" in frame and + "largest(big, 8);" in frame and "largest(baked, 10);" in frame and 'const bool diag_old = lab_ab::on("diagold");' in frame, + "texture_frame sorts a whole list outside the lab's knob") + for line in ('logf("vk: VRAM: %llu MB used of %llu MB allowed (%llu MB of it free in the blocks); textures %llu MB (budget %llu "', + 'logf("vk: textures: decoded ASTC %llu MB, rendered %llu MB (%zu drawn in one frame, kept: %llu KB), other %llu MB; "', + "for (size_t i = 0; i < big.size() && i < 8; i++)", "for (size_t i = 0; i < baked.size() && i < 10; i++) {"): + need(line in frame, "texture_frame's lines changed: " + line) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + draw = (ROOT / "gpu/vk/vk_draw.cpp").read_text(encoding="utf-8") + check_source(draw) + with tempfile.TemporaryDirectory(prefix="dispatch-quiet-") as tmp: + cpp, exe = Path(tmp) / "taau_key.cpp", Path(tmp) / "taau-key.exe" + cpp.write_text(FIXTURE.replace("/* CUT */", block(draw, "bool taau_key(uint64_t key) {")), encoding="utf-8") + subprocess.run([args.cxx, "-std=c++17", "-O1", "-Wall", "-Wextra", "-Werror", str(cpp), "-o", str(exe)], check=True) + for keys, card, want in LISTS: + r = subprocess.run([str(exe), keys, card], capture_output=True, text=True, timeout=120) + row = r.stdout.strip().splitlines()[-1].split() if r.stdout.strip() else [] + need(r.returncode == 0 and len(row) == 6, "RX_VK_TAAU_KEYS=%r (%s): the parsed list and the text compare differ:\n%s" % (keys, card, r.stdout)) + need(tuple(int(v) for v in row[3:]) == want, "RX_VK_TAAU_KEYS=%r (%s): answers %s, want %s" % (keys, card, row[3:], want)) + need((int(row[1]) > 0) == (1 in want) or keys in (",,fd,,0000,",), "RX_VK_TAAU_KEYS=%r: %s keys matched" % (keys, row[1])) + print("RX_VK_TAAU_KEYS=%-36r %s: %s keys, %s match, both ways alike" % (keys, card, row[0], row[1])) + print("PASS: a dispatch's leftover diagnostics are quiet, and taau_key answers as its text compare did") + + +if __name__ == "__main__": + main() diff --git a/tests/test_dlss_checks.cpp b/tests/test_dlss_checks.cpp index 4c09358..7679a75 100644 --- a/tests/test_dlss_checks.cpp +++ b/tests/test_dlss_checks.cpp @@ -46,6 +46,45 @@ int main() { CHECK("wider than the output", !dc::scene_size_fits(1537, 864, 1536, 864)); CHECK("no 32-bit overflow in the third", dc::scene_size_fits(0x60000000u, 1, 0xFFFFFFFFu, 3)); + // ---- ((Triforce, 11-22, the performance audit's PRESENT-38) the quality mode that takes a render size ---- + { + // (NGX's ranges for 2560x1440, in vk_dlss.cpp's order: MaxQuality with its dynamic range, DLAA, Balanced and MaxPerf + // inside MaxQuality's, Ultra Performance one size) + const dc::ModeRange r1440[5] = {{1280, 720, 2560, 1440, true}, {2560, 1440, 2560, 1440, true}, {1280, 720, 2560, 1440, true}, + {1280, 720, 2560, 1440, true}, {853, 480, 853, 480, true}}; + bool wide = true; + CHECK("Quality's share is in the first range", dc::mode_for(r1440, 5, 1707, 960, 2560, 1440, true, &wide) == 0 && !wide); + CHECK("a range that holds the size needs no slack", dc::mode_for(r1440, 5, 1707, 960, 2560, 1440, false, &wide) == 0 && !wide); + CHECK("the row's 33% (854x480) is Ultra Performance's by a rounding", dc::mode_for(r1440, 5, 854, 480, 2560, 1440, true, &wide) == 4 && wide); + CHECK("without the slack it is no mode's, as before", dc::mode_for(r1440, 5, 854, 480, 2560, 1440, false, &wide) == -1 && !wide); + CHECK("NGX's own 853x480 is taken strictly", dc::mode_for(r1440, 5, 853, 480, 2560, 1440, false, &wide) == 4 && !wide); + CHECK("42% is between the ranges: no mode's", dc::mode_for(r1440, 5, 1075, 605, 2560, 1440, true, &wide) == -1); + CHECK("under a third of the output is not DLSS's at all", dc::mode_for(r1440, 5, 851, 478, 2560, 1440, true, &wide) == -1); + CHECK("a size just under the dynamic range's floor is taken by the slack", dc::mode_for(r1440, 5, 1278, 719, 2560, 1440, true, &wide) == 0 && wide); + CHECK("the slack is a 64th of the range's largest size", dc::mode_slack(2560) == 40 && dc::mode_slack(853) == 13 && dc::mode_slack(100) == 2 && dc::mode_slack(0) == 2); + CHECK("past the slack on one axis is refused", dc::mode_for(r1440, 5, 867, 480, 2560, 1440, true, &wide) == -1); + CHECK("14 pixels over Ultra Performance's height is refused", dc::mode_for(r1440, 5, 854, 494, 2560, 1440, true, &wide) == -1); + // (5120x2160: the row's 33% is 1707x720, NGX's 1706x720; 42% is in no range) + const dc::ModeRange r2160[5] = {{2560, 1080, 5120, 2160, true}, {5120, 2160, 5120, 2160, true}, {2560, 1080, 5120, 2160, true}, + {2560, 1080, 5120, 2160, true}, {1706, 720, 1706, 720, true}}; + CHECK("5120x2160 at 33%", dc::mode_for(r2160, 5, 1707, 720, 5120, 2160, true, &wide) == 4 && wide); + CHECK("5120x2160 at 42%", dc::mode_for(r2160, 5, 2150, 907, 5120, 2160, true, &wide) == -1); + // (a mode NGX did not answer is never taken, and the first that fits wins) + dc::ModeRange gaps[5] = {{}, {}, {1280, 720, 2560, 1440, true}, {1280, 720, 2560, 1440, true}, {}}; + CHECK("an unanswered mode is skipped", dc::mode_for(gaps, 5, 1707, 960, 2560, 1440, true, &wide) == 2 && !wide); + CHECK("no mode answered", dc::mode_for(gaps, 2, 1707, 960, 2560, 1440, true, &wide) == -1); + // (every twelfth of every output the menu offers is some mode's with the slack, where NGX's sizes are the share rounded down) + const uint32_t outs[][2] = {{1920, 1080}, {2048, 1152}, {2560, 1440}, {3840, 2160}, {3440, 1440}, {5120, 2160}, {1536, 864}, {4416, 1864}}; + for (const auto& o : outs) { + const dc::ModeRange r[5] = {{o[0] / 2, o[1] / 2, o[0], o[1], true}, {o[0], o[1], o[0], o[1], true}, {o[0] / 2, o[1] / 2, o[0], o[1], true}, + {o[0] / 2, o[1] / 2, o[0], o[1], true}, {o[0] / 3, o[1] / 3, o[0] / 3, o[1] / 3, true}}; + for (const uint32_t twelfths : {4u, 6u, 7u, 8u}) { + const uint32_t w = (o[0] * twelfths + 11) / 12, h = (o[1] * twelfths + 11) / 12; + CHECK("a share of the row at an output of the menu", dc::mode_for(r, 5, w, h, o[0], o[1], true, &wide) >= 0); + } + } + } + // ---- UTF-8 paths as NGX's wide strings, in the C locale (the decoding must not depend on it) ---- std::setlocale(LC_ALL, "C"); // (which paths NGX's Linux library takes as they are: ASCII only. The others go to it as /proc/self/fd/) @@ -80,6 +119,50 @@ int main() { CHECK("good after bad", decode("\xFF\xC3\xA9", bad) == L"�é" && bad); CHECK("no flag asked", dc::utf8_to_wide("\xFF") == L"�"); CHECK("empty", decode("", bad).empty() && !bad); + { // ((Triforce, 11-21, PRESENT-37) the DLSS model row stepped through: no feature for a preset passed over + dc::PresetSettle s; + const unsigned K = 0, M = 1, L = 2; + CHECK("no feature: the row's preset at once", s.pick(K, ~0u, false, 100, 30) == K); + CHECK("the feature's own preset", s.pick(K, K, true, 101, 30) == K); + CHECK("a new preset waits", s.pick(M, K, true, 200, 30) == K && s.pick(M, K, true, 210, 30) == K); + CHECK("stepped on: the count starts again", s.pick(L, K, true, 211, 30) == K && s.pick(L, K, true, 240, 30) == K); + CHECK("held for 30 frames: taken", s.pick(L, K, true, 241, 30) == L); + CHECK("and it is the feature's now", s.pick(L, L, true, 242, 30) == L); + CHECK("away and back within the wait: nothing made", s.pick(M, L, true, 300, 30) == L && s.pick(L, L, true, 310, 30) == L); + CHECK("the feature let go: at once", s.pick(M, L, false, 311, 30) == M); + dc::PresetSettle now; + CHECK("0 frames: at once, as before", now.pick(K, ~0u, false, 1, 0) == K && now.pick(M, K, true, 2, 0) == M && + now.pick(L, M, true, 3, 0) == L); + CHECK("the switch", dc::settle_frames(nullptr) == 30 && dc::settle_frames("") == 30 && dc::settle_frames("30") == 30 && + dc::settle_frames("0") == 0 && dc::settle_frames("120") == 120 && dc::settle_frames("-5") == 0 && + dc::settle_frames("off") == 0); + } + { // ((Triforce, 11-22, the performance audit's PRESENT-32) the one-size (DLAA) modes: before, whenever the render size is + // the output's; with the rule, only where DLAA is the Upscaler's choice + CHECK("as before: one size whenever the sizes are equal", dc::dlaa_feature(false, false, 2560, 1440, 2560, 1440) && + dc::dlaa_feature(false, true, 2560, 1440, 2560, 1440) && + !dc::dlaa_feature(false, true, 1707, 960, 2560, 1440) && + !dc::dlaa_feature(false, false, 2560, 1439, 2560, 1440)); + CHECK("the rule: DLAA at the output's size is one size", dc::dlaa_feature(true, true, 2560, 1440, 2560, 1440)); + CHECK("the rule: DLSS at the output's size is not", !dc::dlaa_feature(true, false, 2560, 1440, 2560, 1440)); + CHECK("the rule: a smaller render size never is", !dc::dlaa_feature(true, true, 1707, 960, 2560, 1440) && + !dc::dlaa_feature(true, false, 1707, 960, 2560, 1440)); + CHECK("RX_VK_DLSS_PREWARM: on only at 1", !dc::prewarm_rule(nullptr) && !dc::prewarm_rule("") && !dc::prewarm_rule("0") && + dc::prewarm_rule("1") && !dc::prewarm_rule("2") && !dc::prewarm_rule("on")); + // (DLSS at full size takes the scaled mode whose range holds it, and that range holds Auto's smaller sizes: NGX's + // ranges as it answers them, Quality's from half the output to the output's own) + const unsigned outs[][2] = {{1920, 1080}, {2560, 1440}, {3840, 2160}, {4416, 1864}, {5120, 2160}}; + for (const auto& o : outs) { + const dc::ModeRange r[5] = {{o[0] / 2, o[1] / 2, o[0], o[1], true}, {o[0], o[1], o[0], o[1], true}, {o[0] / 2, o[1] / 2, o[0], o[1], true}, + {o[0] / 2, o[1] / 2, o[0], o[1], true}, {o[0] / 3, o[1] / 3, o[0] / 3, o[1] / 3, true}}; + bool wide = false; + const int at = dc::mode_for(r, 5, o[0], o[1], o[0], o[1], true, &wide); + CHECK("DLSS at the output's size: the first scaled mode's", at == 0 && !wide); + for (unsigned pct = 50; pct <= 100; pct += 5) + CHECK("Auto's sizes are in that mode's range", o[0] * pct / 100 >= r[at].min_w && o[0] * pct / 100 <= r[at].max_w && + o[1] * pct / 100 >= r[at].min_h && o[1] * pct / 100 <= r[at].max_h); + } + } std::puts("dlss checks: ok"); return 0; } diff --git a/tests/test_dlss_checks.py b/tests/test_dlss_checks.py index 9a84eef..eae942c 100644 --- a/tests/test_dlss_checks.py +++ b/tests/test_dlss_checks.py @@ -33,6 +33,34 @@ def check_sources(): # (NGX's Linux library converts its wide paths back in the "C" locale, ASCII only: a folder named otherwise goes to it # as /proc/self/fd/) assert "dlss_checks::ascii(s)" in dlss and '"/proc/self/fd/"' in dlss, "wide(): non-ASCII folders by descriptor" + # ((Triforce, 11-21, the performance audit's PRESENT-37) prepare reads the DLSS model row's preset only through the settle + # rule, before it asks whether the feature there is can be used: a preset stepped past makes no feature + prepare = dlss[dlss.index("int prepare(VkCommandBuffer cmd"):] + prepare = prepare[:prepare.index("\n}\n")] + assert prepare.count("wanted_preset()") == 1, "prepare: the row's preset read past the settle rule" + settle = ("const unsigned preset = s.settle.pick(wanted_preset(), s.feature_preset, s.feature != nullptr, texture_frame_no(), " + "settle_frames);") + assert settle in prepare and prepare.index(settle) < prepare.index("const bool same_feature ="), \ + "prepare: the preset is not the settle rule's, or is picked after the feature's reuse test" + assert 'dlss_checks::settle_frames(getenv("RX_VK_DLSS_MODEL_SETTLE"))' in prepare, "prepare: no RX_VK_DLSS_MODEL_SETTLE" + assert "dlss_checks::PresetSettle settle;" in dlss, "vk_dlss.cpp: the settle rule's state is not kept with the feature" + # ((Triforce, 11-22, the performance audit's PRESENT-38) the mode is found by mode_for, with its slack unless the switch + # says strict, and the feature there is is let go only after a mode took the size: a refused size keeps it + order = [prepare.find(n) for n in ( + 'static const bool slack_env = !(getenv("RX_VK_DLSS_MODE_SLACK") && atoi(getenv("RX_VK_DLSS_MODE_SLACK")) == 0);', + 'dlss_checks::mode_for(ranges, nmodes, rw, rh, ow, oh, slack_env && !lab_ab::on("dlssstrict"), &widened);', + 'if (!found) return *fail = "no DLSS mode takes this render size", 0;', + "temporal_later([old] { NVSDK_NGX_VULKAN_ReleaseFeature(old); });", + "NGX_VULKAN_CREATE_DLSS_EXT(cmd, 1, 1, &s.feature, s.params, &cp);")] + assert min(order) >= 0 and order == sorted(order), \ + "prepare: the mode is not found by mode_for before the old feature is let go (%s)" % order + assert prepare.count("NVSDK_NGX_VULKAN_ReleaseFeature(old)") == 1, "prepare: the old feature is let go in a second place" + assert "max_w = std::max(r.max_w, rw), max_h = std::max(r.max_h, rh);" in prepare, \ + "prepare: a size taken by the slack is not kept in the feature's range (it would be made again every frame)" + assert "`RX_VK_DLSS_MODE_SLACK=0`" in (ROOT / "docs" / "development.md").read_text(encoding="utf-8"), \ + "docs/development.md: RX_VK_DLSS_MODE_SLACK is not in the switches table" + assert "`RX_VK_DLSS_MODEL_SETTLE=`" in (ROOT / "docs" / "development.md").read_text(encoding="utf-8"), \ + "docs/development.md: RX_VK_DLSS_MODEL_SETTLE is not in the switches table" def main(): diff --git a/tests/test_dlss_feature.py b/tests/test_dlss_feature.py new file mode 100644 index 0000000..80a771e --- /dev/null +++ b/tests/test_dlss_feature.py @@ -0,0 +1,149 @@ +"""NVIDIA DLSS's feature is made for a fixed share's own render size, not for the largest of its mode's range +(gpu/vk/dlss_feature.h, vk_dlss.cpp prepare; no game, no GPU, no NGX). + +(Triforce, 11-21, the upscaler audit's UPS-01.) Every feature was made for the largest render size its quality mode +allows, which NGX answers as the output's own: with the Render resolution row at Quality the owner's log has a feature +"2944x1243 (2208x932 to 4416x1864) -> 4416x1864" made for 4416x1864, 2.25 times the pixels it is ever given, and DLSS's +own video memory at 1069 MB. NVIDIA's guide has the render size passed to the creation, and says the cost of presets L and +M follows the size the feature is initialized for. + +This compiles dlss_feature.h alone and checks the rule: a fixed share's feature is its own render size and takes that size +alone; the row's Auto, a size seen changing and the switch give the mode's largest, which takes its whole range as +before; DLAA's is the output's; a render size over the mode's largest never makes a feature larger than it; a frame of +another size under an exact feature of the same output is what shows the range, and nothing else is. + +And reads the source: prepare asks the rule for the feature it has and for the size it makes, with the row's Auto, what +this run saw and RX_VK_DLSS_FEATURE_MAX; the feature's size, not the mode's largest, is what NGX is given and what the +chain's inputs are made at; the log says which; and the switch is in the switches table. + + python -I -B tests/test_dlss_feature.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +PROGRAM = r""" +#include +#include +#include "dlss_feature.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +using namespace dlss_feature; +int main() { + CHECK(!whole_range(nullptr) && !whole_range("") && !whole_range("0") && whole_range("1")); + // (the owner's log: Quality, 2944x1243 into 4416x1864, the mode's range 2208x932 to 4416x1864) + { // a fixed share: its own render size, and that size alone + const Size s = size(false, false, 2944, 1243, 4416, 1864); + CHECK(s.w == 2944 && s.h == 1243 && s.exact); + CHECK(takes(s.exact, s.w, s.h, 2208, 932, 2944, 1243)); + CHECK(!takes(s.exact, s.w, s.h, 2208, 932, 2943, 1243) && !takes(s.exact, s.w, s.h, 2208, 932, 2944, 1244)); + CHECK(!takes(s.exact, s.w, s.h, 2208, 932, 2208, 932) && !takes(s.exact, s.w, s.h, 2208, 932, 4416, 1864)); + CHECK((uint64_t)s.w * s.h * 2 < (uint64_t)4416 * 1864); // (under half the pixels of the largest) + } + { // the row's Auto, a size seen changing, the switch: the mode's largest, every size of its range, as before + const Size s = size(false, true, 2944, 1243, 4416, 1864); + CHECK(s.w == 4416 && s.h == 1864 && !s.exact); + CHECK(takes(s.exact, s.w, s.h, 2208, 932, 2944, 1243) && takes(s.exact, s.w, s.h, 2208, 932, 2208, 932) && + takes(s.exact, s.w, s.h, 2208, 932, 4416, 1864)); + CHECK(!takes(s.exact, s.w, s.h, 2208, 932, 2207, 932) && !takes(s.exact, s.w, s.h, 2208, 932, 2208, 931) && + !takes(s.exact, s.w, s.h, 2208, 932, 4417, 1864)); + } + { // DLAA: the output's, its one size, whatever the rest says + for (int range = 0; range < 2; range++) { + const Size s = size(true, range != 0, 4416, 1864, 4416, 1864); + CHECK(s.w == 4416 && s.h == 1864 && !s.exact); + CHECK(takes(s.exact, s.w, s.h, 4416, 1864, 4416, 1864) && !takes(s.exact, s.w, s.h, 4416, 1864, 2944, 1243)); + } + } + { // a render size past the mode's largest (never asked: the mode is found by its range) makes none larger than it + const Size s = size(false, false, 5000, 1243, 4416, 1864); + CHECK(s.w == 4416 && s.h == 1864 && !s.exact); + } + { // what shows that the render size changes: an exact feature of the same output, and a frame of another size + CHECK(found_range(true, true, 2944, 1243, 2800, 1182)); + CHECK(found_range(true, true, 2944, 1243, 2944, 1242)); + CHECK(!found_range(true, true, 2944, 1243, 2944, 1243)); // (the same size: nothing) + CHECK(!found_range(false, true, 2944, 1243, 2800, 1182)); // (another output, preset or colour: a feature of its own) + CHECK(!found_range(true, false, 4416, 1864, 2800, 1182)); // (a feature for the range already) + } + { // every share the Render resolution row has, at 3840x2160: the feature's pixels against the largest's + const uint32_t ow = 3840, oh = 2160; + const uint32_t shares[4][2] = {{2560, 1440}, {2240, 1260}, {1920, 1080}, {1280, 720}}; + for (const auto& r : shares) { + const Size s = size(false, false, r[0], r[1], ow, oh); + CHECK(s.exact && s.w == r[0] && s.h == r[1] && takes(s.exact, s.w, s.h, 0, 0, r[0], r[1])); + } + } + return fails ? 1 : 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + src = (VK / "vk_dlss.cpp").read_text(encoding="utf-8") + + check('#include "dlss_feature.h"' in src, "vk_dlss.cpp does not use the rule") + prep = block(src, "int prepare(VkCommandBuffer cmd, uint32_t rw, uint32_t rh, uint32_t ow, uint32_t oh, bool hdr, uint32_t* in_w,") + at = [prep.index(part) for part in ( + "const bool same_feature = s.feature && s.feature_ow == ow && s.feature_oh == oh && s.feature_preset == preset && s.feature_hdr == hdr;", + "if (same_feature && dlss_feature::takes(s.feature_exact, s.feature_rw, s.feature_rh, s.min_rw, s.min_rh, rw, rh))", + "return *in_w = s.feature_rw, *in_h = s.feature_rh, 1;", + "const bool found_range = dlss_feature::found_range(same_feature, s.feature_exact, s.feature_rw, s.feature_rh, rw, rh);", + "if (found_range) s.range_seen = true;", + 'static const bool whole_range = dlss_feature::whole_range(getenv("RX_VK_DLSS_FEATURE_MAX"));', + "dlss_feature::size(aa, whole_range || s.range_seen || opts.resolution == 0, rw, rh, max_w, max_h);", + "cp.Feature.InWidth = made.w, cp.Feature.InHeight = made.h, cp.Feature.InTargetWidth = ow, cp.Feature.InTargetHeight = oh;", + "NGX_VULKAN_CREATE_DLSS_EXT(cmd, 1, 1, &s.feature, s.params, &cp);", + "s.feature_rw = made.w, s.feature_rh = made.h, s.feature_ow = ow, s.feature_oh = oh, s.min_rw = min_w, s.min_rh = min_h;", + "s.feature_exact = made.exact;", + "*in_w = s.feature_rw, *in_h = s.feature_rh;", + "return 2;")] + check(at == sorted(at), "prepare does not ask the rule for the feature it has, then for the size it makes, and keep what it made") + check("cp.Feature.InWidth = max_w" not in prep and "s.feature_rw = max_w" not in prep, + "a feature is still made for its mode's largest render size whatever the row says") + check("this render size alone (RX_VK_DLSS_FEATURE_MAX=1: its mode's largest)" in prep and "the largest of its range" in prep, + "the log does not say what size the feature was made for") + check("for its mode's whole range from now on" in prep, "the log does not say why a feature is made again when the render size changes") + rel = block(src, "void release() {") + check("s.feature_exact = false;" in rel, "a feature let go keeps its size's kind") + # (unchanged, as the chain reads them: the evaluation is given the frame's own render size, whatever the feature's) + ev = block(src, "bool evaluate(VkCommandBuffer cmd, const TemporalFrame& f) {") + check("ep.InRenderSubrectDimensions = {f.render_w, f.render_h};" in ev, "the evaluation is not given the frame's render size") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`RX_VK_DLSS_FEATURE_MAX=1`" in dev, "docs/development.md: RX_VK_DLSS_FEATURE_MAX=1 is not in the switches table") + + with tempfile.TemporaryDirectory(prefix="dlss-feature-") as tmp: + c, exe = Path(tmp) / "feature.cpp", Path(tmp) / "feature.exe" + c.write_text(PROGRAM, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(VK), str(c), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + check(r.returncode == 0, "the rule as made:\n" + r.stdout) + print("PASS: NVIDIA DLSS's feature is made for a fixed share's own render size, the mode's largest only where the size changes") + + +if __name__ == "__main__": + main() diff --git a/tests/test_dlss_prewarm.py b/tests/test_dlss_prewarm.py new file mode 100644 index 0000000..e73a4eb --- /dev/null +++ b/tests/test_dlss_prewarm.py @@ -0,0 +1,160 @@ +"""NVIDIA DLSS at the output's size takes a scaled mode with RX_VK_DLSS_PREWARM=1, so the Render resolution row's Auto +makes one feature, not two (gpu/vk/dlss_checks.h dlaa_feature, vk_dlss.cpp prepare; no game, no GPU, no NVIDIA SDK). + +(Triforce, 11-22, the performance audit's PRESENT-32, after Source A's: its first step.) prepare made a one-size feature +(NGX's DLAA mode) whenever the render size was the output's. With the row at Auto the game starts at the output's size +and lowers it some frames later: a second feature made in play. With the rule only NVIDIA DLAA gets the one-size mode; +DLSS at full size takes the scaled mode whose range holds it, which the smaller sizes reuse. + +The entry's other steps (the feature, the chain's images and a first evaluation made ahead in a light frame) are not +built: see the commit. This checks what is. + +It compiles dlss_checks.h and dlss_feature.h and runs prepare's decisions (the reuse test, the one-size test, the mode, +the size the feature is made for: the same functions in the same order, NGX's ranges as it answers them) over a load-in: +17 frames at the output's size, then the game's dynamic resolution going down and up. With the row at Auto: two +features without the rule, one with it, and that one takes every later size. With NVIDIA DLAA: one feature, of one +size, either way. With a fixed share: the same feature either way (the rule changes nothing there). Without the switch +every decision is the one from before. + +Then the source: prepare asks dlaa_feature with the switch (read once) and whether DLAA is the Upscaler's choice, the +switch is on only at 1, and its row is in the development guide. + + python -I -B tests/test_dlss_prewarm.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +PROGRAM = r""" +#include +#include +#include +#include "dlss_checks.h" +#include "dlss_feature.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +namespace dc = mkwvk::dlss_checks; + +// vk_dlss.cpp prepare's decisions, without NGX: the feature there is, and what a frame of rw x rh into ow x oh does to it +struct Feature { + bool has = false, exact = false, one_size = false; + uint32_t w = 0, h = 0, ow = 0, oh = 0, min_w = 0, min_h = 0; + int mode = -1; // (the index into the scaled modes; -1: the one-size list's first, DLAA's) +}; +struct Run { + Feature f; + bool range_seen = false; + int made = 0, refused = 0; + bool rule, dlaa, auto_row; + Run(bool rule, bool dlaa, bool auto_row) : rule(rule), dlaa(dlaa), auto_row(auto_row) {} + void frame(uint32_t rw, uint32_t rh, uint32_t ow, uint32_t oh) { + const bool same = f.has && f.ow == ow && f.oh == oh; + if (same && dlss_feature::takes(f.exact, f.w, f.h, f.min_w, f.min_h, rw, rh)) return; // (the feature there is draws it) + if (dlss_feature::found_range(same, f.exact, f.w, f.h, rw, rh)) range_seen = true; + const bool aa = dc::dlaa_feature(rule, dlaa, rw, rh, ow, oh); + uint32_t max_w = 0, max_h = 0, min_w = 0, min_h = 0; + int mode = -1; + if (aa) { + max_w = min_w = ow, max_h = min_h = oh; + } else { + // (NGX's ranges for the scaled list: Quality, DLAA, Balanced, Performance from half the output to the output's own, + // DLAA's and Ultra Performance's one size) + const dc::ModeRange r[5] = {{ow / 2, oh / 2, ow, oh, true}, {ow, oh, ow, oh, true}, {ow / 2, oh / 2, ow, oh, true}, + {ow / 2, oh / 2, ow, oh, true}, {ow / 3, oh / 3, ow / 3, oh / 3, true}}; + bool widened = false; + mode = dc::mode_for(r, 5, rw, rh, ow, oh, true, &widened); + if (mode < 0) { refused++; return; } + min_w = std::min(r[mode].min_w, rw), min_h = std::min(r[mode].min_h, rh); + max_w = std::max(r[mode].max_w, rw), max_h = std::max(r[mode].max_h, rh); + } + const dlss_feature::Size s = dlss_feature::size(aa, range_seen || auto_row, rw, rh, max_w, max_h); + f = {true, s.exact, aa, s.w, s.h, ow, oh, min_w, min_h, mode}; + made++; + } +}; +/// A load-in at an output of ow x oh: 17 frames at `start` percent, then the game's dynamic resolution moving +static void load_in(Run& run, uint32_t ow, uint32_t oh, unsigned start) { + for (int i = 0; i < 17; i++) run.frame(ow * start / 100, oh * start / 100, ow, oh); + for (unsigned pct : {95u, 90u, 80u, 75u, 60u, 50u, 65u, 85u, 100u, 70u}) run.frame(ow * pct / 100, oh * pct / 100, ow, oh); +} +int main() { + const uint32_t outs[][2] = {{1920, 1080}, {2560, 1440}, {3840, 2160}, {4416, 1864}, {5120, 2160}}; + for (const auto& o : outs) { + // NVIDIA DLSS, the Render resolution row at Auto, the game starting at the output's size + Run before(false, false, true), with(true, false, true); + load_in(before, o[0], o[1], 100), load_in(with, o[0], o[1], 100); + CHECK(before.made == 2 && before.refused == 0); // (the one-size feature, then the scaled mode's) + CHECK(with.made == 1 && with.refused == 0); // (the scaled mode's from the first frame) + CHECK(with.f.mode == 0 && !with.f.one_size && !with.f.exact && with.f.w == o[0] && with.f.h == o[1]); + CHECK(before.f.mode == with.f.mode && before.f.w == with.f.w && before.f.h == with.f.h && before.f.exact == with.f.exact); + // NVIDIA DLAA: the output's one size, either way, and never a second feature at that size + Run aa_before(false, true, false), aa_with(true, true, false); + for (int i = 0; i < 40; i++) aa_before.frame(o[0], o[1], o[0], o[1]), aa_with.frame(o[0], o[1], o[0], o[1]); + CHECK(aa_before.made == 1 && aa_with.made == 1 && aa_before.f.one_size && aa_with.f.one_size); + CHECK(aa_with.f.w == o[0] && aa_with.f.h == o[1] && aa_with.f.mode == -1); + // a fixed share (the row's Quality, 8 twelfths rounded up as the menu does): the rule changes nothing + const uint32_t rw = (o[0] * 8 + 11) / 12, rh = o[1] * 8 / 12; + Run fixed_before(false, false, false), fixed_with(true, false, false); + for (int i = 0; i < 40; i++) fixed_before.frame(rw, rh, o[0], o[1]), fixed_with.frame(rw, rh, o[0], o[1]); + CHECK(fixed_before.made == 1 && fixed_with.made == 1 && fixed_with.f.exact && fixed_with.f.w == rw && fixed_with.f.h == rh); + CHECK(fixed_before.f.mode == fixed_with.f.mode && fixed_before.f.w == fixed_with.f.w); + // DLSS with a fixed share of all of the output (one saved with the game's own): one size before, the scaled mode's with + Run full_before(false, false, false), full_with(true, false, false); + for (int i = 0; i < 40; i++) full_before.frame(o[0], o[1], o[0], o[1]), full_with.frame(o[0], o[1], o[0], o[1]); + CHECK(full_before.made == 1 && full_with.made == 1 && full_before.f.one_size && !full_with.f.one_size && full_with.f.mode == 0); + } + std::printf("the row's Auto from the output's size down: 2 features before, 1 with the rule, at %d outputs\n", + (int)(sizeof outs / sizeof outs[0])); + return fails ? 1 : 0; +} +""" + + +def check_source(): + dlss = (VK / "vk_dlss.cpp").read_text(encoding="utf-8") + prepare = dlss[dlss.index("int prepare(VkCommandBuffer cmd"):] + prepare = prepare[:prepare.index("\n}\n")] + check('static const bool dlaa_rule = dlss_checks::prewarm_rule(getenv("RX_VK_DLSS_PREWARM"));' in prepare, + "prepare does not read RX_VK_DLSS_PREWARM once") + one = "const bool aa = dlss_checks::dlaa_feature(dlaa_rule, temporal_mode() == upscaler_rows::DLAA, rw, rh, ow, oh);" + check(one in prepare and "const bool aa = rw == ow && rh == oh;" not in prepare and prepare.count("const bool aa =") == 1, + "prepare's one-size test is not dlaa_feature's, with the switch and the Upscaler's choice") + check(prepare.index(one) < prepare.index("const NVSDK_NGX_PerfQuality_Value* modes = aa ? same : scaled;") + < prepare.index("dlss_feature::size(aa, "), "prepare picks its modes or its size before the one-size test") + checks = (VK / "dlss_checks.h").read_text(encoding="utf-8") + check("return rule ? full && dlaa : full;" in checks and "return text && *text && std::atoi(text) == 1;" in checks, + "dlss_checks.h: the rule is not off unless RX_VK_DLSS_PREWARM=1, or is not the entry's") + docs = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("| `RX_VK_DLSS_PREWARM=1` |" in docs, "docs/development.md has no row for RX_VK_DLSS_PREWARM") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + check_source() + with tempfile.TemporaryDirectory(prefix="dlss-prewarm-") as tmp: + src, exe = Path(tmp) / "prewarm.cpp", Path(tmp) / "prewarm.exe" + src.write_text(PROGRAM, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I", str(VK), str(src), "-o", str(exe)], check=True) + out = subprocess.run([str(exe)], stdout=subprocess.PIPE, text=True, timeout=60) + print(out.stdout.strip()) + if out.returncode: + raise SystemExit("FAIL: the rule's run") + print("PASS: with RX_VK_DLSS_PREWARM=1 DLSS at the output's size takes a scaled mode (one feature for the row's Auto); " + "off, every decision is the one from before") + + +if __name__ == "__main__": + main() diff --git a/tests/test_dlss_sdk_cache.py b/tests/test_dlss_sdk_cache.py index f6abac4..1f43e20 100644 --- a/tests/test_dlss_sdk_cache.py +++ b/tests/test_dlss_sdk_cache.py @@ -48,7 +48,8 @@ def main(): ns = dict(os=os, hashlib=hashlib, kit=SimpleNamespace(ROOT=str(root)), NGX_SDK=str(sdk_path) if sdk_path else "", vk=str(vk), GPU=str(root / "gpu"), vk_jobs=[], cxx=a.cxx, obj_dir=str(root / "obj"), vflags=["-std=c++17", "-O0"], imgui_flags=[], imgui_users=(), shader_stamp="", stamp_def="", - vk_lib_deps=b"", logo_text=b"", imgui_deps=b"", vk_sources=lambda: [str(src), str(other)]) + vk_lib_deps=b"", logo_text=b"", imgui_deps=b"", vk_sources=lambda: [str(src), str(other)], + vk_pgo=[], VK_PGO="") # ((Triforce, 11-21, LIFT-10) the renderer profile's flags and switch, set before the loop: none here exec(code, ns) return {Path(j[1]).name: (ns["object_key"](j[1], j[3], j[4]), j) for j in ns["vk_jobs"]} diff --git a/tests/test_dlssg_depth_clip.py b/tests/test_dlssg_depth_clip.py new file mode 100644 index 0000000..ab80225 --- /dev/null +++ b/tests/test_dlssg_depth_clip.py @@ -0,0 +1,186 @@ +"""NVIDIA DLSS frame generation's camera matrices are in the clip space of the depth it is given (gpu/vk/dlss_math.h +depth_clip_projection and depth_clip_reprojection, vk_dlssg.cpp frame, fsr3_math.h Camera::flip_depth; no game, no GPU). + +(Triforce, 11-20) The owner's "DLSS flicker": the whole picture and the shadows flickered while the camera moved, with +NVIDIA DLSS frame generation on and not without it. Streamline takes its matrices in the clip space of its depth buffer: +the depth is clip z over w, 1 at the near plane with depthInverted. It was given the game's own projection and +reprojection, whose clip z is NVN's (-1 at the near plane, 1 at the far one), beside FSR's copy of the depth (1 at the +near plane, 0 far). Read with those matrices, that depth put what is far a fifth of a metre from the camera and what is +near at the far plane: wrong for everything as soon as the camera moved, and right while it stood. Only DLSS frame +generation takes matrices, so no other path showed it. + +This compiles dlss_math.h and fsr3_math.h alone and checks, with the game's projection (near 0.1, far 1000.53, 45 +degrees) and two cameras a step and a turn apart: + +- a point's clip z over w through depth_clip_projection is the depth FSR's copy holds for it (1 - d where the game's + buffer has 0 at the near plane; d itself where it is reversed already), 1 at the near plane and 0 at the far one; +- with the game's own matrix that depth is not the point's clip z (the error the flicker came from); +- a pixel and its depth through depth_clip_reprojection land on the same pixel of the frame before as the game's own + reprojection puts the point, with the frame before's depth; the identity (a reset) stays the identity; +- Streamline's linear depth for an inverted buffer, 1 / depth, grows with the distance. + +And reads the source: frame gives Streamline the converted matrices and their inverses, by the frame's flip_depth, with +depthInverted, and RX_VK_DLSSG_CLIP=0 is in the switches table. + + python -I -B tests/test_dlssg_depth_clip.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +PROGRAM = r""" +#include +#include +#include +#include +#include "dlss_math.h" +#include "fsr3_math.h" +using namespace mkwvk; +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +static bool near(double a, double b, double tol) { return std::fabs(a - b) <= tol; } +static void mul(const float a[16], const float b[16], float out[16]) { + for (int r = 0; r < 4; r++) + for (int c = 0; c < 4; c++) { + double v = 0; + for (int k = 0; k < 4; k++) v += (double)a[r * 4 + k] * b[k * 4 + c]; + out[r * 4 + c] = (float)v; + } +} +static void apply(const float m[16], const double p[4], double out[4]) { + for (int r = 0; r < 4; r++) out[r] = m[r * 4] * p[0] + m[r * 4 + 1] * p[1] + m[r * 4 + 2] * p[2] + m[r * 4 + 3] * p[3]; +} +/// NVN's projection, row major (clip = P * view position, the camera looking down -z): clip z -1 at near, 1 at far; +/// reversed: 1 at near, -1 at far (a buffer with 1 at the near plane) +static void projection(float p[16], float n, float f, bool reversed) { + std::memset(p, 0, 16 * sizeof(float)); + const float t = 1.0f / std::tan(0.5f * 0.785398f); + p[0] = t / 2.369f, p[5] = t; + p[10] = -(f + n) / (f - n), p[11] = -2.0f * f * n / (f - n); + if (reversed) p[10] = -p[10], p[11] = -p[11]; + p[14] = -1.0f; +} +/// A view matrix: a turn of a radians about y, then a move of the camera by (x, 0, z) +static void view(float v[16], float a, float x, float z) { + std::memset(v, 0, 16 * sizeof(float)); + v[0] = std::cos(a), v[2] = -std::sin(a), v[5] = 1, v[8] = std::sin(a), v[10] = std::cos(a), v[15] = 1; + v[3] = -(v[0] * x + v[2] * z), v[11] = -(v[8] * x + v[10] * z); +} + +static void run(bool reversed) { + const float n = 0.1f, f = 1000.53f; + float p[16], vn[16], vb[16], cur[16], prev[16], game_to_prev[16], dp[16], dr[16]; + projection(p, n, f, reversed); + view(vn, 0.02f, 0.3f, 0.1f), view(vb, 0.0f, 0.0f, 0.0f); // (now: a small turn and a step; before: at the origin) + mul(p, vn, cur), mul(p, vb, prev); + CHECK(dlss_math::reprojection(cur, prev, game_to_prev)); + fsr3_math::Camera cam; + CHECK(fsr3_math::camera_from_projection(p, cam)); + CHECK(cam.flip_depth == !reversed && near(cam.near_plane, n, 1e-4)); + dlss_math::depth_clip_projection(p, cam.flip_depth, dp); + dlss_math::depth_clip_reprojection(game_to_prev, cam.flip_depth, dr); + + double last_linear = 0; + const double dist[] = {0.1, 0.5, 2.0, 15.0, 120.0, 900.0, 1000.53}; + for (double v : dist) { + // (a point v in front of this frame's camera, off the axis: in its view space) + const double pv[4] = {0.31 * v, -0.17 * v, -v, 1.0}; + double c[4], d[4]; + apply(p, pv, c), apply(dp, pv, d); + const double ndc = c[2] / c[3], stored = (ndc + 1.0) * 0.5; // (the game's depth buffer: fsr3_math.h) + const double copy = cam.flip_depth ? 1.0 - stored : stored; // (FSR's copy: what frame generation is given) + CHECK(near(d[2] / d[3], copy, 2e-5)); + CHECK(near(d[0] / d[3], c[0] / c[3], 1e-6) && near(d[1] / d[3], c[1] / c[3], 1e-6) && near(d[3], c[3], 1e-6)); + CHECK(copy > -1e-4 && copy < 1.0 + 1e-4); + if (v > 0.2 && v < 900.0) CHECK(std::fabs(ndc - copy) > 0.3); // (the game's clip z is not that depth) + if (copy > 1e-6) { // (Streamline's linear depth of an inverted buffer) + CHECK(1.0 / copy > last_linear); + last_linear = 1.0 / copy; + } + // (the pixel and its depth to the frame before: as the game's own reprojection puts the point) + const double game_clip[4] = {c[0] / c[3], c[1] / c[3], ndc, 1.0}, depth_clip[4] = {c[0] / c[3], c[1] / c[3], copy, 1.0}; + double g[4], s[4]; + apply(game_to_prev, game_clip, g), apply(dr, depth_clip, s); + CHECK(near(s[0] / s[3], g[0] / g[3], 2e-4) && near(s[1] / s[3], g[1] / g[3], 2e-4)); + const double before_stored = (g[2] / g[3] + 1.0) * 0.5, before_copy = cam.flip_depth ? 1.0 - before_stored : before_stored; + CHECK(near(s[2] / s[3], before_copy, 2e-4)); + if (v > 0.5 && v < 120.0) { // (and what the game's own matrix makes of that depth: somewhere else, by far more) + double w[4]; + apply(game_to_prev, depth_clip, w); + CHECK(std::fabs(w[0] / w[3] - g[0] / g[3]) > 20 * 2e-4); + } + } + // (the near and far planes: 1 and 0) + const double pn[4] = {0, 0, -n, 1}, pf[4] = {0, 0, -f, 1}; + double a[4], b[4]; + apply(dp, pn, a), apply(dp, pf, b); + CHECK(near(a[2] / a[3], 1.0, 1e-4) && near(b[2] / b[3], 0.0, 1e-4)); + // (a reset: the identity stays the identity) + float id[16], out[16]; + for (int i = 0; i < 16; i++) id[i] = i % 5 == 0 ? 1.0f : 0.0f; + dlss_math::depth_clip_reprojection(id, cam.flip_depth, out); + for (int i = 0; i < 16; i++) CHECK(near(out[i], id[i], 1e-6)); +} + +int main() { + run(false); // (the game's: 0 at the near plane, flipped for FSR's copy) + run(true); // (a buffer reversed already) + return fails ? 1 : 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + + sl = (VK / "vk_dlssg.cpp").read_text(encoding="utf-8") + frame = block(sl, "void frame(VkCommandBuffer cmd, uint32_t out_width, uint32_t out_height) {") + check("dlss_math::depth_clip_projection(in.projection, in.camera.flip_depth, projection);" in frame and + "dlss_math::depth_clip_reprojection(in.clip_to_prev, in.camera.flip_depth, clip_to_prev);" in frame, + "frame: the matrices are not brought into the clip space of the depth Streamline is given") + check("if (!inverse(projection, clip_to_view) || !inverse(clip_to_prev, prev_to_clip)) {" in frame, + "frame: the inverses are not the converted matrices'") + check("matrix(c.cameraViewToClip, projection);" in frame and "matrix(c.clipToPrevClip, clip_to_prev);" in frame and + "matrix(c.clipToCameraView, clip_to_view);" in frame and "matrix(c.prevClipToClip, prev_to_clip);" in frame, + "frame: Streamline is not given the converted matrices") + check("matrix(c.cameraViewToClip, in.projection);" not in frame and "matrix(c.clipToPrevClip, in.clip_to_prev);" not in frame, + "frame: Streamline is still given the game's own matrices") + check("c.depthInverted = sl::Boolean::eTrue;" in frame, "frame: the depth is FSR's copy, 1 at the near plane") + check('getenv("RX_VK_DLSSG_CLIP")' in frame, "frame: no switch for the game's own matrices, for comparing") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`RX_VK_DLSSG_CLIP=0`" in dev, "docs/development.md: RX_VK_DLSSG_CLIP=0 is not in the switches table") + + with tempfile.TemporaryDirectory(prefix="dlssg-depth-clip-") as tmp: + src, exe = Path(tmp) / "clip.cpp", Path(tmp) / "clip.exe" + src.write_text(PROGRAM, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(VK), str(src), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + check(r.returncode == 0, "the matrices in the depth's clip space:\n" + r.stdout) + print("PASS: DLSS frame generation's matrices are in the clip space of the depth it is given") + + +if __name__ == "__main__": + main() diff --git a/tests/test_dmabuf_driver_gate.py b/tests/test_dmabuf_driver_gate.py new file mode 100644 index 0000000..64bb7fe --- /dev/null +++ b/tests/test_dmabuf_driver_gate.py @@ -0,0 +1,208 @@ +"""No udmabuf dma-bufs are offered to NVIDIA's proprietary driver on Linux (gpu/vk/dmabuf_gate.h; no game, no GPU). + +(Triforce, 11-22, the performance audit's OS-08, after Source A's.) On Linux each pool of guest memory is first offered +to the driver as a dma-buf made by udmabuf. NVIDIA's proprietary driver refuses every one and the pool ends as a +host-pointer import anyway, after a refused import each (one took 198 ms during a load). With that driver +/dev/udmabuf is no longer opened. + +Compiles the rule and import_dma_buf's first-call block, cut out of gpu/vk/vk_memory.cpp, over stand-ins for the +device, dlsym, open, fopen, the environment and the log (so it runs on Windows too), and checks: + RADV, NVK /dev/udmabuf opened once, "yes", pieces of 64 MB + NVIDIA's proprietary driver not opened, "no (NVIDIA's driver refuses them: host pointers)", no pieces line + ... with RX_VK_DMABUF_NVIDIA=1 opened, "yes" (as before); =0: not opened + RADV, /dev/udmabuf does not open "no (/dev/udmabuf cannot be opened)" + an anonymous window, RX_VK_NO_DMABUF, not opened, each with its own reason (and before NVIDIA's, when both hold) + no dma-buf import on the device +And reads the source: the rule is asked before the open, the answer comes from the header, the fd guard of a failed +import is still there; the switches table has the row. + + python -I -B tests/test_dmabuf_driver_gate.py --headers /include [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +RUN_ALL_ARGS = ["--cxx", "{cxx}", "--headers", "{vulkan_headers}"] + + +def block(text, marker, start=0): + at = text.find(marker, start) + if at < 0: + raise SystemExit("FAIL: vk_memory.cpp has no '%s'" % marker) + end = text.index("{", at) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[at:end] + + +FIXTURE = r'''#define VK_NO_PROTOTYPES +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "dmabuf_gate.h" +#ifndef O_CLOEXEC +#define O_CLOEXEC 0 +#endif + +static int checks, fails; +#define CHECK(x) do { checks++; if (!(x)) { fails++; printf("FAIL line %d: %s (logged: %s)\n", __LINE__, #x, logged.c_str()); } } while (0) + +// stand-ins: the device, the host's memfd function, /dev/udmabuf, the limit file, the environment, the log +static struct { bool dma_buf_import = true, sparse_buffers = true; VkDriverId driver_id{}; } dev; +static bool vkGetMemoryFdPropertiesKHR = true; // (the fd properties call is there) +static int t_memfd = 5, t_opens, t_dlsyms; +static bool t_open_ok = true; +static std::string logged; +static const char *t_no_dmabuf, *t_piece_mb, *t_nvidia_too; +static int memfd_fn() { return t_memfd; } +#define RTLD_DEFAULT nullptr +static void* dlsym(void*, const char* name) { + if (strcmp(name, "hos_posix_guest_memfd")) printf("FAIL: dlsym(%s)\n", name), exit(1); + t_dlsyms++; + return (void*)memfd_fn; +} +static int t_open(const char* path, int) { + if (strcmp(path, "/dev/udmabuf")) printf("FAIL: opened %s\n", path), exit(1); + t_opens++; + return t_open_ok ? 9 : -1; +} +static FILE* t_fopen(const char*, const char*) { return nullptr; } // (the limit file not readable: 64 MB) +static const char* t_getenv(const char* name) { + if (!strcmp(name, "RX_VK_NO_DMABUF")) return t_no_dmabuf; + if (!strcmp(name, "RX_VK_DMABUF_PIECE_MB")) return t_piece_mb; + if (!strcmp(name, "RX_VK_DMABUF_NVIDIA")) return t_nvidia_too; + printf("FAIL: getenv(%s)\n", name), exit(1); +} +static void logf(const char* fmt, ...) { + char line[512]; + va_list ap; + va_start(ap, fmt); + vsnprintf(line, sizeof line, fmt, ap); + va_end(ap); + logged += line, logged += "\n"; +} +#define open t_open +#define fopen t_fopen +#define getenv t_getenv + +static int dmabuf_memfd = -2, dmabuf_dev = -1; +static uint64_t dmabuf_limit = 0; +static uint64_t piece; +static void first_call() { + @CUT@ +} + +// One start. Returns the number of opens +static int start(VkDriverId driver, bool open_ok = true, int memfd = 5, bool ext = true, const char* no_dmabuf = nullptr, + const char* nvidia_too = nullptr) { + dev.driver_id = driver, dev.dma_buf_import = ext; + t_open_ok = open_ok, t_memfd = memfd, t_no_dmabuf = no_dmabuf, t_nvidia_too = nvidia_too, t_piece_mb = nullptr; + t_opens = t_dlsyms = 0, logged.clear(); + dmabuf_memfd = -2, dmabuf_dev = -1, dmabuf_limit = 0, piece = 0; + first_call(); + CHECK(dmabuf_memfd != -2); // (decided once) + return t_opens; +} +static bool said(const char* what) { return logged.find(what) != std::string::npos; } + +int main() { + using dmabuf_gate::Facts; + // the rule by itself + Facts f; + f.fn = f.memfd = f.ext = true; + CHECK(dmabuf_gate::try_udmabuf(f) && !strcmp(dmabuf_gate::answer(f, true), "yes")); + CHECK(!strcmp(dmabuf_gate::answer(f, false), "no (/dev/udmabuf cannot be opened)")); + f.nvidia = true; + CHECK(!dmabuf_gate::try_udmabuf(f) && dmabuf_gate::try_udmabuf_before(f)); // (as it was: offered to NVIDIA's too) + CHECK(!strcmp(dmabuf_gate::answer(f, false), "no (NVIDIA's driver refuses them: host pointers)")); + CHECK(strcmp(dmabuf_gate::answer(f, true), "yes")); // (never "yes" where it is not tried) + f.ext = false; + CHECK(!dmabuf_gate::try_udmabuf(f) && !strcmp(dmabuf_gate::answer(f, false), "no (VK_EXT_external_memory_dma_buf)")); + f.memfd = false; + CHECK(!strcmp(dmabuf_gate::answer(f, false), "no (the window is anonymous memory)")); + f.fn = false; + CHECK(!strcmp(dmabuf_gate::answer(f, false), "no (no memfd window, or RX_VK_NO_DMABUF)")); + CHECK(!dmabuf_gate::try_udmabuf(Facts{}) && !dmabuf_gate::try_udmabuf_before(Facts{})); + + // the first-call block of import_dma_buf + CHECK(start(VK_DRIVER_ID_MESA_RADV) == 1 && dmabuf_memfd == 5 && dmabuf_dev == 9); + CHECK(said("vk: guest memory as dma-bufs: yes\n") && said("vk: dma-bufs of at most 64 MB (udmabuf's limit 64 MB)")); + CHECK(piece == 64ull << 20 && dmabuf_limit == 64ull << 20); + CHECK(start(VK_DRIVER_ID_MESA_NVK) == 1 && dmabuf_memfd == 5 && said("dma-bufs: yes\n")); + CHECK(start(VK_DRIVER_ID_AMD_OPEN_SOURCE) == 1 && said("dma-bufs: yes\n")); + + CHECK(start(VK_DRIVER_ID_NVIDIA_PROPRIETARY) == 0 && dmabuf_memfd == -1 && dmabuf_dev == -1); + CHECK(said("vk: guest memory as dma-bufs: no (NVIDIA's driver refuses them: host pointers)\n") && !said("dma-bufs of at most")); + CHECK(t_dlsyms == 1); // (the window's memfd is still asked for: only the offer is skipped) + CHECK(start(VK_DRIVER_ID_NVIDIA_PROPRIETARY, true, 5, true, nullptr, "1") == 1 && dmabuf_memfd == 5 && said("dma-bufs: yes\n")); + CHECK(start(VK_DRIVER_ID_NVIDIA_PROPRIETARY, true, 5, true, nullptr, "0") == 0 && said("NVIDIA's driver refuses them")); + + CHECK(start(VK_DRIVER_ID_MESA_RADV, false) == 1 && dmabuf_memfd == -1 && said("no (/dev/udmabuf cannot be opened)\n")); + CHECK(start(VK_DRIVER_ID_MESA_RADV, true, -1) == 0 && dmabuf_memfd == -1 && said("no (the window is anonymous memory)\n")); + CHECK(start(VK_DRIVER_ID_MESA_RADV, true, 5, false) == 0 && said("no (VK_EXT_external_memory_dma_buf)\n")); + CHECK(start(VK_DRIVER_ID_MESA_RADV, true, 5, true, "1") == 0 && t_dlsyms == 0 && said("no (no memfd window, or RX_VK_NO_DMABUF)\n")); + // NVIDIA's with an anonymous window, or RX_VK_NO_DMABUF: the earlier reason is the one said + CHECK(start(VK_DRIVER_ID_NVIDIA_PROPRIETARY, true, -1) == 0 && said("no (the window is anonymous memory)\n")); + CHECK(start(VK_DRIVER_ID_NVIDIA_PROPRIETARY, true, 5, true, "1") == 0 && said("no (no memfd window, or RX_VK_NO_DMABUF)\n")); + + if (fails) printf("dmabuf driver gate: FAILED (%d of %d checks)\n", fails, checks); + else printf("dmabuf driver gate: all %d checks passed\n", checks); + return fails != 0; +} +''' + + +def check_source(): + src = (ROOT / "gpu/vk/vk_memory.cpp").read_text(encoding="utf-8") + if '#include "dmabuf_gate.h"' not in src: + raise SystemExit("FAIL: vk_memory.cpp does not include dmabuf_gate.h") + fn_at = src.find("bool import_dma_buf(uint64_t lo, uint64_t hi, Pool& p) {") + if fn_at < 0: + raise SystemExit("FAIL: vk_memory.cpp has no import_dma_buf") + first = block(src, " if (dmabuf_memfd == -2) {", fn_at) + order = [first.find(n) for n in ( + "gate.nvidia = dev.driver_id == VK_DRIVER_ID_NVIDIA_PROPRIETARY &&", + 'if (dmabuf_gate::try_udmabuf(gate)) dmabuf_dev = open("/dev/udmabuf", O_RDWR | O_CLOEXEC);', + 'logf("vk: guest memory as dma-bufs: %s", dmabuf_gate::answer(gate, dmabuf_dev >= 0));')] + if min(order) < 0 or order != sorted(order) or first.count("open(") != 2: # (the open and the limit file's fopen) + raise SystemExit("FAIL: import_dma_buf's first call does not ask dmabuf_gate.h before its one open of " + "/dev/udmabuf, or its line is not the header's answer (%r)" % (order,)) + guard = block(src, "VkDeviceMemory dma_buf_memory(uint64_t lo, uint64_t size) {") + if "fcntl(fd, F_DUPFD_CLOEXEC, 1024)" not in guard or "if (fcntl(fd, F_GETFD) != -1) close(fd);" not in guard: + raise SystemExit("FAIL: dma_buf_memory lost the guard for a failed import's fd (other drivers and layers need it)") + docs = (ROOT / "docs/development.md").read_text(encoding="utf-8") + if "| `RX_VK_DMABUF_NVIDIA=1`, `RX_VK_NO_DMABUF=1`, `RX_VK_DMABUF_PIECE_MB=` |" not in docs or "`gpu/vk/dmabuf_gate.h`" not in docs: + raise SystemExit("FAIL: docs/development.md: the switches table has no RX_VK_DMABUF_NVIDIA row") + return first + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="g++") + parser.add_argument("--headers", required=True, help="Vulkan-Headers' include folder") + args = parser.parse_args() + first = check_source() + with tempfile.TemporaryDirectory(prefix="dmabuf-gate-") as tmp: + cpp, exe = Path(tmp) / "dmabuf_gate_test.cpp", Path(tmp) / "dmabuf-gate.exe" + cpp.write_text(FIXTURE.replace("@CUT@", first), encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-Wall", "-Wextra", "-Werror", "-Wno-unused-function", + "-I" + args.headers, "-I" + str(ROOT / "gpu/vk"), str(cpp), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + print(r.stdout.rstrip()) + if r.returncode: + raise SystemExit("FAIL: the dma-buf driver gate's checks failed") + + +if __name__ == "__main__": + main() diff --git a/tests/test_dmb_light.py b/tests/test_dmb_light.py new file mode 100644 index 0000000..946e811 --- /dev/null +++ b/tests/test_dmb_light.py @@ -0,0 +1,266 @@ +"""DMB of the load or the store kind as a compiler barrier in code lifted with A64_DMB_LIGHT=1 (game-free). + +(Triforce, 11-22, the performance audit's LIFT-47, after Source B's.) The kit's translator writes every DMB and DSB as +recomp_barrier(), an out-of-line full fence. x86-64 keeps load-load, load-store and store-store order in hardware, so a +DMB that asks only for those (ISHLD, ISHST and the OSH, NSH and full-system forms: eight words) needs no instruction +there, only that the compiler moves no guest access across it. With A64_DMB_LIGHT=1 at lift time nx2/insn.py writes +A64_DMB_LDST() for those eight, which nx2/rtgen.py's a64.h defines as an empty asm with a "memory" clobber on x86-64 and +as recomp_barrier() on any other host. Off by default: unset, empty or 0 is the kit's text. + +The rule (nx2/insn.py emit, dmb_light): with the switch the eight words give the macro; a full DMB (ISH, SY, OSH, NSH), +every DSB (the LD and ST kinds too, SSBB, PSSBB), ISB and CLREX give None, the kit's text; without the switch, or with +it empty or 0, every word gives None. + +The lifted text: each word goes through the real translator (a64emit.translate under insn.install(), then +a64_lift.op_text) twice, with A64_DMB_LIGHT=1 and without. The eight must be the macro alone with the switch and the +kit's recomp_barrier() without; the others must be the same text both times. The kit's mark grouping +(a64_lift.mark_groups) must end a run of stores at the macro, as it does at the fence: no mark moves across it. + +The run: tests/dmb_light_test.c runs every word both ways, alone and between a store and a load of the same guest +address (STR; DMB; LDR; STR; DMB; LDR), from the same random Ctx (every flag combination, registers and vector +registers random) and guest memory, and compares the whole Ctx and the bytes stored. It also counts the calls of +recomp_barrier: none in the light text on x86-64, one a barrier in the kit's. Built at -march=x86-64 and -march=native. +What a single thread cannot show, the order other threads see, is not tested here. + +And reads the source: nx2/rtgen.py's a64.h holds the macro once, guarded by __x86_64__, with the fence for other +hosts; the switch is an A64_ name, which nx2/make_game.py's lift_key takes, and nx2/insn.py and nx2/rtgen.py are files of +the lift key; the switch is in the switches table. + + python -I -B tests/test_dmb_light.py [--cc clang] [--cxx clang++] [--random 20000] + +Skipped (exit 0) on a host that is not x86-64 or has no C or C++ compiler. +""" +import argparse +import os +import platform +import re +import sys +import tempfile +import time + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(ROOT, "tests")) +sys.path.insert(0, os.path.join(ROOT, "nx2")) +import test_fpx_sticky as fs # noqa: E402 (the build helpers the kit's other lifted-code tests use) + +RUN_ALL_ARGS = ["--cc", "clang", "--cxx", "{cxx}"] +SEED = 20261122 +MARCHES = ("x86-64", "native") +SWITCH = "A64_DMB_LIGHT" +MACRO = "A64_DMB_LDST();" +KINDS = {0: "OSH", 1: "NSH", 2: "ISH", 3: "SY"} +STR_X1, LDR_X3, STR_X4_8, LDR_X5_8 = 0xF9000041, 0xF9400043, 0xF9000444, 0xF9400445 # ([x2], [x2], [x2,#8], [x2,#8]) + + +def dmb(crm): + return 0xD50330BF | (crm << 8) + + +def dsb(crm): + return 0xD503309F | (crm << 8) + + +def light_words(): + """(name, word): the eight DMBs of the load or store kind""" + return [("DMB %s%s" % (KINDS[crm >> 2], "LD" if crm & 3 == 1 else "ST"), dmb(crm)) + for crm in range(16) if crm & 3 in (1, 2)] + + +def full_words(): + """(name, word): what keeps the fence. A full DMB of every domain, DSBs of every kind, the reserved CRm<1:0> = 00""" + out = [("DMB %s" % KINDS[crm >> 2], dmb(crm)) for crm in (3, 7, 11, 15)] + out += [("DMB crm=%d (reserved)" % crm, dmb(crm)) for crm in (0, 4, 8, 12)] + out += [("DSB %sLD" % KINDS[crm >> 2], dsb(crm)) for crm in (1, 5, 9, 13)] + out += [("DSB %sST" % KINDS[crm >> 2], dsb(crm)) for crm in (2, 6, 10, 14)] + out += [("DSB %s" % KINDS[crm >> 2], dsb(crm)) for crm in (3, 7, 11, 15)] + out += [("SSBB", dsb(0)), ("PSSBB", dsb(4))] + return out + + +def with_switch(value, fn): + old = os.environ.get(SWITCH) + if value is None: + os.environ.pop(SWITCH, None) + else: + os.environ[SWITCH] = value + try: + return fn() + finally: + if old is None: + os.environ.pop(SWITCH, None) + else: + os.environ[SWITCH] = old + + +def check_rule(insn): + assert len(light_words()) == 8 + assert (dmb(11), dmb(15), dsb(9)) == (0xD5033BBF, 0xD5033FBF, 0xD503399F) # (DMB ISH, DMB SY, DSB ISHLD) + others = full_words() + [("ISB", 0xD5033FDF), ("CLREX", 0xD503305F), ("NOP", 0xD503201F)] + for name, w in light_words(): + if with_switch("1", lambda: insn.emit(w)) != MACRO: + raise SystemExit("dmb light: %s (%08x) is not %s with %s=1" % (name, w, MACRO, SWITCH)) + for off in (None, "", "0"): + if with_switch(off, lambda: insn.emit(w)) is not None: + raise SystemExit("dmb light: %s (%08x) is rewritten with %s=%r" % (name, w, SWITCH, off)) + for name, w in others: + for value in ("1", None): + if with_switch(value, lambda: insn.emit(w)) is not None: + raise SystemExit("dmb light: %s (%08x) is rewritten (%s=%r): it keeps the kit's text" % (name, w, SWITCH, value)) + + +def shapes(): + """(name, guest words, barriers in it, of which light)""" + out = [] + for name, w in light_words(): + out.append((name, [w], 1, 1)) + out.append(("STR; %s; LDR; STR; %s; LDR" % (name, name), [STR_X1, w, LDR_X3, STR_X4_8, w, LDR_X5_8], 2, 2)) + for name, w in full_words(): + out.append((name, [w], 1, 0)) + out.append(("STR; DMB ISHST; LDR; STR; DMB ISH; LDR", [STR_X1, dmb(10), LDR_X3, STR_X4_8, dmb(11), LDR_X5_8], 2, 1)) + return out + + +def lifted(a64emit, a64_lift, used): + """dmb_lifted.h: each shape as lifted with the switch and without, as functions, and the CASES table""" + all_shapes = shapes() + out = ["/* Generated by tests/test_dmb_light.py: each shape lifted with A64_DMB_LIGHT=1 and without. */"] + for i, (name, words, barriers, light) in enumerate(all_shapes): + items = [(0x1000 + 4 * k, w) for k, w in enumerate(words)] + + def lift(): + texts, flags = a64emit.translate(items) + if any(flags): + raise SystemExit("dmb light: a64emit did not translate %s: %s" % (name, flags)) + return [a64_lift.op_text(t, used).strip() for t in texts] + now, old = with_switch("1", lift), with_switch(None, lift) + for w, n, o in zip(words, now, old): + is_light = (w & 0xFFFFF0FF) == 0xD50330BF and (w >> 8) & 3 in (1, 2) + if is_light and (n != MACRO or "recomp_barrier();" not in o or "A64_DMB_LDST" in o): + raise SystemExit("dmb light: %08x in %s: with the switch %r, without %r" % (w, name, n, o)) + if not is_light and (n != o or "A64_DMB_LDST" in n): + raise SystemExit("dmb light: %08x in %s changes with the switch: %r, %r" % (w, name, n, o)) + if sum(t.count("recomp_barrier();") for t in old) != barriers or \ + sum(t.count("recomp_barrier();") for t in now) != barriers - light: + raise SystemExit("dmb light: %s: the fences in the lifted text are not %d and %d" % (name, barriers, barriers - light)) + out.append("static void g_now%d(Ctx *__restrict c) { %s }" % (i, " ".join(now))) + out.append("static void g_old%d(Ctx *__restrict c) { %s }" % (i, " ".join(old))) + out.append("static const struct { const char *name; void (*now)(Ctx *), (*old)(Ctx *); int barriers, light; } CASES[] = {%s};" + % ", ".join('{"%s", g_now%d, g_old%d, %d, %d}' % (s[0], i, i, s[2], s[3]) for i, s in enumerate(all_shapes))) + return "\n".join(out) + "\n", len(all_shapes) + + +def check_marks(a64_lift): + """The kit's mark grouping ends a run of stores at the macro, as at the fence: the group's mark stays before it""" + for end in (" " + MACRO, " recomp_barrier();"): + if not a64_lift.mg_ends_run(end): + raise SystemExit("dmb light: mark_groups does not end a run at %r" % end.strip()) + body = "\n".join(["void f_80000000(Ctx *A64_NOESC c) {", " recomp_store32(c,c->x[21] + 0,c->x[1]);", + " recomp_store32(c,c->x[21] + 4,c->x[2]);", end, " recomp_store32(c,c->x[21] + 8,c->x[3]);", + " return;", "}", ""]) + got = a64_lift.mark_groups(body).split("\n") + at = got.index(end) + if "A64_MARK(" not in got[at - 1] or any("A64_MARK(" in line for line in got[at + 1:at + 2]): + raise SystemExit("dmb light: mark_groups moved a mark across %r:\n%s" % (end.strip(), "\n".join(got))) + + +def generate(cxx, tmp, flags): + """a64.h (nx2/rtgen.py's, in tmp/rt64), a64_fpx.h, the kit's runtime C and dmb_lifted.h, by the generators the lift + runs (a64emit built into tmp). Returns (the shapes, rt64, the runtime's C file)""" + dll = os.path.join(tmp, "a64emit.dll" if os.name == "nt" else "liba64emit.so") + fs.build([cxx, "-std=c++20", "-O0", "-shared", "-Wno-everything", "-o", dll, os.path.join(fs.KIT, "lift", "a64emit.cpp")] + + flags + ([] if os.name == "nt" else ["-fPIC"]), "a64emit") + import kit # noqa: F401 (a64-kit on the path) + import build_emit + build_emit.build = lambda force=False: dll # (not a64emit.dll beside the source) + import a64_lift + import a64_rtgen + import a64emit + import insn + import rtgen + check_rule(insn) + insn.install() # (the override point under test, around the kit's translate) + check_marks(a64_lift) + used = set() + try: + a64_rtgen.write(tmp) + text, n = lifted(a64emit, a64_lift, used) + finally: + if a64emit._lib is not None: # (unloaded, so the folder can go) + import _ctypes + (_ctypes.FreeLibrary if os.name == "nt" else _ctypes.dlclose)(a64emit._lib._handle) + a64emit._lib = None + with open(os.path.join(tmp, "dmb_lifted.h"), "w", newline="\n") as f: + f.write(text) + with open(os.path.join(tmp, "a64_fpx.h"), "w", newline="\n") as f: + f.write(a64_lift.fpx_header(used)) + rt64 = rtgen.write(os.path.join(tmp, "rt64")) + with open(os.path.join(rt64, "a64.h"), encoding="utf-8") as f: + header = f.read() + want = re.compile(r"#if defined\(__x86_64__\)\n#define A64_DMB_LDST\(\) __asm__ __volatile__\(\"\" ::: \"memory\"\)\n" + r"#else\n#define A64_DMB_LDST\(\) recomp_barrier\(\)\n#endif\n") + if len(want.findall(header)) != 1 or header.count("#define A64_DMB_LDST") != 2: + raise SystemExit("dmb light: nx2/rtgen.py's a64.h does not define A64_DMB_LDST once, guarded by __x86_64__") + rt_c = [f for f in sorted(os.listdir(tmp)) if f.startswith("a64_") and f.endswith(".c")] # (what a64_rtgen wrote) + if len(rt_c) != 1: + raise SystemExit("dmb light: a64_rtgen wrote %s, not one C file" % rt_c) + return n, rt64, os.path.join(tmp, rt_c[0]) + + +def check_sources(): + with open(os.path.join(ROOT, "nx2", "insn.py"), encoding="utf-8") as f: + insn_text = f.read() + for line in (' return (w & 0xFFFFF0FF) == 0xD50330BF and ((w >> 8) & 3) in (1, 2) and ' + 'os.environ.get("A64_DMB_LIGHT", "0") not in ("", "0")', + ' return "A64_DMB_LDST();" if dmb_light(w) else None'): + if insn_text.count(line) != 1: + raise SystemExit("dmb light: nx2/insn.py no longer has the line %r" % line.strip()) + with open(os.path.join(ROOT, "nx2", "rtgen.py"), encoding="utf-8") as f: + if "(RET_MISMATCH, RET_MISMATCH + POLL_DECL + DMB_LDST)," not in f.read(): + raise SystemExit("dmb light: nx2/rtgen.py no longer adds DMB_LDST to a64.h") + with open(os.path.join(ROOT, "nx2", "make_game.py"), encoding="utf-8") as f: + make_game = f.read() + # (the lift key: every A64_ variable of the environment, and every file of nx2/ but LIFT_KEY_NOT's) + if 'if k.startswith(("A64_", "OOT_", "RX_"))' not in make_game or not SWITCH.startswith("A64_"): + raise SystemExit("dmb light: nx2/make_game.py's lift_key no longer takes the A64_ variables: %s is not in the key" % SWITCH) + not_keyed = re.search(r"^LIFT_KEY_NOT = \((.*?)\) #", make_game, re.S | re.M) + if not not_keyed or '"insn.py"' in not_keyed.group(1) or '"rtgen.py"' in not_keyed.group(1): + raise SystemExit("dmb light: nx2/insn.py and nx2/rtgen.py must stay files of the lift key") + with open(os.path.join(ROOT, "docs", "development.md"), encoding="utf-8") as f: + if "| `A64_DMB_LIGHT=1` |" not in f.read(): + raise SystemExit("dmb light: docs/development.md: A64_DMB_LIGHT is not in the switches table") + + +def main(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--cc") + p.add_argument("--cxx") + p.add_argument("--random", type=int, default=20000) + a = p.parse_args() + check_sources() + if platform.machine().lower() not in ("x86_64", "amd64"): + fs.skip("not an x86-64 host: %s" % platform.machine()) + cc = fs.compiler(a.cc, "CC", ("clang", "gcc", "cc"), "clang") + cxx = fs.compiler(a.cxx, "CXX", ("clang++", "g++", "c++"), "clang++") + if not cc or not cxx: + fs.skip("no C compiler" if not cc else "no C++ compiler") + t0 = time.time() + failed = [] + with tempfile.TemporaryDirectory(prefix="triforce-dmb-light-") as tmp: + n, rt64, rt_c = generate(cxx, tmp, fs.target_flags(cxx, tmp)) + for march in MARCHES: + exe = os.path.join(tmp, "dmb_light_test_%s.exe" % march) + fs.build([cc] + fs.target_flags(cc, tmp) + ["-march=" + march] + fs.FLAGS + + ["-D_CRT_SECURE_NO_WARNINGS", "-I" + rt64, "-I" + os.path.join(fs.KIT, "rt"), "-I" + tmp, + os.path.join(ROOT, "tests", "dmb_light_test.c"), rt_c, "-o", exe], "the driver at " + march) + r = fs.run([exe, str(a.random), str(SEED)], timeout=1200) + print("%-10s %s" % (march, r.stdout.rstrip())) + if r.returncode: + failed.append("%s (exit %d)" % (march, r.returncode)) + if failed: + raise SystemExit("FAIL: dmb light: %s" % ", ".join(failed)) + print("PASS: %d shapes lifted with A64_DMB_LIGHT=1 leave the same registers, flags and memory as the kit's text, " + "the eight light DMBs without a fence (%.0f s)" % (n, time.time() - t0)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_draw_binds.py b/tests/test_draw_binds.py index 6c6416c..bfcfb7a 100644 --- a/tests/test_draw_binds.py +++ b/tests/test_draw_binds.py @@ -35,6 +35,11 @@ def main(): raise SystemExit("a vertex set remembers an empty size as empty") if "gpu_bind_stamp() == vstamp" not in draw: raise SystemExit("a vertex set is remembered across a mirror") + # (a mesh pipeline's bind leaves the vertex strides undefined: the next vertex draw binds all 16 streams again) + bind = draw.index("vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, pl)") + rebind = draw.find("if (p.mesh && mesh_vb_rebind) last_vb_valid = false, vertex_set.on = false;", bind) + if rebind < 0 or rebind > draw.index("vertex_set_same(", bind): + raise SystemExit("a mesh pipeline's bind keeps the vertex streams and strides of the draws before it") if draw.count("bind_index(") != 3: raise SystemExit("an indexed draw binds its index buffer on its own") bound = block(source, "void bind_index(") diff --git a/tests/test_draw_side_effects.py b/tests/test_draw_side_effects.py index c44c08a..99a0b43 100644 --- a/tests/test_draw_side_effects.py +++ b/tests/test_draw_side_effects.py @@ -25,6 +25,7 @@ FIXTURE = r""" #include #include #include "pipe_wait.h" +#include "translate_wait.h" #define PROF_EXACT() ((void)0) #define PROF_HOT(x) ((void)0) #define SKIP_HERE() return @@ -54,13 +55,14 @@ struct Stages { }; struct Program { std::atomic ready{false}; + uint64_t key = 0; // ((Triforce, PIPE-30) read by the bounded wait) bool failed = false, fallback = false, fallback_task = false; int layout = 1, fallback_topology = 0; Stages stages{&ready, {}}; }; struct PipeEntry { std::atomic ready{false}; VkPipeline pipeline = 1; }; -struct Targets { bool persistent = true; }; -struct NvnState {}; +struct Targets { bool persistent = true; void* depth = nullptr; }; // ((Triforce, PIPE-29) depth: read by the budget) +struct NvnState { struct { bool dt = false; } ds; }; struct ParLock {}; struct DrawArgs { bool indirect = true; int groups[3]{1, 1, 1}; int first_task = 0; }; int fallback_task_groups(const Program&) { std::abort(); } @@ -101,6 +103,21 @@ uint64_t texture_frame_no() { return 1; } pipe_wait::Pool& pipe_pool() { static pipe_wait::Pool p(50 * pipe_wait::kMs); return p; } pipe_wait::Pool& pipe_last() { static pipe_wait::Pool p(40 * pipe_wait::kMs); return p; } bool pipe_wait_for(const std::atomic&, int64_t) { std::abort(); } +// ((Triforce, 11-22, PIPE-30) RX_VK_TRANSLATE_WAIT_MS unset, as here by default: the block waits as before. +// tests/test_translate_wait.py runs it set) +int translate_wait_ms() { return 0; } +translate_wait::Notes& translate_notes() { static translate_wait::Notes n; return n; } +pipe_wait::Pool& translate_pool() { static pipe_wait::Pool p; return p; } +bool jobs_wait_for(const std::atomic&, int) { std::abort(); } +std::atomic st_translate_bounded{0}, st_translate_left{0}; +// ((Triforce, 11-22, PIPE-29) RX_VK_PIPE_DEPTH_BUDGET unset, as here by default: depth-tested draws into new passes wait +// as before. tests/test_pipe_depth_budget.py checks the rule set) +bool pipe_depth_budget() { return false; } +bool occ_counting_now() { return false; } +bool pipe_depth_timed_out(const void*) { std::abort(); } +void pipe_depth_time_out(const void*) { std::abort(); } +std::atomic st_pipe_depth_waits{0}, st_pipe_depth_left{0}; +void logf(const char*, ...) {} int64_t st_pipe_new_waits, st_pipe_full_waits, st_pipe_full_held, st_pipe_full_left; bool exposure_log_on() { return false; } void exposure_note(NvnState&, const Program&) {} diff --git a/tests/test_draw_stream_visibility.py b/tests/test_draw_stream_visibility.py index 7711451..bb6ede8 100644 --- a/tests/test_draw_stream_visibility.py +++ b/tests/test_draw_stream_visibility.py @@ -35,7 +35,7 @@ def main(): actual += "\n" + block(draw, "BufferRef stream_copy(") fixture = fixture.replace("/* STREAM */", actual) flush = block(draw, "void draw_flush_recording(") if "void draw_flush_recording(" in draw else "" - fixture = fixture.replace("/* FLUSH */", flush).replace("/* BEGIN */", block(draw, "void draw_begin_recording(")) + fixture = fixture.replace("/* FLUSH */", flush).replace("/* BEGIN */", block(draw, "void slot_reset_locked(") + "\n" + block(draw, "void draw_begin_recording(")) fixture = fixture.replace("/* SUBMIT */", block(replay, "void submit_recording(")) start = replay.index(" if (j->inits < 0) init_lost = true;") end = replay.index(" if (res == VK_ERROR_DEVICE_LOST) device_lost();", start) diff --git a/tests/test_drawn_page_map.py b/tests/test_drawn_page_map.py new file mode 100644 index 0000000..17e84a5 --- /dev/null +++ b/tests/test_drawn_page_map.py @@ -0,0 +1,188 @@ +"""A texture the GPU drew first gets its page map at that draw (gpu/vk/drawn_page_map.h, vk_texture.cpp texture_written and +texture_dirty; no game, no GPU). + +(Triforce, 11-21, the performance audit's TEX-21, after Source A's.) A texture in a virtual pool is checked through its +mapping, and its page map was recorded at its first check. One the GPU drew first had none: once the game finalized it, +the first check took its whole storage as mapped since and uploaded it from memory over what the GPU drew. + +This compiles drawn_page_map.h and texture_dirty cut out of vk_texture.cpp as it is, over a mapping and write generations +of the test's own: +- without the seed, the first check of a drawn texture gives its whole storage (the fault); +- seeded as texture_written seeds it (generation 0), the same check gives nothing; a CPU write gives its pages alone; a + page mapped anew gives that page alone; +- the rule: which writes seed (a reason, a virtual pool, no map of the storage's size yet) and the switch. + +And reads the source: texture_written seeds by the rule, under the parallel replay's lock, with generation 0, before the +texture is marked GPU-written and uploaded; every caller that draws or copies gives a reason; the switch is in the +switches table. + + python -I -B tests/test_drawn_page_map.py [--cxx clang++] +""" +import argparse +import re +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, head): + at = text.index(head) + return text[at:text.index("\n}\n", at) + 3] + + +FIXTURE = r""" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "drawn_page_map.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +using Ranges = std::vector>; +struct WriteGen { + uint64_t v = 0; + constexpr WriteGen() = default; + constexpr explicit WriteGen(uint64_t x) : v(x) {} + constexpr explicit operator bool() const { return v != 0; } +}; +struct Texture { + uint32_t id = 1, width = 256, height = 256, layers = 1; + int nvn_format = 0x25; + bool gpu_written = false, uploaded = false; + uint64_t gpu = 0, guest = 0, guest_size = 0, map_sig = 0; + std::vector page_map; + struct Run { uint64_t off, guest, n; }; + std::vector runs; + uint32_t runs_maps_gen = ~0u; +}; +// the test's mapping: a virtual address to guest memory, 64 KB a page; and what the CPU wrote, with its generation +static std::map mapping; +static uint32_t maps_gen = 1; +struct Host { + uint32_t maps_generation() const { return maps_gen; } + uint32_t vmap_generation() const { return 0; } + uint64_t gpu_guest(uint64_t a) const { + const auto it = mapping.find(a & ~0xFFFFull); + return it == mapping.end() ? 0 : it->second + (a & 0xFFFF); + } +}; +static Host host_obj; +static Host* host = &host_obj; +struct Write { uint64_t lo, hi, gen; }; +static std::vector writes; +static void memory_written_runs(uint64_t guest, uint64_t n, WriteGen gen, Ranges& out, bool = false) { + for (const Write& w : writes) + if (w.gen > gen.v && w.lo < guest + n && w.hi > guest) + out.push_back({std::max(w.lo, guest) - guest, std::min(w.hi, guest + n) - guest}); + std::sort(out.begin(), out.end()); +} +static uint64_t mix(uint64_t h, uint64_t v) { return (h ^ v) * 0x100000001b3ull + 0x9E3779B97F4A7C15ull; } +static std::map> textures; +static void logf(const char*, ...) {} + +// (vk_texture.cpp's own, cut out by the test) +@TEXTURE_DIRTY@ + +static Texture drawn() { // (a 256 KB texture in a virtual pool, four pages, mapped to guest memory that is not in one piece) + Texture t; + t.gpu = (1ull << 40) + 0x200000, t.guest_size = 4 * 65536; + return t; +} +int main() { + using namespace drawn_page_map; + // the rule + const uint64_t V = kVirtual + 0x200000; + CHECK(on(nullptr) && on("") && on("1") && !on("0")); + CHECK(pages(1) == 1 && pages(65536) == 1 && pages(65537) == 2 && pages(4 * 65536) == 4); + CHECK(seeds(true, "a draw", V, 4 * 65536, 0)); + CHECK(!seeds(false, "a draw", V, 4 * 65536, 0)); // (the switch) + CHECK(!seeds(true, "?", V, 4 * 65536, 0)); // (no reason given: a clear, the movie upscaler) + CHECK(!seeds(true, nullptr, V, 4 * 65536, 0)); + CHECK(!seeds(true, "a draw", 0x80000000, 4 * 65536, 0)); // (not a virtual pool: checked in guest memory directly) + CHECK(!seeds(true, "a draw", V, 0, 0)); // (no storage) + CHECK(!seeds(true, "a draw", V, 4 * 65536, 4)); // (the map is there: once a texture) + CHECK(seeds(true, "a draw", V, 4 * 65536, 3)); // (a map of another size is none) + + mapping = {{V, 0x10000000}, {V + 65536, 0x10010000}, {V + 2 * 65536, 0x30000000}, {V + 3 * 65536, 0x30010000}}; + writes = {{0x10000000, 0x10001000, 5}}; // (the CPU wrote a page of it long ago, generation 5) + Ranges out; + { // without the seed: drawn, finalized at generation 9, and its first check takes the whole storage + Texture t = drawn(); + CHECK(texture_dirty(t, WriteGen{9}, out) && out.size() == 1 && out[0].first == 0 && out[0].second == 4 * 65536); + } + { // seeded at the draw, as texture_written does: generation 0 + Texture t = drawn(); + CHECK(seeds(true, "a draw", t.gpu, t.guest_size, t.page_map.size())); + Ranges seeded; + texture_dirty(t, WriteGen{}, seeded); + CHECK(t.page_map.size() == 4 && t.page_map[0] == 0x10000000 && t.page_map[2] == 0x30000000); + CHECK(!seeds(true, "a draw", t.gpu, t.guest_size, t.page_map.size())); // (once) + CHECK(!texture_dirty(t, WriteGen{9}, out) && out.empty()); // (finalized, checked: nothing to upload) + writes.push_back({0x30010000, 0x30012000, 12}); // (the CPU writes two 4 KB pages after that) + CHECK(texture_dirty(t, WriteGen{9}, out) && out.size() == 1 && out[0].first == 3 * 65536 && out[0].second == 3 * 65536 + 0x2000); + mapping[V + 65536] = 0x50000000, maps_gen++; // (and one of its pages is mapped anew) + CHECK(texture_dirty(t, WriteGen{12}, out) && out.size() == 1 && out[0].first == 65536 && out[0].second == 2 * 65536); + CHECK(!texture_dirty(t, WriteGen{12}, out)); // (taken once) + } + return fails ? 1 : 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + tex = (VK / "vk_texture.cpp").read_text(encoding="utf-8").replace("\r\n", "\n") + written = block(tex, "void texture_written(Texture* tex, const char* why) {") + at = 0 + for part in ('static const bool drawn_map = drawn_page_map::on(getenv("RX_VK_DRAWN_PAGE_MAP"));', + "if (drawn_page_map::seeds(drawn_map, why, tex->gpu, tex->guest_size, tex->page_map.size())) {", + "ParLock lk;", "texture_dirty(*tex, WriteGen{}, seeded);", "tex->gpu_written = true;", "tex->uploaded = true;"): + at = written.find(part, at) + check(at >= 0, "texture_written does not record a drawn texture's page map by the rule, under the parallel replay's lock, " + "with generation 0, before it marks the texture written and uploaded (missing, or out of order: " + part + ")") + at += len(part) + check('#include "drawn_page_map.h"' in tex, "vk_texture.cpp does not use the rule") + # (the writers that draw or copy say who they are; none of them passes the "?" that keeps the old path) + reasons = 0 + for name in ("vk_draw.cpp", "vk_xfer.cpp"): + src = (VK / name).read_text(encoding="utf-8") + for m in re.finditer(r"\btexture_written\(([^;]*)\);", src): + call = m.group(1) + check('"?"' not in call, name + ": a writer passes \"?\" itself: say who writes, or give no reason") + reasons += call.count('"') >= 2 + check(reasons >= 8, "the writers that give a reason were not found (%d): the scan no longer reads the source" % reasons) + check('void texture_written(Texture* tex, const char* why = "?");' in (VK / "vk_texture.h").read_text(encoding="utf-8"), + "texture_written's default reason is no longer \"?\": a write without one would seed") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`RX_VK_DRAWN_PAGE_MAP=0`" in dev, "docs/development.md: RX_VK_DRAWN_PAGE_MAP is not in the switches table") + + with tempfile.TemporaryDirectory(prefix="drawn-page-map-") as tmp: + c, exe = Path(tmp) / "map.cpp", Path(tmp) / "map.exe" + c.write_text(FIXTURE.replace("@TEXTURE_DIRTY@", block(tex, "bool texture_dirty(Texture& tex, WriteGen gen, Ranges& out, bool same_ok = false) {")), + encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Wno-unused-function", "-Wno-unused-variable", "-Werror", + "-I" + str(VK), str(c), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + check(r.returncode == 0, "the map at the draw:\n" + r.stdout) + print("PASS: a texture the GPU drew first gets its page map at that draw") + + +if __name__ == "__main__": + main() diff --git a/tests/test_driver_cache_folder.py b/tests/test_driver_cache_folder.py new file mode 100644 index 0000000..48f0b0b --- /dev/null +++ b/tests/test_driver_cache_folder.py @@ -0,0 +1,215 @@ +"""The folder __GL_SHADER_DISK_CACHE_PATH names is made before Vulkan loads; the Linux binary keeps its name (no game). + +(Triforce, 11-22, the performance audit's PIPE-98, after Source A's.) NVIDIA's Linux driver keeps no shader cache at +all when __GL_SHADER_DISK_CACHE_PATH names a folder that is not there: every prepared pipeline is compiled again at +each start. The Linux main makes the folder and its parents (0700) and never changes the value. The driver files its +cache by the program's file name, so that name is pinned here too. + +Cuts driver_cache_folder out of host/hos_main.c and runs it on real folders under a temporary one (mkdir through a +stand-in that notes the mode, so it runs on Windows too), over an environment of the test's own: + a missing folder three levels deep made, every mkdir with 0700, no line + a folder that is there left as it is, with what it holds, no line + a trailing slash, a double slash made + a path under a file, a file at that name not made, one "is not a folder and cannot be made one" line + an empty value, or unset nothing made, nothing asked of the file system + HOS_DRIVER_CACHE_FOLDER=0 nothing made, no line +And reads the source: the function sets no variable; the last main calls it after the env-file loader and before +hos_display_init; game_main logs the kept line; nx2/build.py still has EXE = "triforce" if LINUX and +__GL_SHADER_DISK_CACHE_APP_NAME appears nowhere in host/ or gpu/vk as something set; the switches table has the row. + + python -I -B tests/test_driver_cache_folder.py [--cc clang] +""" +import argparse +import re +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +RUN_ALL_ARGS = ["--cc", "clang"] + + +def block(text, marker): + start = text.find(marker) + if start < 0: + raise SystemExit("FAIL: host/hos_main.c has no '%s'" % marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +FIXTURE = r'''#include +#include +#include +#include +#ifdef _WIN32 +#include +#endif +#define CHECK(x) do { checks++; if (!(x)) { fprintf(stdout, "FAIL:%d %s (line kept: \"%s\")\n", __LINE__, #x, driver_folder_line); fails++; } } while (0) +static int checks, fails; +static int t_mkdirs, t_other_modes; +static int t_mkdir(const char *path, int mode) { /* (the real folder, and the mode asked for noted) */ + t_mkdirs++, t_other_modes += mode != 0700; +#ifdef _WIN32 + return _mkdir(path); +#else + return mkdir(path, (mode_t)mode); +#endif +} +/* the test's own environment */ +static const char *t_path, *t_off; +static char *t_getenv(const char *name) { + if (!strcmp(name, "__GL_SHADER_DISK_CACHE_PATH")) return (char *)t_path; + if (!strcmp(name, "HOS_DRIVER_CACHE_FOLDER")) return (char *)t_off; + fprintf(stdout, "FAIL: the environment was asked for %s\n", name); + exit(1); +} +#define getenv t_getenv +#define mkdir(path, mode) t_mkdir(path, mode) +@CUT@ +#undef getenv +#undef mkdir +static int is_dir(const char *path) { struct stat st; return stat(path, &st) == 0 && S_ISDIR(st.st_mode); } +static int is_file(const char *path) { struct stat st; return stat(path, &st) == 0 && S_ISREG(st.st_mode); } +static const char *root; +static const char *under(const char *rest) { + static char p[8][1024]; + static int k; + k = (k + 1) % 8; + snprintf(p[k], sizeof p[k], "%s/%s", root, rest); + return p[k]; +} +static void start(const char *path, const char *off) { + t_path = path, t_off = off, t_mkdirs = t_other_modes = 0, driver_folder_line[0] = 0; + driver_cache_folder(); + CHECK(t_other_modes == 0); /* (for the user alone) */ +} +int main(int argc, char **argv) { + if (argc < 2) return 2; + root = argv[1]; /* (a temporary folder holding "there" with "kept.bin" in it, and the file "afile") */ + + start(under("a/b/c"), NULL); /* missing, three levels deep */ + CHECK(is_dir(under("a")) && is_dir(under("a/b")) && is_dir(under("a/b/c")) && !driver_folder_line[0] && t_mkdirs >= 3); + start(under("a/b/c"), NULL); /* and again: there already */ + CHECK(is_dir(under("a/b/c")) && !driver_folder_line[0]); + + start(under("there"), NULL); /* a folder that is there, with a file in it */ + CHECK(is_dir(under("there")) && is_file(under("there/kept.bin")) && !driver_folder_line[0]); + + start(under("t1/t2/"), NULL); /* a trailing slash */ + CHECK(is_dir(under("t1/t2")) && !driver_folder_line[0]); + + start(under("d1//d2"), NULL); /* a double slash */ + CHECK(is_dir(under("d1/d2")) && !driver_folder_line[0]); + + start(under("afile/sub"), NULL); /* under a file */ + CHECK(!is_dir(under("afile/sub")) && is_file(under("afile"))); + CHECK(strstr(driver_folder_line, "boot: __GL_SHADER_DISK_CACHE_PATH=") && strstr(driver_folder_line, "afile/sub is not a folder and cannot be made one")); + + start(under("afile"), NULL); /* a file at that name */ + CHECK(is_file(under("afile")) && strstr(driver_folder_line, "afile is not a folder and cannot be made one")); + + start("", NULL); /* empty, unset: nothing asked of the file system */ + CHECK(t_mkdirs == 0 && !driver_folder_line[0]); + start(NULL, NULL); + CHECK(t_mkdirs == 0 && !driver_folder_line[0]); + + start(under("off/x"), "0"); /* the switch: as before, nothing made */ + CHECK(t_mkdirs == 0 && !is_dir(under("off")) && !driver_folder_line[0]); + start(under("off/x"), "1"); + CHECK(is_dir(under("off/x")) && !driver_folder_line[0]); + start(under("afile/sub"), "0"); /* and nothing said */ + CHECK(t_mkdirs == 0 && !driver_folder_line[0]); + + { /* a path longer than the function copies: not made, said */ + static char longp[2200]; + size_t n = (size_t)snprintf(longp, sizeof longp, "%s/long", root); + while (n < 1500) longp[n++] = 'x'; + longp[n] = 0; + start(longp, NULL); + CHECK(t_mkdirs == 0 && strstr(driver_folder_line, "is not a folder and cannot be made one")); + } + + if (fails) printf("driver cache folder: FAILED (%d of %d checks)\n", fails, checks); + else printf("driver cache folder: all %d checks passed\n", checks); + return fails != 0; +} +''' + + +def check_source(main_c): + at = main_c.rfind("\nint main(int argc, char **argv) {") + if at < 0: + raise SystemExit("FAIL: host/hos_main.c has no main") + body = main_c[at:main_c.index("\n}\n", at)] + loader = re.search(r"\n load_\w+_env\(\);\n", body) # (the env-file loader's call: the switch can be in that file) + order = [loader.start() if loader else -1] + [body.find(n) for n in (" driver_cache_folder();", "hos_display_init();")] + if min(order) < 0 or order != sorted(order): + raise SystemExit("FAIL: the Linux main does not call driver_cache_folder() after the env-file loader and before " + "hos_display_init() (%r)" % (order,)) + game = main_c[main_c.index("static int game_main(int argc, char **argv) {"):at] + order = [game.find(n) for n in ('hos_log("triforce %s",', 'if (driver_folder_line[0]) hos_log("%s", driver_folder_line);', + "rx_prepare_shaders()")] + if min(order) < 0 or order != sorted(order): + raise SystemExit("FAIL: game_main does not log the kept line after the version line, before the preparation") + fn = block(main_c, "static void driver_cache_folder(void) {") + if "setenv" in fn or "putenv" in fn: + raise SystemExit("FAIL: driver_cache_folder() changes the environment (the variable's value must stay)") + if fn.count("mkdir(dir, 0700);") != 2: + raise SystemExit("FAIL: driver_cache_folder() no longer makes each parent and the folder for the user alone") + before = main_c[:main_c.index("static void driver_cache_folder(void) {")] + # (the block for the systems other than Windows: #ifndef _WIN32, or the Linux block driver_env_choose is in) + if max(before.rfind("#ifndef _WIN32\n"), before.rfind("#ifdef __linux__\n")) < before.rfind("#endif"): + raise SystemExit("FAIL: driver_cache_folder() is not in the block for the systems other than Windows") + # (the driver files its cache by the program's file name: a rename starts every Linux NVIDIA player's cache over) + build = (ROOT / "nx2/build.py").read_text(encoding="utf-8") + if '\nEXE = "triforce" if LINUX else ' not in build: + raise SystemExit('FAIL: nx2/build.py no longer has EXE = "triforce" if LINUX: renaming the Linux binary starts ' + "every NVIDIA player's driver shader cache over") + for path in sorted((ROOT / "host").rglob("*.c")) + sorted((ROOT / "gpu/vk").glob("*.cpp")): + text = path.read_text(encoding="utf-8", errors="replace") + if re.search(r'(setenv|putenv)\(\s*"__GL_SHADER_DISK_CACHE_APP_NAME', text): + raise SystemExit("FAIL: %s sets __GL_SHADER_DISK_CACHE_APP_NAME: every player's cache would start over" % path.name) + docs = (ROOT / "docs/development.md").read_text(encoding="utf-8") + if "| `HOS_DRIVER_CACHE_FOLDER=0` |" not in docs or "`__GL_SHADER_DISK_CACHE_PATH`" not in docs: + raise SystemExit("FAIL: docs/development.md: the switches table has no HOS_DRIVER_CACHE_FOLDER row") + if "The game builds as `Triforce.exe` + `triforce_vk.dll` (`triforce` + `triforce_vk.so` on Linux)." not in docs: + raise SystemExit("FAIL: docs/development.md does not give the game's file names as nx2/build.py builds them") + return "static char driver_folder_line[1200];\n" + fn + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cc", default="clang") + args = parser.parse_args() + main_c = (ROOT / "host/hos_main.c").read_text(encoding="utf-8") + if "static char driver_folder_line[1200];\nstatic void driver_cache_folder(void) {" not in main_c: + raise SystemExit("FAIL: host/hos_main.c has no driver_cache_folder() with its kept line") + fn = check_source(main_c) + with tempfile.TemporaryDirectory(prefix="driver-cache-folder-") as tmp: + tmp = Path(tmp) + src, exe = tmp / "driver_cache_folder_test.c", tmp / "driver-cache-folder.exe" + src.write_text(FIXTURE.replace("@CUT@", fn), encoding="utf-8") + subprocess.run([args.cc, "-std=c11", "-D_POSIX_C_SOURCE=200809L", "-Wall", "-Wextra", "-Werror", str(src), + "-o", str(exe)], check=True) + root = tmp / "root" + (root / "there").mkdir(parents=True) + (root / "there" / "kept.bin").write_bytes(b"kept") + (root / "afile").write_bytes(b"a file") + r = subprocess.run([str(exe), root.as_posix()], capture_output=True, text=True) + print(r.stdout.rstrip()) + if r.returncode: + raise SystemExit("FAIL: the driver cache folder's checks failed") + if (root / "there" / "kept.bin").read_bytes() != b"kept" or (root / "afile").read_bytes() != b"a file": + raise SystemExit("FAIL: a file that was there was changed") + lines = r.stderr.splitlines() + if len(lines) != 3 or not all(l.startswith("boot: __GL_SHADER_DISK_CACHE_PATH=") for l in lines): + raise SystemExit("FAIL: stderr has %d lines, 3 expected (under a file, a file at that name, too long):\n%s" + % (len(lines), r.stderr)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_driver_cache_room.py b/tests/test_driver_cache_room.py new file mode 100644 index 0000000..88838e5 --- /dev/null +++ b/tests/test_driver_cache_room.py @@ -0,0 +1,176 @@ +"""Room in the driver's shader cache for the pipelines the game prepares (Linux; no game, no Linux needed to check). + +(Triforce, 11-22, the performance audit's PIPE-97, after Source A's and Source B's.) The game builds its ~41,400 +predicted pipelines into the driver's own disk cache. NVIDIA's and Mesa's caches hold 1 GiB unless told otherwise and +trim themselves to it, so part of a full preparation is dropped and compiled again in play. The Linux main sets both +drivers' sizes to 10 GiB before Vulkan loads, without overwriting a size the player set. + +Cuts driver_cache_room out of host/hos_main.c and runs it over a small environment of the test's own: + nothing set __GL_SHADER_DISK_CACHE_SIZE=10737418240 (at least 4 GiB, in bytes) and + MESA_SHADER_CACHE_MAX_SIZE=10G, both set without overwriting + the player's own sizes kept, each by itself + HOS_DRIVER_CACHE_ROOM=0 neither set, the line says so; =1 or empty: set + __GL_SHADER_DISK_CACHE_SKIP_CLEANUP never set + one line on stderr, kept for the log +And reads the source: the last main calls it after the env-file loader and before hos_display_init, so the builder's +--prepare-shaders run gets it too; game_main logs the kept line; the switches table has the row. + + python -I -B tests/test_driver_cache_room.py [--cc clang] +""" +import argparse +import re +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +RUN_ALL_ARGS = ["--cc", "clang"] + + +def block(text, marker): + start = text.find(marker) + if start < 0: + raise SystemExit("FAIL: host/hos_main.c has no '%s'" % marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +FIXTURE = r'''#include +#include +#include +#define CHECK(x) do { checks++; if (!(x)) { fprintf(stdout, "FAIL:%d %s (line kept: \"%s\")\n", __LINE__, #x, driver_room_line); fails++; } } while (0) +static int checks, fails; +/* the test's own environment */ +static const char *const t_names[] = {"HOS_DRIVER_CACHE_ROOM", "__GL_SHADER_DISK_CACHE_SIZE", "MESA_SHADER_CACHE_MAX_SIZE"}; +static char t_values[3][128]; +static int t_set[3], t_overwrites, t_sets; +static int t_index(const char *name) { + for (int i = 0; i < 3; i++) + if (!strcmp(name, t_names[i])) return i; + fprintf(stdout, "FAIL: the environment was asked for %s\n", name); /* (SKIP_CLEANUP, APP_NAME or any other) */ + exit(1); +} +static char *t_getenv(const char *name) { const int i = t_index(name); return t_set[i] ? t_values[i] : NULL; } +static int t_setenv(const char *name, const char *value, int overwrite) { + const int i = t_index(name); + t_sets++, t_overwrites += overwrite != 0; + if (t_set[i] && !overwrite) return 0; + t_set[i] = 1; + snprintf(t_values[i], sizeof t_values[i], "%s", value); + return 0; +} +static void put(const char *name, const char *value) { + const int i = t_index(name); + t_set[i] = value != NULL; + if (value) snprintf(t_values[i], sizeof t_values[i], "%s", value); +} +#define getenv t_getenv +#define setenv t_setenv +@CUT@ +#undef getenv +#undef setenv +static int is(const char *name, const char *want) { + const char *v = t_getenv(name); + return want ? v && !strcmp(v, want) : !v; +} +static void start(const char *room, const char *nv, const char *mesa) { + put("HOS_DRIVER_CACHE_ROOM", room), put("__GL_SHADER_DISK_CACHE_SIZE", nv), put("MESA_SHADER_CACHE_MAX_SIZE", mesa); + driver_room_line[0] = 0, t_sets = t_overwrites = 0; + driver_cache_room(); + CHECK(t_overwrites == 0); /* (never over a size the player set) */ + CHECK(is("HOS_DRIVER_CACHE_ROOM", room)); +} +int main(void) { + start(NULL, NULL, NULL); /* out of the box */ + CHECK(is("__GL_SHADER_DISK_CACHE_SIZE", "10737418240") && is("MESA_SHADER_CACHE_MAX_SIZE", "10G") && t_sets == 2); + CHECK(strtoull(t_getenv("__GL_SHADER_DISK_CACHE_SIZE"), NULL, 10) >= 4ull << 30); /* (in bytes, well past the 1 GiB default) */ + CHECK(strstr(driver_room_line, "__GL_SHADER_DISK_CACHE_SIZE=10737418240, MESA_SHADER_CACHE_MAX_SIZE=10G")); + CHECK(!strncmp(driver_room_line, "boot: ", 6)); + + start(NULL, "2147483648", NULL); /* the player's own size for NVIDIA's stays; Mesa's is still set */ + CHECK(is("__GL_SHADER_DISK_CACHE_SIZE", "2147483648") && is("MESA_SHADER_CACHE_MAX_SIZE", "10G")); + CHECK(strstr(driver_room_line, "__GL_SHADER_DISK_CACHE_SIZE=2147483648, MESA_SHADER_CACHE_MAX_SIZE=10G")); + start(NULL, NULL, "2G"); + CHECK(is("__GL_SHADER_DISK_CACHE_SIZE", "10737418240") && is("MESA_SHADER_CACHE_MAX_SIZE", "2G")); + start(NULL, "1", "1K"); + CHECK(is("__GL_SHADER_DISK_CACHE_SIZE", "1") && is("MESA_SHADER_CACHE_MAX_SIZE", "1K")); + + start("0", NULL, NULL); /* the switch: as before, nothing set */ + CHECK(is("__GL_SHADER_DISK_CACHE_SIZE", NULL) && is("MESA_SHADER_CACHE_MAX_SIZE", NULL) && t_sets == 0); + CHECK(strstr(driver_room_line, "left as they are (HOS_DRIVER_CACHE_ROOM=0)")); + start("0", "5", "6"); + CHECK(is("__GL_SHADER_DISK_CACHE_SIZE", "5") && is("MESA_SHADER_CACHE_MAX_SIZE", "6") && t_sets == 0); + start("1", NULL, NULL); + CHECK(is("__GL_SHADER_DISK_CACHE_SIZE", "10737418240") && is("MESA_SHADER_CACHE_MAX_SIZE", "10G")); + start("", NULL, NULL); + CHECK(is("__GL_SHADER_DISK_CACHE_SIZE", "10737418240") && is("MESA_SHADER_CACHE_MAX_SIZE", "10G")); + + if (fails) printf("driver cache room: FAILED (%d of %d checks)\n", fails, checks); + else printf("driver cache room: all %d checks passed\n", checks); + return fails != 0; +} +''' + + +def check_source(main_c): + at = main_c.rfind("\nint main(int argc, char **argv) {") + if at < 0: + raise SystemExit("FAIL: host/hos_main.c has no main") + body = main_c[at:main_c.index("\n}\n", at)] + loader = re.search(r"\n load_\w+_env\(\);\n", body) # (the env-file loader's call: the switch can be in that file) + order = [loader.start() if loader else -1] + [body.find(n) for n in (" driver_cache_room();", "hos_display_init();")] + if min(order) < 0 or order != sorted(order): + raise SystemExit("FAIL: the Linux main does not call driver_cache_room() after the env-file loader and before " + "hos_display_init() (%r)" % (order,)) + if body.find(" driver_cache_room();") > body.find("if (!strcmp(argv[i], \"--prepare-shaders\")) prepare_shaders = 1;"): + raise SystemExit("FAIL: driver_cache_room() is called after the --prepare-shaders parse") + game = main_c[main_c.index("static int game_main(int argc, char **argv) {"):at] + order = [game.find(n) for n in ('hos_log("triforce %s",', 'if (driver_room_line[0]) hos_log("%s", driver_room_line);', + "rx_prepare_shaders()")] + if min(order) < 0 or order != sorted(order): + raise SystemExit("FAIL: game_main does not log the kept line after the version line, before the preparation") + fn = block(main_c, "static void driver_cache_room(void) {") + for needle in ('setenv("__GL_SHADER_DISK_CACHE_SIZE", "10737418240", 0);', 'setenv("MESA_SHADER_CACHE_MAX_SIZE", "10G", 0);'): + if needle not in fn: + raise SystemExit("FAIL: driver_cache_room() no longer has: " + needle) + if "SKIP_CLEANUP" in fn or "APP_NAME" in fn: + raise SystemExit("FAIL: driver_cache_room() sets more than the two sizes") + before = main_c[:main_c.index("static void driver_cache_room(void) {")] + # (the block for the systems other than Windows: #ifndef _WIN32, or the Linux block driver_env_choose is in) + if max(before.rfind("#ifndef _WIN32\n"), before.rfind("#ifdef __linux__\n")) < before.rfind("#endif"): + raise SystemExit("FAIL: driver_cache_room() is not in the block for the systems other than Windows") + docs = (ROOT / "docs/development.md").read_text(encoding="utf-8") + if "| `HOS_DRIVER_CACHE_ROOM=0` |" not in docs or "`__GL_SHADER_DISK_CACHE_SIZE`" not in docs: + raise SystemExit("FAIL: docs/development.md: the switches table has no HOS_DRIVER_CACHE_ROOM row") + return "static char driver_room_line[320];\n" + fn + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cc", default="clang") + args = parser.parse_args() + main_c = (ROOT / "host/hos_main.c").read_text(encoding="utf-8") + if "static char driver_room_line[320];\nstatic void driver_cache_room(void) {" not in main_c: + raise SystemExit("FAIL: host/hos_main.c has no driver_cache_room() with its kept line") + fn = check_source(main_c) + with tempfile.TemporaryDirectory(prefix="driver-cache-room-") as tmp: + tmp = Path(tmp) + src, exe = tmp / "driver_cache_room_test.c", tmp / "driver-cache-room.exe" + src.write_text(FIXTURE.replace("@CUT@", fn), encoding="utf-8") + subprocess.run([args.cc, "-std=c11", "-D_POSIX_C_SOURCE=200809L", "-Wall", "-Wextra", "-Werror", str(src), + "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + print(r.stdout.rstrip()) + if r.returncode: + raise SystemExit("FAIL: the driver cache room's checks failed") + lines = r.stderr.splitlines() + if len(lines) != 8 or not all(l.startswith("boot: ") for l in lines): + raise SystemExit("FAIL: stderr has %d lines, one for each of the 8 starts expected:\n%s" % (len(lines), r.stderr)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_driver_faults.py b/tests/test_driver_faults.py new file mode 100644 index 0000000..6ba34a9 --- /dev/null +++ b/tests/test_driver_faults.py @@ -0,0 +1,243 @@ +"""Programs the driver faulted on kept out of it, and one build of a program at a time, on AMD's Windows driver +(gpu/vk/driver_faults.h; vk_draw.cpp fault_quarantine, driver_fault_file_note, guarded_graphics, build_graphics; no game, +no GPU). + +(Triforce, 11-22, the performance audit's PIPE-57, after Source A's.) The guards caught a fault in the driver's compiler and +a fault during the start restarted the game, but nothing remembered the program: the next start faulted on it again, and in +play a second build of the same program could go into the driver while the first was still inside. Now, on AMD's own +Windows driver (or everywhere with RX_VK_DRIVER_FAULTS=2), a faulted program is kept out of the driver for the run and, by +a line the fault's handler writes to triforce_driver_faults.txt, for later starts with that driver; and a program has one +build inside the driver at a time. Other drivers: nothing of it. 0: nowhere. + +The header is compiled and run: +- the switch, and whether it is on for many made-up PCs (Windows and Linux; AMD's own driver, RADV, AMD's open driver, + NVIDIA's, Intel's, Mesa's, MoltenVK's): on for AMD's own driver on Windows alone at 1, everywhere at 2, nowhere at 0; + this PC (NVIDIA on Windows) has it off, as before; +- the file's line (35 bytes, made without the C library) and its parse: this vendor's and driver's lines only, CRLF + taken, broken lines passed over; +- the gate with 8 threads on one program: never two inside; a probe that may not wait is told busy; a fault keeps the + program out for every later build, said once; a key from the file is kept out from the start. +Then driver_fault_file_note, cut out of vk_draw.cpp, appends the line to a file with the system's own calls, and nothing +without a program in the gate. And the source: the graphics guard clears the handle, notes the fault and writes the line +only where the gate set a program, before driver_fault_restart; build_graphics goes through the gate around the creation +and its retry (none after a fault), ends it before the probe's return, and the rule asks for AMD's own driver; the +switches table has the row. + + python -I -B tests/test_driver_faults.py [--cxx clang++] +""" +import argparse +import os +import subprocess +import tempfile +from pathlib import Path + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] + +RULES = r""" +#include +#include +#include +#include +#include "driver_faults.h" +#define CHECK(x) do { if (!(x)) { std::printf("line %d: %s\n", __LINE__, #x); return 1; } } while (0) +using namespace driver_faults; +struct Pc { const char* what; bool windows; bool amd_proprietary; bool want1; }; +int main() { + CHECK(mode(nullptr) == 1 && mode("") == 1 && mode("0") == 0 && mode("1") == 1 && mode("2") == 2 && mode("5") == 2 && mode("-1") == 0); + const Pc pcs[] = { + {"this PC: NVIDIA RTX, Windows", true, false, false}, + {"RX 6800 XT, AMD's Windows driver", true, true, true}, + {"RX 5700 XT, AMD's Windows driver", true, true, true}, + {"RX 7900 XT, AMD's Windows driver", true, true, true}, + {"RX 6600, RADV on Linux", false, false, false}, + {"RX 6600, AMD's own driver on Linux", false, true, false}, + {"Arc B580, Intel's Windows driver", true, false, false}, + {"Iris Xe, Mesa on Linux", false, false, false}, + {"GTX 1060, NVIDIA on Linux", false, false, false}, + {"RTX 4070, NVIDIA on Windows", true, false, false}, + }; + for (const Pc& pc : pcs) { + CHECK(on(1, pc.windows, pc.amd_proprietary) == pc.want1); + CHECK(on(2, pc.windows, pc.amd_proprietary)); + CHECK(!on(0, pc.windows, pc.amd_proprietary)); + } + // the line and its parse + char l[kLine + 1]; + CHECK(line(l, 0x1002, 0x800161, 0x6eaa646d07ea4e43ull) == kLine && !strcmp(l, "00001002 00800161 6eaa646d07ea4e43\n")); + std::string file = l; + line(l, 0x1002, 0x800161, 0x5fa56485a6b07619ull); + file += std::string(l, kLine - 1) + "\r\n"; // (CRLF) + line(l, 0x1002, 0x800170, 0x1111ull); // (another driver version) + file += l; + line(l, 0x10de, 0x800161, 0x2222ull); // (another vendor) + file += l; + file += "garbage line\n00001002 00800161 zz\n00001002 00800161 4e5ce183a33b0855"; // (broken, then one with no newline) + const std::vector keys = parse(file, 0x1002, 0x800161); + CHECK(keys == (std::vector{0x6eaa646d07ea4e43ull, 0x5fa56485a6b07619ull, 0x4e5ce183a33b0855ull})); + CHECK(parse(file, 0x1002, 0x800170) == std::vector{0x1111ull} && parse("", 1, 2).empty()); + // the gate: 8 threads on one program, never two inside + Quarantine q; + std::atomic inside{0}, most{0}, went{0}; + std::vector threads; + for (int t = 0; t < 8; t++) + threads.emplace_back([&] { + for (int i = 0; i < 200; i++) { + if (q.begin(42, true) != Quarantine::Begin::Go) continue; + const int now = ++inside; + int m = most.load(); + while (now > m && !most.compare_exchange_weak(m, now)) {} + std::this_thread::yield(); + --inside, went++; + q.end(42, false); + } + }); + for (auto& th : threads) th.join(); + CHECK(most == 1 && went == 1600 && !q.faulted(42)); + // a probe that may not wait is told busy; another program goes in beside it + CHECK(q.begin(7, true) == Quarantine::Begin::Go && q.begin(7, false) == Quarantine::Begin::Busy && q.begin(8, false) == Quarantine::Begin::Go); + // a fault: kept out from then on, said once + CHECK(q.end(7, true) && q.faulted(7) && q.begin(7, true) == Quarantine::Begin::Faulted && q.begin(7, false) == Quarantine::Begin::Faulted); + CHECK(!q.end(8, false) && q.count() == 1); + // a waiting build whose program faults while it waits is told so + { + CHECK(q.begin(9, true) == Quarantine::Begin::Go); + std::atomic got{-1}; + std::thread waiter([&] { got = (int)q.begin(9, true); }); + std::this_thread::sleep_for(std::chrono::milliseconds(30)); + CHECK(got == -1); // (waiting at the gate) + q.end(9, true); + waiter.join(); + CHECK(got == (int)Quarantine::Begin::Faulted); + } + // keys from the file: kept out from the start + Quarantine f; + f.load(keys); + CHECK(f.begin(0x5fa56485a6b07619ull, true) == Quarantine::Begin::Faulted && f.begin(0x1111ull, true) == Quarantine::Begin::Go && f.count() == 3); + std::printf("ok\n"); + return 0; +} +""" + +WRITER = r""" +#include +#include +#include +#include +#include "driver_faults.h" +#define CHECK(x) do { if (!(x)) { std::printf("line %d: %s\n", __LINE__, #x); return 1; } } while (0) +thread_local uint64_t fault_key = 0; +uint32_t fault_vendor = 0x1002, fault_driver = 0x800161; +wchar_t fault_file_w[MAX_PATH * 2] = L""; +/* CUT */ +static std::string read(const char* path) { + std::string s; + if (FILE* f = fopen(path, "rb")) { + char b[256]; + for (size_t n; (n = fread(b, 1, sizeof b, f)) > 0;) s.append(b, n); + fclose(f); + } + return s; +} +int main(int argc, char** argv) { + if (argc < 2) return 2; + const std::string path = std::string(argv[1]) + "/triforce_driver_faults.txt"; + MultiByteToWideChar(CP_UTF8, 0, path.c_str(), -1, fault_file_w, MAX_PATH * 2); + driver_fault_file_note(); // (no program in the gate: nothing) + CHECK(read(path.c_str()).empty()); + fault_key = 0x6eaa646d07ea4e43ull; + driver_fault_file_note(); + fault_key = 0x5fa56485a6b07619ull; + driver_fault_file_note(); + const std::string got = read(path.c_str()); + CHECK(got == "00001002 00800161 6eaa646d07ea4e43\n00001002 00800161 5fa56485a6b07619\n"); + CHECK(driver_faults::parse(got, 0x1002, 0x800161).size() == 2); + std::printf("ok\n"); + return 0; +} +""" + + +def block(text, marker): + start = text.index(marker) + at = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[at] == "{") - (text[at] == "}") + at += 1 + return text[start:at] + + +def need(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def ordered(text, parts, why): + last = 0 + for p in parts: + k = text.find(p, last) + need(k >= 0, why + " (missing, or out of order: " + p + ")") + last = k + len(p) + + +def check_source(draw): + need('#include "driver_faults.h"' in draw, "vk_draw.cpp does not include driver_faults.h") + rule = block(draw, "bool driver_faults_on() {") + need(rule.count("dev.driver_id == VK_DRIVER_ID_AMD_PROPRIETARY") == 2 and "driver_faults::on(driver_faults_mode(), true," in rule + and "driver_faults::on(driver_faults_mode(), false," in rule, "the rule does not ask for AMD's own driver, on Windows") + need('static const int mode = driver_faults::mode(getenv("RX_VK_DRIVER_FAULTS"));' in block(draw, "int driver_faults_mode() {"), + "the switch is not RX_VK_DRIVER_FAULTS, read once") + guard = block(draw, "__declspec(noinline) VkResult guarded_graphics(") + ordered(guard, ['__except (driver_fault_note(GetExceptionInformation(), "graphics")) {', + "if (fault_key) *pl = VK_NULL_HANDLE, driver_fault_hit = true, driver_fault_file_note();", + "driver_fault_restart();", "return VK_ERROR_UNKNOWN;"], + "the graphics guard does not clear the handle and write the line before the restart, only for a gated build") + quar = block(draw, "driver_faults::Quarantine& fault_quarantine() {") + ordered(quar, ["std::call_once(once,", '"triforce_driver_faults.txt"', "fault_vendor = dev.props.vendorID, fault_driver = dev.props.driverVersion;", + "driver_faults::parse(text, fault_vendor, fault_driver);", "q.load(keys);", "MultiByteToWideChar(CP_UTF8, 0, path.c_str(), -1, fault_file_w,", + 'logf("vk: driver faults: %zu programs'], + "fault_quarantine does not load the file once, make the handler's path or say it") + build = block(draw, "VkPipeline build_graphics(const Program& p, const NvnState& s, const Targets& at, VkPrimitiveTopology topo, int part, bool* cold) {") + ordered(build, ["const bool fault_gated = driver_faults_on();", "if (fault_gated) {", + "fault_quarantine().begin(p.key, cold == nullptr);", + "if (b == driver_faults::Quarantine::Begin::Busy) return *cold = true, VK_NULL_HANDLE;", + "if (b == driver_faults::Quarantine::Begin::Faulted) {", "return VK_NULL_HANDLE;", "fault_key = p.key, driver_fault_hit = false;", + "const PcacheUse use(pcache_door, wanted);", "VkResult r = cached_graphics(cache, &ci, &pl);", + "&& !driver_fault_hit) {", "r = cached_graphics(cache, &ci, &pl);", "if (fault_gated) {", "fault_key = 0;", + "if (fault_quarantine().end(p.key, driver_fault_hit))", "driver_fault_hit = false;", + "if (cold && r == VK_PIPELINE_COMPILE_REQUIRED) return *cold = true, VK_NULL_HANDLE;"], + "build_graphics does not go through the gate around the creation and its retry, or ends it after a return") + docs = (ROOT / "docs/development.md").read_text(encoding="utf-8") + need("`RX_VK_DRIVER_FAULTS=0` or `2`" in docs, "docs/development.md: the switches table has no RX_VK_DRIVER_FAULTS row") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + draw = (ROOT / "gpu/vk/vk_draw.cpp").read_text(encoding="utf-8") + need("void driver_fault_file_note() {" in draw, "vk_draw.cpp has no driver_fault_file_note") + env = {k: v for k, v in os.environ.items() if not k.startswith("RX_VK_")} + with tempfile.TemporaryDirectory(prefix="driver-faults-") as tmp: + tmp = Path(tmp) + tests = [("rules", RULES)] + if os.name == "nt": # (the handler's writer is Windows code) + tests.append(("writer", WRITER.replace("/* CUT */", block(draw, "void driver_fault_file_note() {")))) + for name, text in tests: + src, exe = tmp / (name + ".cpp"), tmp / (name + ".exe") + src.write_text(text, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(ROOT / "gpu/vk"), str(src), "-o", str(exe)], + check=True) + work = tmp / ("work-" + name) + work.mkdir() + r = subprocess.run([str(exe), str(work).replace("\\", "/")], capture_output=True, text=True, timeout=120, env=env) + need(r.returncode == 0 and r.stdout.strip() == "ok", "driver faults (%s): %s" % (name, r.stdout.strip() or r.returncode)) + check_source(draw) + print("PASS: on AMD's Windows driver a faulted program is kept out of the driver (and in the file for later starts), one build " + "of a program at a time; other drivers as before; RX_VK_DRIVER_FAULTS=0 nowhere") + + +if __name__ == "__main__": + main() diff --git a/tests/test_evict_excess.py b/tests/test_evict_excess.py new file mode 100644 index 0000000..da2e227 --- /dev/null +++ b/tests/test_evict_excess.py @@ -0,0 +1,121 @@ +"""The 90% and 97% rules take their excess, not an eighth of the cache (gpu/vk/vram_compact.h rule_target's three-argument +form, vk_texture.cpp texture_frame, RX_VK_EVICT_EXCESS; no game/GPU). + +With the compaction on, the hold's pass let textures unused for 60 presents go down to 7/8 of the budget the 90% and 97% +rules left. Those budgets are "the textures less the excess", not sizes: 62 MB over the line on an 8 GB card took about +400 MB, all on screen again within the camera's turn, made again, and over the line again. Now the budget's size is kept +apart: 7/8 of the size when over it, and for the two rules their budget less 64 MB. Checked: + +1. The rule, compiled from the header: the Market on an 8 GB card (2,873 MB held, a size of 3,285 MB, the 90% rule's budget + 2,811 MB) lets 126 MB go, not 413; the size alone as before; both over, the lower; never under nothing. +2. texture_frame: the size taken after the live line and before the 90% rule; the rules' budget after the 90% rule and + through the 97% one; rule_to from both with the switch on, the old form with RX_VK_EVICT_EXCESS=0 or the lab's knob + noexcess, read the way the other switches are (on unless 0); the switch has its docs row. + + python -I -B tests/test_evict_excess.py [--cxx clang++] +""" +import argparse +import os +from pathlib import Path +import subprocess +import tempfile + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def fail(message): + raise SystemExit("FAIL: " + message) + + +FIXTURE = r'''#include +#include +#include "vram_compact.h" +#define CHECK(x) do { if (!(x)) { std::printf("line %d: %s\n", __LINE__, #x); return 1; } } while (0) +using namespace vram_compact; +int main() { + const uint64_t MB = 1ull << 20; + // (the Market entered from Hyrule Field, an 8 GB card: 62 MB over the 90% line) + const uint64_t total = 2873 * MB, size = 3285 * MB, rule = 2811 * MB; + CHECK(total - rule_target(total, size, rule) == 126 * MB); + CHECK(total - rule_target(total, rule) == total - rule / 8 * 7); // (the old form: 413 MB) + CHECK(total - rule / 8 * 7 > 413 * MB - MB && total - rule / 8 * 7 < 414 * MB); + // (under every line: nothing; the size alone: 7/8 of it, as the old form; both: the lower) + CHECK(rule_target(rule, size, rule) == UINT64_MAX && rule_target(size, size, size) == UINT64_MAX); + CHECK(rule_target(3500 * MB, size, size) == rule_target(3500 * MB, size)); + CHECK(rule_target(3500 * MB, size, 3000 * MB) == size / 8 * 7); + CHECK(rule_target(3500 * MB, size, 2600 * MB) == 2600 * MB - 64 * MB); + CHECK(rule_target(3500 * MB, size, 10 * MB) == 0 && rule_target(3500 * MB, size, 0) == 0); + std::printf("OK\n"); + return 0; +} +''' + + +def check_source(tex, dev): + frame = block(tex, "void texture_frame(uint64_t completed)") + order = ["if (press.live) budget = tex_pressure::budget(", "const uint64_t size_budget = budget;", + "if (known && pressure > allowed / 10 * 9) {", "uint64_t rule_budget = budget;", + "rule_budget = std::min(rule_budget, total_bytes > over ? total_bytes - over : 0);", + "const uint64_t rule_to = ", "texture_sweep(completed, "] + at = -1 + for needle in order: + i = frame.find(needle, at + 1) + if i < 0: + fail("texture_frame: %r missing or out of order (the size after the live line and before the 90%% rule, the " + "rules' budget after both rules, rule_to after them)" % needle) + at = i + rule = frame[frame.index("const uint64_t rule_to = "):] + rule = rule[:rule.index(";\n") + 1] + if "vram_compact::rule_target(total_bytes, size_budget, rule_budget)" not in rule or \ + "vram_compact::rule_target(total_bytes, rule_budget)" not in rule: + fail("rule_to is not the three-argument rule with the old form for the switch off") + if 'lab_ab::on("noexcess")' not in rule: + fail("the lab's knob noexcess does not turn the rule back") + env = frame[frame.index('getenv("RX_VK_EVICT_EXCESS")') - 120:frame.index("const uint64_t rule_to = ")] + if "return !e || atoi(e) != 0;" not in env: + fail("RX_VK_EVICT_EXCESS is not on unless 0") + if "`RX_VK_EVICT_EXCESS=0`" not in dev: + fail("docs/development.md has no RX_VK_EVICT_EXCESS row") + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--cxx", default="g++") + a = p.parse_args() + vk = ROOT / "gpu" / "vk" + tex = (vk / "vk_texture.cpp").read_text(encoding="utf-8") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + try: + check_source(tex, dev) + except ValueError as e: + fail("a text this test reads moved (%s)" % e) + with tempfile.TemporaryDirectory(prefix="evict-excess-") as tmp: + cpp = Path(tmp) / "fixture.cpp" + exe = Path(tmp) / ("fixture" + (".exe" if os.name == "nt" else "")) + cpp.write_text(FIXTURE, encoding="utf-8") + built = subprocess.run([a.cxx, "-std=c++20", "-Wall", "-Wextra", "-Werror", "-I", str(vk), str(cpp), "-o", str(exe)], + capture_output=True, text=True) + if built.returncode: + print(built.stdout + built.stderr) + fail("the fixture did not build") + done = subprocess.run([str(exe)], capture_output=True, text=True, timeout=30) + if done.returncode or done.stdout.strip() != "OK": + fail("the rule: " + (done.stdout + done.stderr).strip()) + print("PASS: the 90% and 97% rules go down to their budget less 64 MB, the size to 7/8 of it; RX_VK_EVICT_EXCESS=0 and " + "noexcess as before") + + +if __name__ == "__main__": + main() diff --git a/tests/test_evict_reason.cpp b/tests/test_evict_reason.cpp new file mode 100644 index 0000000..247ba11 --- /dev/null +++ b/tests/test_evict_reason.cpp @@ -0,0 +1,77 @@ +// The eviction's stale write-back rule (gpu/vk/evict_reason.h); run by tests/test_evict_reason.py. +// (Triforce, 11-22, the performance audit's TEX-R404, after Source A's.) +#include "../gpu/vk/evict_reason.h" + +#include + +#define CHECK(name, condition) do { if (!(condition)) { std::fprintf(stderr, "FAIL %s at line %d\n", name, __LINE__); return 1; } } while (0) + +int main() { + using namespace evict_reason; + // the switch: unset, empty and 0 leave a stale write-back out + CHECK("the switch unset", !flush_stale(nullptr) && !flush_stale("") && !flush_stale("0")); + CHECK("the switch set", flush_stale("1") && flush_stale("2")); + // Source A's frames: a target last drawn in frame 3804, a label drawn inside it in 3987 + CHECK("a later drawing makes it stale", write_back_stale(3804, 3987)); + CHECK("no other drawing", !write_back_stale(3804, 0)); + CHECK("an older drawing", !write_back_stale(3804, 1468)); + CHECK("a drawing in the same frame keeps it", !write_back_stale(3804, 3804)); + CHECK("the frame after", write_back_stale(3804, 3805)); + // the trace's addresses: a 1920 x 1080 target of 8,640 KB and a label inside it + const uint64_t target = 0x1001e88fc00ull, target_end = target + 8640ull * 1024; + const uint64_t label = 0x1001f039400ull, label_end = label + 110ull * 110 * 4; + CHECK("the label is inside the target", label > target && label_end < target_end); + CHECK("they overlap, either way round", overlaps(target, target_end, label, label_end) && overlaps(label, label_end, target, target_end)); + CHECK("ranges that touch are apart", !overlaps(target, target_end, target_end, target_end + 4096) && + !overlaps(target_end, target_end + 4096, target, target_end)); + CHECK("one byte shared", overlaps(target, target_end, target_end - 1, target_end + 4096)); + CHECK("apart", !overlaps(target, target_end, target_end + 4096, target_end + 8192)); + CHECK("an empty range overlaps nothing", !overlaps(target, target, target - 1, target_end) && !overlaps(target, target_end, label, label)); + CHECK("the same range", overlaps(target, target_end, target, target_end)); + + // the walk: candidates in any order, the others noted one by one + int self_target = 0, self_label = 0, self_far = 0, other = 0; + std::vector c; + c.push_back({label, label_end, 7, &self_label, 0}); + c.push_back({target_end + (64ull << 20), target_end + (65ull << 20), 9, &self_far, 0}); + c.push_back({target, target_end, 3, &self_target, 0}); + const uint64_t longest = by_start(c); + CHECK("in the order of their starts", c[0].lo == target && c[1].lo == label && c[2].self == &self_far); + CHECK("each keeps its place in the caller's list", c[0].index == 3 && c[1].index == 7 && c[2].index == 9); + CHECK("the longest", longest == 8640ull * 1024); + note(c, longest, target, target_end, 3804, &self_target); // (the target itself) + CHECK("a candidate is not another image over its own storage", c[0].newest == 0); + CHECK("but it is one over the label's", c[1].newest == 3804 && c[2].newest == 0); + note(c, longest, label, label_end, 3987, &self_label); // (the label itself) + CHECK("the label, drawn later, is the target's newest other", c[0].newest == 3987 && c[1].newest == 3804); + CHECK("the target's write-back is stale, the label's is not", + write_back_stale(3804, c[0].newest) && !write_back_stale(3987, c[1].newest)); + note(c, longest, target + 4096, target + 8192, 1468, &other); // (an older image inside the target) + CHECK("an older one does not lower it", c[0].newest == 3987); + note(c, longest, target_end, target_end + 4096, 5000, &other); // (one that touches the target's end) + CHECK("one that only touches counts nothing", c[0].newest == 3987 && c[1].newest == 3804 && c[2].newest == 0); + note(c, longest, target - 4096, target, 5000, &other); // (and its start) + CHECK("nor one that ends where it starts", c[0].newest == 3987); + note(c, longest, target - (16ull << 20), target + 1, 4100, &other); // (a long one from far before, a byte into it) + CHECK("a long image from before the start is found", c[0].newest == 4100 && c[1].newest == 3804); + note(c, longest, 0, ~0ull, 4200, &other); // (one over everything) + CHECK("one over everything reaches all", c[0].newest == 4200 && c[1].newest == 4200 && c[2].newest == 4200); + note(c, longest, label, label, 9000, &other); // (an empty one) + CHECK("an empty image counts nothing", c[1].newest == 4200); + { // (no candidates: nothing to do, nothing read) + std::vector none; + const uint64_t l = by_start(none); + note(none, l, target, target_end, 1, &other); + CHECK("no candidates", l == 0 && none.empty()); + } + { // (many candidates side by side, one other over three of them) + std::vector v; + static int selves[64]; + for (int i = 63; i >= 0; i--) v.push_back({0x1000ull * (uint64_t)i, 0x1000ull * (uint64_t)i + 0x1000, (size_t)i, &selves[i], 0}); + const uint64_t l = by_start(v); + note(v, l, 0x1000ull * 10 + 0x800, 0x1000ull * 12 + 1, 77, &other); + for (int i = 0; i < 64; i++) CHECK("only the three it covers", v[(size_t)i].newest == (i >= 10 && i <= 12 ? 77u : 0u)); + } + std::printf("evict reason: all checks passed\n"); + return 0; +} diff --git a/tests/test_evict_reason.py b/tests/test_evict_reason.py new file mode 100644 index 0000000..e959abe --- /dev/null +++ b/tests/test_evict_reason.py @@ -0,0 +1,221 @@ +"""An eviction leaves out the write-back of a texture another image drew into since (gpu/vk/evict_reason.h, +vk_texture.cpp texture_sweep and drawn_others; no game, no GPU). + +(Triforce, 11-22, the performance audit's TEX-R404, after Source A's.) Over the budget the eviction writes each chosen +texture the GPU drew back into the game's memory. The game draws its HUD labels inside the storage of a 1920 x 1080 +target it no longer uses; when the target went, its old texels landed over the labels, and the labels were uploaded from +them as noise. + +Three parts: + 1. tests/test_evict_reason.cpp against the header: Source A's frames (3804 against 3987 stale; 0, 1468 and 3804 not), + the overlap on the trace's addresses (the target at 0x1001e88fc00, 8,640 KB; the label at 0x1001f039400 inside it; + ranges that touch are apart), the walk's candidates and the switch's values. + 2. drawn_others and texture_sweep's block, cut out of vk_texture.cpp and compiled with stand-ins: the target's + write-back is left out and the label's kept, one drawn in the same frame, one that touches, an image the GPU never + drew and one of no size change nothing, the skips are counted and logged; with RX_VK_EVICT_STALE_FLUSH=1 every + write-back stays, as before. + 3. the source: the block sits between the loop that chooses and the batches that write back, the bookkeeping loop is + as it was, drawn_others walks the cache and not the pass's ordered list, the switch is in the switches table. + + python -I -B tests/test_evict_reason.py [--cxx clang++] +""" +import argparse +import os +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +BLOCK_HEAD = ' static const bool stale_flush = evict_reason::flush_stale(getenv("RX_VK_EVICT_STALE_FLUSH"));\n' +BATCHES = " { // (the write-backs: batches of up to 64 textures or ~512 MB of staging memory, each one submission and one wait)\n" + +HARNESS = r''' +#include +#include +#include +#include +#include +#include +#include +#include "evict_reason.h" + +struct Texture { + uint64_t gpu = 0, guest_size = 0, last_written = 0; + bool gpu_written = false; + uint32_t id = 0, width = 0, height = 0; +}; +static std::unordered_map> textures; +static int logged; +static void logf(const char* fmt, ...) { + char line[512]; + va_list ap; + va_start(ap, fmt); + std::vsnprintf(line, sizeof line, fmt, ap); + va_end(ap); + logged++; + std::printf("log: %s\n", line); +} +struct Evict { Texture* tex; bool flush, shifted; }; + +@DRAWN_OTHERS@ + +static uint64_t pass(std::vector& evict) { +@BLOCK@ + return st_stale_skipped; +} + +static int fails; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) + +static Texture* add(uint64_t gpu, uint64_t size, uint64_t written, bool drawn = true) { + static int ids; + auto t = std::make_unique(); + t->gpu = gpu, t->guest_size = size, t->last_written = written, t->gpu_written = drawn, t->id = (uint32_t)++ids; + t->width = t->height = 16; + Texture* p = t.get(); + textures[ids] = std::move(t); + return p; +} + +int main(int argc, char** argv) { + const bool as_before = argc > 1 && argv[1][0] == '1'; // (run with RX_VK_EVICT_STALE_FLUSH=1) + const uint64_t target_at = 0x1001e88fc00ull, target_size = 8640ull * 1024; + Texture* target = add(target_at, target_size, 3804); + Texture* label = add(0x1001f039400ull, 110ull * 110 * 4, 3987); // (inside the target, drawn later) + Texture* beside = add(target_at + target_size, 1 << 20, 3804); // (touches the target's end) + Texture* touching = add(target_at + target_size + (1 << 20), 4096, 3990); // (touches `beside`, drawn later) + Texture* pair_a = add(0x20000000000ull, 1 << 20, 500); // (two drawn in the same frame, overlapping) + Texture* pair_b = add(0x20000000000ull + 4096, 1 << 20, 500); + Texture* cpu = add(0x30000000000ull, 1 << 20, 100); + add(0x30000000000ull + 4096, 4096, 900, false); // (over `cpu`, but never drawn by the GPU) + Texture* sized = add(0x40000000000ull, 1 << 20, 100); + add(0x40000000000ull, 0, 900); // (an image of no size at its start) + Texture* unflushed = add(0x50000000000ull, 1 << 20, 100); // (chosen, not written back: not asked) + add(0x50000000000ull + 4096, 4096, 900); + Texture* older = add(0x60000000000ull, 1 << 20, 800); + add(0x60000000000ull + 8192, 4096, 799); // (inside it, drawn a frame before) + + std::vector evict = {{beside, true, false}, {label, true, false}, {pair_b, true, false}, {target, true, false}, + {pair_a, true, false}, {cpu, true, false}, {sized, true, false}, {unflushed, false, true}, + {older, true, false}}; + (void)touching; + const uint64_t skipped = pass(evict); + const auto flush_of = [&](const Texture* t) { + for (const Evict& e : evict) + if (e.tex == t) return e.flush; + return false; + }; + CHECK(evict.size() == 9 && evict[0].tex == beside && evict[3].tex == target && evict[7].tex == unflushed); // (the list as it was) + CHECK(evict[7].shifted && !evict[7].flush); + if (as_before) { + CHECK(skipped == 0 && logged == 0); + CHECK(flush_of(target) && flush_of(label) && flush_of(beside) && flush_of(pair_a) && flush_of(pair_b) && flush_of(cpu) && + flush_of(sized) && flush_of(older)); + } else { + CHECK(!flush_of(target)); // (the label was drawn inside it since) + CHECK(flush_of(label)); // (the target over it was drawn before it) + CHECK(flush_of(beside)); // (its neighbours only touch it) + CHECK(flush_of(pair_a) && flush_of(pair_b)); // (the same frame keeps both) + CHECK(flush_of(cpu) && flush_of(sized) && flush_of(older)); + CHECK(skipped == 1 && logged == 1); + // a second pass: the count goes on, and a write-back already left out is not asked again + std::vector again = {{target, false, false}, {older, true, false}}; + add(0x60000000000ull + 16384, 4096, 801); // (now a later image inside `older`) + CHECK(pass(again) == 2 && !again[1].flush && !again[0].flush && logged == 2); + // the first 20 are logged, the rest counted + for (int i = 0; i < 30; i++) { + std::vector more = {{older, true, false}}; + pass(more); + } + std::vector last = {{older, true, false}}; + CHECK(pass(last) == 33 && logged == 20); + } + if (!fails) std::printf("the eviction's stale write-backs (%s): all checks passed\n", as_before ? "the switch set" : "the default"); + return fails != 0; +} +''' + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def function(text, head): + start = text.index(head) + return text[start:text.index("\n}\n", start) + 3] + + +def check_source(tex): + check('#include "evict_reason.h"' in tex, "vk_texture.cpp does not include evict_reason.h") + others = function(tex, "void drawn_others(std::vector& cands) {") + check("for (auto& kv : textures) {" in others and "order" not in others, + "drawn_others does not walk the cache itself (a pass's ordered list is filtered)") + check("if (!o->gpu_written || !o->guest_size) continue;" in others and + "evict_reason::note(cands, longest, o->gpu, o->gpu + o->guest_size, o->last_written, o);" in others, + "drawn_others does not note every sized image the GPU drew with its last drawing") + start = tex.index("void texture_sweep(uint64_t completed, bool sweep, uint64_t budget, uint64_t evict_to, bool known, uint64_t used, " + "uint64_t allowed,\n bool dead_only, uint64_t hold_to, uint64_t rule_to) {") + sweep = tex[start:] + at = [sweep.index(n) for n in ( + " for (auto& [last, tex] : order) {\n // (total_bytes can have drifted", # the loop that chooses + " evict.push_back({tex, !no_flush && !overwritten && tex->gpu_written && !tex->released && flushable(*tex), shifted});", + " come_back_noted.store(true, std::memory_order_relaxed);", + BLOCK_HEAD, + " if (evict[i].flush && evict[i].tex->guest_size)", + " if (!cands.empty()) drawn_others(cands);", + " if (!evict_reason::write_back_stale(tex->last_written, c.newest)) continue;\n evict[c.index].flush = false;", + BATCHES, + " if (!e.flush) continue;\n batch.push_back(Flush{e.tex, e.tex->gpu, e.tex->gpu + e.tex->guest_size});", + " for (const Evict& e : evict) { // (and each one's bookkeeping and its end, as before)", + " destroy_later(*tex);\n forget_gpu(tex);\n gone.push_back(tex);")] + check(at == sorted(at), "the stale write-backs are not left out between the loop that chooses and the batches that write back") + check(sweep.count("drawn_others(") == 1 and sweep.count("for (auto& [last, tex] : order)") == 6, + "texture_sweep asks drawn_others more than once, or walks its ordered list a new time") + check("(%llu not: \"\n" in sweep[sweep.index(BATCHES):] and "(unsigned long long)st_stale_skipped);" in sweep[sweep.index(BATCHES):], + "the write-backs' line no longer counts the ones left out") + docs = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("| `RX_VK_EVICT_STALE_FLUSH=1` |" in docs and "`gpu/vk/evict_reason.h`" in docs, + "docs/development.md: RX_VK_EVICT_STALE_FLUSH is not in the switches table") + block = sweep[sweep.index(BLOCK_HEAD):sweep.index(BATCHES)] + return others, block + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + check((VK / "evict_reason.h").is_file(), "gpu/vk/evict_reason.h is not there") + tex = (VK / "vk_texture.cpp").read_text(encoding="utf-8") + others, block = check_source(tex) + flags = ["-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror"] + with tempfile.TemporaryDirectory(prefix="evict-reason-") as tmp: + rule = Path(tmp) / "rule.exe" + subprocess.run([args.cxx, *flags, str(ROOT / "tests" / "test_evict_reason.cpp"), "-o", str(rule)], check=True) + r = subprocess.run([str(rule)], capture_output=True, text=True) + print((r.stdout + r.stderr).rstrip()) + check(r.returncode == 0, "the rule's checks failed") + cpp, exe = Path(tmp) / "pass.cpp", Path(tmp) / "pass.exe" + cpp.write_text(HARNESS.replace("@DRAWN_OTHERS@", others).replace("@BLOCK@", block), encoding="utf-8") + subprocess.run([args.cxx, *flags, "-I" + str(VK), str(cpp), "-o", str(exe)], check=True) + env = {k: v for k, v in os.environ.items() if k != "RX_VK_EVICT_STALE_FLUSH"} + for value, arg in ((None, "0"), ("0", "0"), ("1", "1")): + e = dict(env) + if value is not None: + e["RX_VK_EVICT_STALE_FLUSH"] = value + r = subprocess.run([str(exe), arg], capture_output=True, text=True, env=e) + tail = [line for line in r.stdout.splitlines() if not line.startswith("log: ")] + print("\n".join(tail)) + check(r.returncode == 0, "the cut-out pass with RX_VK_EVICT_STALE_FLUSH %s:\n%s" % ("unset" if value is None else "=" + value, r.stdout)) + if value is None: + check("log: vk: texture #1 0x1001e88fc00 16x16 the GPU last drew in frame 3804 goes without its write-back: another image " + "was drawn over its storage in frame 3987 (1 so far; RX_VK_EVICT_STALE_FLUSH=1: written back, as before)" in r.stdout, + "the skip's log line does not name the texture and both frames:\n" + r.stdout) + print("PASS: an eviction leaves out the write-back of a texture another image drew into since") + + +if __name__ == "__main__": + main() diff --git a/tests/test_failed_compile_keeps_objects.py b/tests/test_failed_compile_keeps_objects.py index aa36915..015c7fd 100644 --- a/tests/test_failed_compile_keeps_objects.py +++ b/tests/test_failed_compile_keeps_objects.py @@ -10,10 +10,17 @@ Game-free: build.py's own lines, authored. python -I -B tests/test_failed_compile_keeps_objects.py """ +import ast +import contextlib import importlib.util +import io +import os from pathlib import Path import sys +import tempfile +import types import unittest +from unittest import mock ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "nx2")) @@ -51,8 +58,140 @@ class WhichFailureKeepsTheObjects(unittest.TestCase): body = body[body.index('run(PY + ["nx2/build.py", "--standalone"'):body.index("compile_seconds = time.time() - compile_started")] asked, deleted = body.index("if compile_alone_failed(said):"), body.index('os.path.join(work, "objcache")') self.assertLess(asked, deleted) - # (the lift is forgotten either way: a lifted file cut short fails the same way at every try otherwise) + # (the lift is forgotten before that, a lifted file cut short fails the same way at every try otherwise; unless the + # compile failed only in the program's own files, once: the performance audit's LIFT-27, KeptLift below) self.assertLess(body.index("os.remove(key_file)"), asked) + self.assertLess(body.index("lift_kept = keep_lift_after_failure(said, work)"), body.index("os.remove(key_file)")) + self.assertIn("if os.path.exists(key_file) and not lift_kept:", body) + + +def failed_build_handler(): + """The statements main() runs when nx2/build.py failed, cut out of make_game.py: its except block, behind a raise""" + source = (ROOT / "nx2/make_game.py").read_text(encoding="utf-8") + main = next(n for n in ast.parse(source).body if isinstance(n, ast.FunctionDef) and n.name == "main") + tries = [n for n in ast.walk(main) if isinstance(n, ast.Try) and any( + isinstance(c, ast.Constant) and c.value == "nx2/build.py" for s in n.body for c in ast.walk(s))] + assert len(tries) == 1 and len(tries[0].handlers) == 1 + stop = ast.parse("raise SystemExit(1)").body + block = ast.Try(body=stop, handlers=tries[0].handlers, orelse=[], finalbody=[]) + return compile(ast.fix_missing_locations(ast.Module(body=[block], type_ignores=[])), "make_game.py", "exec") + + +class KeptLift(unittest.TestCase): + """(11-22, the performance audit's LIFT-27, after Source A's) A compile that failed only in the program's own files + keeps the translated game too, once: failed_only_own_sources, keep_lift_after_failure, and main()'s own except block + run on a made-up work folder""" + OWN = ["FAILED vk_draw.cpp (1 MB, 12s, alone too):", "vk_draw.cpp:10:1: error: a made-up error", "compiled in 412s, 1 failed"] + + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="triforce-kept-lift-") + self.addCleanup(self.temporary.cleanup) + self.work = Path(self.temporary.name) / "work" + self.gen = self.work / "sa" / "gen" + self.gen.mkdir(parents=True) + (self.work / "sa" / "build" / "gen").mkdir(parents=True) + for name in ("code_000.c", "code_hot_0.c", "a64_table.c", "a64_fpx.h"): + (self.gen / name).write_text("// lifted\n") + (self.work / "sa" / "build" / "gen" / "ui_fonts.cpp").write_text("// generated\n") + self.key = self.gen / make_game.LIFT_KEY + self.key.write_text("key\nprint\n") + self.objects = self.work / "objcache" + self.objects.mkdir() + (self.objects / "one.o").write_bytes(b"object") + self.mark = self.work / "sa" / make_game.LIFT_KEPT_ONCE + environ = {k: v for k, v in os.environ.items() if k != "HOOKSHOT_KEEP_LIFT"} + patched = mock.patch.dict(os.environ, environ, clear=True) + patched.start() + self.addCleanup(patched.stop) + + def lines(self, *names, count=None): + return ["FAILED %s (3 MB, 40s, alone too):" % n for n in names] + \ + ["compiled in 412s, %d failed" % (len(names) if count is None else count)] + + def test_only_the_programs_own_files(self): + own = lambda *names, **k: make_game.failed_only_own_sources(self.lines(*names, **k), str(self.work)) + self.assertTrue(own("vk_draw.cpp")) + self.assertTrue(own("hos_main.c", "vk_shader.cpp", "imgui.cpp")) + self.assertTrue(make_game.failed_only_own_sources(self.OWN, str(self.work))) + self.assertFalse(own("code_021.c")) # (a lifted file) + self.assertFalse(own("code_hot_3.c")) + self.assertFalse(own("vk_draw.cpp", "code_021.c")) # (one of them is) + self.assertFalse(own("a64_table.c")) # (a file of the lift's folder) + self.assertFalse(own("ui_fonts.cpp")) # (a file the build generated) + self.assertFalse(own("vk_draw.cpp", count=2)) # (a failed file that was not named) + self.assertFalse(own()) # (none named) + self.assertFalse(make_game.failed_only_own_sources([], str(self.work))) + self.assertFalse(make_game.failed_only_own_sources(["FAILED vk_draw.cpp (1 MB, 2s, alone too):"], str(self.work))) + + def test_kept_once_and_not_twice_in_a_row(self): + keep = lambda lines, environ=None: make_game.keep_lift_after_failure(lines, str(self.work), environ) + self.assertTrue(keep(self.OWN)) + self.assertTrue(self.mark.exists()) + self.assertFalse(keep(self.OWN)) # (the same kind of failure again: the lift is forgotten this time) + self.assertFalse(self.mark.exists()) + self.assertTrue(keep(self.OWN)) # (and after that lift, once again) + self.mark.unlink() + self.assertFalse(keep(self.lines("code_021.c"))) + self.assertFalse(self.mark.exists()) + self.assertFalse(keep(["compiled in 412s, 0 failed", "link FAILED (3s), 20 undefined symbols"])) + self.assertFalse(keep(self.OWN + ["link FAILED (3s), 1 undefined symbols"])) + self.assertFalse(keep([])) + self.assertFalse(keep(self.OWN, {"HOOKSHOT_KEEP_LIFT": "0"})) # (the switch: forgotten after every failed build) + self.assertFalse(self.mark.exists()) + self.assertTrue(keep(self.OWN, {"HOOKSHOT_KEEP_LIFT": "1"})) + + def fail_build(self, said): + """main()'s own except block after a failed build.py that said these lines: what it printed""" + stopped = [] + printed = io.StringIO() + scope = dict(make_game.__dict__, said=list(said), work=str(self.work), + prompts=types.SimpleNamespace(stop=lambda: stopped.append(True))) + with contextlib.redirect_stdout(printed), self.assertRaises(SystemExit): + exec(failed_build_handler(), scope) + self.assertEqual(stopped, [True]) + return printed.getvalue() + + def test_the_failed_builds_own_statements(self): + # (the program's own file failed: the key and the objects stay) + said = self.fail_build(self.OWN) + self.assertTrue(self.key.exists() and (self.objects / "one.o").exists() and self.mark.exists()) + self.assertIn("the translated game and the files that did compile are kept", said) + # (the same again at the next build: the key goes, the objects stay) + said = self.fail_build(self.OWN) + self.assertFalse(self.key.exists() or self.mark.exists()) + self.assertTrue((self.objects / "one.o").exists()) + self.assertIn("the next build translates the game again and compiles only the rest", said) + # (a lifted file failed: the key goes at once) + self.key.write_text("key\nprint\n") + said = self.fail_build(self.lines("code_021.c")) + self.assertFalse(self.key.exists() or self.mark.exists()) + self.assertTrue((self.objects / "one.o").exists()) + self.assertIn("the next build translates the game again and compiles only the rest", said) + # (a link failed: the key and the objects go, as before) + self.key.write_text("key\nprint\n") + said = self.fail_build(["compiled in 412s, 0 failed", "link FAILED (3s), 20 undefined symbols"]) + self.assertFalse(self.key.exists() or self.objects.exists()) + self.assertIn("nothing of this one is reused", said) + + def test_the_switch_forgets_after_every_failed_build(self): + with mock.patch.dict(os.environ, {"HOOKSHOT_KEEP_LIFT": "0"}): + said = self.fail_build(self.OWN) + self.assertFalse(self.key.exists() or self.mark.exists()) + self.assertTrue((self.objects / "one.o").exists()) + self.assertIn("the next build translates the game again and compiles only the rest", said) + + def test_the_lines_and_the_marks_place(self): + build = (ROOT / "nx2/build.py").read_text(encoding="utf-8") + self.assertIn('print("FAILED %s (%d MB, %.0fs, alone too):\\n%s" % (os.path.basename(src), size, secs, err), flush=True)', build) + source = (ROOT / "nx2/make_game.py").read_text(encoding="utf-8") + self.assertIn('if line.startswith("compiled in ") or "link FAILED" in line or line.startswith("FAILED "):', source) + # (the mark is not in sa/gen: gen_print vouches for every file there, and a new one would be a new lift) + self.assertIn('mark = os.path.join(work, "sa", LIFT_KEPT_ONCE)', source) + after = source[source.index(" prompts.stop()\n raise\n"):source.index("compile_seconds = time.time() - compile_started")] + self.assertIn('os.remove(os.path.join(work, "sa", LIFT_KEPT_ONCE))', after) + before = make_game.gen_print(str(self.gen)) + make_game.keep_lift_after_failure(self.OWN, str(self.work)) + self.assertEqual(make_game.gen_print(str(self.gen)), before) if __name__ == "__main__": diff --git a/tests/test_fallback_buffer_cleanup.py b/tests/test_fallback_buffer_cleanup.py index e3cfcd7..7472e18 100644 --- a/tests/test_fallback_buffer_cleanup.py +++ b/tests/test_fallback_buffer_cleanup.py @@ -11,7 +11,9 @@ RUN_ALL_ARGS = ["--cxx", "{cxx}"] ROOT = Path(__file__).resolve().parents[1] CASES = ("initial_allocation_failure", "payload_allocation_failure", "records_address_zero", "records_only_address_zero", "payload_address_zero", "records_only_success", "task_success", - "invalid_program", "program_isolation") + "invalid_program", "program_isolation", + # ((11-21, the performance audit's MEM-154) a payload growth keeps the thread's records buffer) + "payload_growth_keeps_records", "shared_records_payload_failure") def block(source, signature, semicolon=False): diff --git a/tests/test_fallback_loaded.py b/tests/test_fallback_loaded.py new file mode 100644 index 0000000..5b265b9 --- /dev/null +++ b/tests/test_fallback_loaded.py @@ -0,0 +1,199 @@ +"""The mesh fallback's programs already in memory are handed to the precompile too (gpu/vk/vk_shader.cpp +program_cache_load_some; vk_draw.cpp precompile_run; no game, no GPU). + +(Triforce, 11-22, the performance audit's PIPE-62, after Source A's.) On the task and mesh fallback the precompile builds +the fallback programs' compute pipelines on workers before the title, for the programs program_cache_load_some hands it +with `fallbacks` set. That call handed over only the records still on the disk; in a start that prepares, +state_list_prepare has taken the state lists' records before it (with fallbacks off), so most fallback programs play draws +got no pipelines before the title, and each one's first draw waited up to 50 ms and was left out until they were built. +The call now also hands over every fallback program already in memory that is ready and not failed. +RX_VK_FALLBACK_LOADED=0: not, as before. + +Cuts the end of program_cache_load_some (from its lock to its return) out of vk_shader.cpp and runs it over authored +programs: what was read from the disk is handed over as before, with what was in memory under the same key, each once; +a fallback program in memory under another key, ready and not failed, is handed over too, one that is not ready, failed +or no fallback program is not; the count returned and the log line say so; without `fallbacks`, or with the switch at 0, +only what came from the disk. + +Reads the source: the walk is after the loop over what was read, under the same shader_mu, skips that loop's keys, tests +fallback, failed and ready (acquire), and is behind the switch; precompile_run still asks with fallbacks true and keeps +the fallback programs for its workers, which take compute_state first; the edit lies inside vk_shader.cpp's part marked +as outside the translation stamp (tests/test_program_carry.py holds the stamp itself); the switches table has the row. + + python -I -B tests/test_fallback_loaded.py [--cxx clang++] +""" +import argparse +import os +import subprocess +import tempfile +from pathlib import Path + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] + +FIXTURE = r""" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define CHECK(x) do { if (!(x)) { std::printf("line %d: %s\n", __LINE__, #x); return 1; } } while (0) + +struct Program { + uint64_t key = 0; + int id = 0; + bool fallback = false, failed = false; + std::atomic ready{false}; +}; +struct DiskRecord { int kind; }; +std::mutex shader_mu; +std::unordered_map>> programs; +std::string said; +int lines; +void logf(const char* f, ...) { + char b[512]; + va_list ap; + va_start(ap, f); + vsnprintf(b, sizeof b, f, ap); + va_end(ap); + said = b, lines++; +} +static std::unique_ptr make(uint64_t key, int id, bool fallback, bool ready = true, bool failed = false) { + auto p = std::make_unique(); + p->key = key, p->id = id, p->fallback = fallback, p->failed = failed, p->ready = ready; + return p; +} +// program_cache_load_some's end, as it is: `work` and `made` are what its workers read from the disk +size_t tail(std::vector>>& work, std::vector>>& made, + const std::function& fn, bool fallbacks) { +/* TAIL */ +} + +int main(int argc, char** argv) { + const bool off = argc > 1 && argv[1][0] == '0'; + // in memory before the call (what state_list_prepare and the draws read in) + programs[0xA].push_back(make(0xA, 1, true)); // (and a record of the same key still on the disk: below) + programs[0xB].push_back(make(0xB, 2, true)); // a fallback program, ready + programs[0xC].push_back(make(0xC, 3, true, false)); // one still being translated + programs[0xD].push_back(make(0xD, 4, true, true, true)); // one that failed + programs[0xE].push_back(make(0xE, 5, false)); // a compute or native program + programs[0xF].push_back(make(0xF, 6, true)); // a key with two variants, one of them a fallback's + programs[0xF].push_back(make(0xF, 7, false)); + const auto run = [&](bool fallbacks, std::vector* got) { + std::vector>> work; + work.push_back({0xA, {DiskRecord{4}}}); + work.push_back({0x9, {DiskRecord{4}}}); + std::vector>> made(2); + made[0].push_back(make(0xA, 10 + (int)got->size(), true)); + made[1].push_back(make(0x9, 20 + (int)got->size(), true)); + got->clear(); + return tail(work, made, [&](Program& p) { got->push_back(p.id); }, fallbacks); + }; + const auto count = [](const std::vector& v, int id) { + int n = 0; + for (const int x : v) n += x == id; + return n; + }; + std::vector got; + // without `fallbacks` (the state list's call): only what was read, with what its keys had in memory + size_t n = run(false, &got); + CHECK(n == 3 && got.size() == 3 && count(got, 1) == 1 && count(got, 10) == 1 && count(got, 20) == 1 && lines == 0); + // with it (the precompile's) + n = run(true, &got); + if (off) { // (RX_VK_FALLBACK_LOADED=0: as before) + CHECK(n == 5 && got.size() == 5 && count(got, 2) == 0 && count(got, 6) == 0 && lines == 0); + std::printf("ok off\n"); + return 0; + } + // (0xA has three variants now: the one in memory and the two read; 0x9 two) + CHECK(n == 7 && got.size() == 7); + CHECK(count(got, 1) == 1 && count(got, 10) == 1 && count(got, 13) == 1); // (a key that was read: each of its variants once, not again) + CHECK(count(got, 2) == 1 && count(got, 6) == 1); // (the fallback programs in memory under other keys) + CHECK(count(got, 3) == 0 && count(got, 4) == 0 && count(got, 5) == 0 && count(got, 7) == 0); // (not ready, failed, no fallback) + CHECK(lines == 1 && said == "vk: 2 mesh fallback programs read in earlier in this start are made ready with the rest"); + // one that became ready since is handed over at a later call + programs[0xC][0]->ready = true; + n = run(true, &got); + CHECK(count(got, 3) == 1 && count(got, 2) == 1 && lines == 2 && said.rfind("vk: 3 mesh fallback programs", 0) == 0); + std::printf("ok on\n"); + return 0; +} +""" + + +def block(text, marker): + start = text.index(marker) + at = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[at] == "{") - (text[at] == "}") + at += 1 + return text[start:at] + + +def need(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def ordered(text, parts, why): + last = 0 + for p in parts: + k = text.find(p, last) + need(k >= 0, why + " (missing, or out of order: " + p + ")") + last = k + len(p) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + shader = (ROOT / "gpu/vk/vk_shader.cpp").read_text(encoding="utf-8") + load = block(shader, "size_t program_cache_load_some(") + ordered(load, ["std::lock_guard lk(shader_mu);", "for (size_t i = 0; i < work.size(); i++) {", "for (auto& p : variants) n++, fn(*p);", + "if (fallbacks) {", 'static const bool loaded = !(getenv("RX_VK_FALLBACK_LOADED") && atoi(getenv("RX_VK_FALLBACK_LOADED")) == 0);', + "if (loaded) {", "for (const auto& w : work) taken.insert(w.first);", "for (auto& kv : programs) {", + "if (taken.count(kv.first)) continue;", + "if (p->fallback && !p->failed && p->ready.load(std::memory_order_acquire)) found++, n++, fn(*p);", + 'logf("vk: %zu mesh fallback programs read in earlier in this start are made ready with the rest", found);', "return n;"], + "program_cache_load_some does not hand over the fallback programs already in memory after what it read, under the same lock") + need(load.count("std::lock_guard lk(shader_mu);") == 1 and load.count("return n;") == 1, "the walk is not under the loop's own lock") + need("Only a GPU without mesh shaders has such programs" not in load, "the comment that only a GPU without mesh shaders has such programs is back") + # the edit is inside the part of vk_shader.cpp marked as outside the translation stamp + out_mark, in_mark = "// ---- (Triforce) Outside the translation stamp", "// ---- (Triforce) Inside the translation stamp again" + at = shader.index("size_t program_cache_load_some(") + need(shader.rfind(in_mark, 0, at) < shader.rfind(out_mark, 0, at) < at < at + len(load) < shader.index(in_mark, at), + "program_cache_load_some is no longer inside a part marked as outside the translation stamp") + need(shader.count("RX_VK_FALLBACK_LOADED") == load.count("RX_VK_FALLBACK_LOADED"), "the switch is read outside program_cache_load_some") + draw = (ROOT / "gpu/vk/vk_draw.cpp").read_text(encoding="utf-8") + pre = block(draw, "void precompile_run(void (*progress)(size_t done, size_t total)) {") + ordered(pre, ["const size_t programs_n = program_cache_load_some(wanted, [&](Program& p) {", "ensure_layout(p);", + "else if (p.fallback) fallbacks.push_back(&p);", "}, all_computes, true);", + "Program& p = *fallbacks[i - computes.size()];", "if (p.compute_state.compare_exchange_strong(none, 1)) {", + "fallback_compute_pipelines(p);", "p.compute_state.store(2, std::memory_order_release);"], + "precompile_run does not ask with fallbacks true, keep the fallback programs, or take compute_state before it builds") + docs = (ROOT / "docs/development.md").read_text(encoding="utf-8") + need("`RX_VK_FALLBACK_LOADED=0`" in docs, "docs/development.md: the switches table has no RX_VK_FALLBACK_LOADED row") + tail = load[load.index(" std::lock_guard lk(shader_mu);"):load.rindex("}")] + env = {k: v for k, v in os.environ.items() if not k.startswith("RX_VK_")} + with tempfile.TemporaryDirectory(prefix="fallback-loaded-") as tmp: + cpp, exe = Path(tmp) / "fixture.cpp", Path(tmp) / "fixture.exe" + cpp.write_text(FIXTURE.replace("/* TAIL */", tail), encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", str(cpp), "-o", str(exe)], check=True) + for arg, more, want in (("on", {}, "ok on"), ("0", {"RX_VK_FALLBACK_LOADED": "0"}, "ok off")): + r = subprocess.run([str(exe), arg], capture_output=True, text=True, timeout=60, env={**env, **more}) + need(r.returncode == 0 and r.stdout.strip() == want, "program_cache_load_some's end (%s): %s" % (arg, r.stdout.strip() or r.returncode)) + print(r.stdout.strip()) + print("PASS: the mesh fallback's programs already in memory are handed to the precompile; RX_VK_FALLBACK_LOADED=0 as before") + + +if __name__ == "__main__": + main() diff --git a/tests/test_fallback_size_policy.py b/tests/test_fallback_size_policy.py new file mode 100644 index 0000000..bc6082b --- /dev/null +++ b/tests/test_fallback_size_policy.py @@ -0,0 +1,185 @@ +"""Mesh-fallback compute stages too big for RADV on GCN or Mesa's Intel driver are not built (gpu/vk/fallback_size_policy.h). + +(Triforce, 11-22, the performance audit's PIPE-56, after Source A's.) No game, no GPU. Every fallback stage went to the +driver; on those two drivers the largest ones took tens of GB and minutes and ended with the game killed. A program with +a stage over its driver's limit is now left out before anything is built, as a program the driver refused is. + +Compiles a small driver against the header and checks: + the counters OpPhi is opcode 245 and OpFunctionCall 57; a made-up module with 3 phis and 2 calls counts 3 and 2; + a cut-off module, a wrong magic word and no module count 0 of both; + the driver RADV on GCN first, then Mesa's Intel driver, any other: none; + the limits GCN: 50,000 phis, no call limit; Mesa's Intel driver: 20,000 phis, 200 calls; any other driver: none, + whatever the switches say; + the switches unset, empty or not a number: the driver's own; 0: none; a number: that number; + refused just over a limit yes, at the limit no; 79,947 phis on GCN yes, on Mesa's Intel driver yes, on any other + driver no; 251 calls on Mesa's Intel driver yes, on GCN no; nothing with both limits 0. + +And reads the source: fallback_compute_pipelines asks the rule after the failed-set lookup and before the first stage is +built, only where fallback_size_driver() names one of the two drivers, and puts the program into the failed set; +vk_shader.cpp counts in make_module, moves the counts in program_pack and forgets them in stage_module_destroy, all +between its first pair of "Outside the translation stamp" marks; and the switches table has the two rows. + + python -I -B tests/test_fallback_size_policy.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +DRIVER = r''' +#include "fallback_size_policy.h" +#include +#include +#include + +using namespace fallback_size; +static int checks, fails; +static void want(const char* what, bool got, bool wanted) { + checks++; + if (got != wanted) fails++, printf("FAIL %s: %d, want %d\n", what, (int)got, (int)wanted); +} + +int main() { + want("OpPhi is 245", kOpPhi == 245, true); + want("OpFunctionCall is 57", kOpFunctionCall == 57, true); + std::vector m{0x07230203, 0x00010600, 0, 100, 0}; + const auto op = [&m](uint32_t code, std::initializer_list words) { + m.push_back((uint32_t)((words.size() + 1) << 16) | code); + for (uint32_t w : words) m.push_back(w); + }; + op(17, {1}); + for (int i = 0; i < 3; i++) op(245, {2, 20u + i, 7, 8, 9, 10}); // OpPhi + op(57, {3, 30, 4}); // OpFunctionCall + op(57, {3, 31, 4, 30}); + op(253, {}); + want("3 phis", spirv_phis(m.data(), m.size()) == 3, true); + want("2 calls", spirv_calls(m.data(), m.size()) == 2, true); + std::vector cut = m; + cut.back() = (5u << 16) | 245; // (the last instruction, one word, made one longer than what is left) + want("a cut-off module counts 0", count(cut.data(), cut.size()).phis == 0 && count(cut.data(), cut.size()).calls == 0, true); + std::vector wrong = m; + wrong[0] = 0; + want("a wrong magic word counts 0", spirv_phis(wrong.data(), wrong.size()) == 0 && spirv_calls(wrong.data(), wrong.size()) == 0, true); + want("no module counts 0", spirv_phis(nullptr, 0) == 0, true); + + want("RADV on GCN", driver_of(true, false) == Driver::RadvGcn, true); + want("Mesa's Intel driver", driver_of(false, true) == Driver::MesaIntel, true); + want("any other driver", driver_of(false, false) == Driver::Other, true); + + const Limits gcn = limits(Driver::RadvGcn, nullptr, nullptr), intel = limits(Driver::MesaIntel, nullptr, nullptr); + want("GCN: 50,000 phis, no call limit", gcn.phis == 50000 && gcn.calls == 0, true); + want("Mesa's Intel driver: 20,000 phis, 200 calls", intel.phis == 20000 && intel.calls == 200, true); + for (const char* v : {(const char*)nullptr, "", "0", "5", "100000"}) { + const Limits other = limits(Driver::Other, v, v); + want("no limit on any other driver, whatever is set", other.phis == 0 && other.calls == 0, true); + want("nothing refused on any other driver", refused(other, Size{4000000, 4000000}), false); + } + want("unset: the driver's own", parse_limit(nullptr, 50000) == 50000, true); + want("empty: the driver's own", parse_limit("", 50000) == 50000, true); + want("not a number: the driver's own", parse_limit("many", 50000) == 50000 && parse_limit("12x", 50000) == 50000, true); + want("0: none", parse_limit("0", 50000) == 0, true); + want("a number: that number", parse_limit("30000", 50000) == 30000, true); + want("a number too large: the largest", parse_limit("99999999999", 5) == 0xffffffffu, true); + const Limits set = limits(Driver::MesaIntel, "0", "300"); + want("the switches on Mesa's Intel driver", set.phis == 0 && set.calls == 300, true); + + want("just over the phi limit", refused(gcn, Size{50001, 0}), true); + want("at the phi limit", refused(gcn, Size{50000, 0}), false); + want("79,947 phis on GCN", refused(gcn, Size{79947, 12}), true); + want("79,947 phis on Mesa's Intel driver", refused(intel, Size{79947, 12}), true); + want("251 calls on Mesa's Intel driver", refused(intel, Size{100, 251}), true); + want("200 calls on Mesa's Intel driver", refused(intel, Size{100, 200}), false); + want("251 calls on GCN: no call limit there", refused(gcn, Size{100, 251}), false); + want("both limits 0: nothing refused", refused(limits(Driver::RadvGcn, "0", "0"), Size{4000000, 4000000}), false); + want("a stage nobody counted is built", refused(intel, Size{}) || refused(gcn, Size{}), false); + if (fails) printf("fallback size policy: FAILED (%d of %d checks)\n", fails, checks); + else printf("fallback size policy: all %d checks passed\n", checks); + return fails != 0; +} +''' + + +def check_source(): + draw = (ROOT / "gpu/vk/vk_draw.cpp").read_text(encoding="utf-8") + at = draw.index("bool fallback_compute_pipelines(Program& p) {") + body = draw[at:draw.index("\n}\n", at)] + order = [body.find(n) for n in ( + "if (failed.count({p.key, p.variant})) return false;", + "if (const fallback_size::Driver size_driver = fallback_size_driver(); size_driver != fallback_size::Driver::Other) {", + 'fallback_size::limits(size_driver, getenv("RX_VK_FALLBACK_PHI_LIMIT"), getenv("RX_VK_FALLBACK_CALL_LIMIT"));', + "const fallback_size::Size size = stage_size(st);", + "if (!fallback_size::refused(limits, size)) continue;", + "failed.insert({p.key, p.variant});", + "std::vector made(p.stages.size(), VK_NULL_HANDLE);", + "guarded_compute(pcache_use().cache, &ci, &made[i])")] + if min(order) < 0 or order != sorted(order): + raise SystemExit("fallback_compute_pipelines does not ask the size rule after the failed-set lookup and before the first " + "build, putting a program over the limit into the failed set") + if "return false;" not in body[order[5]:order[6]]: + raise SystemExit("fallback_compute_pipelines goes on to build a program the size rule put into the failed set") + if body.count("failed.insert({p.key, p.variant});") != 2: + raise SystemExit("fallback_compute_pipelines: wanted the failed set written by the size rule and by a refused build") + if "size_told++ < 8" not in body: + raise SystemExit("fallback_compute_pipelines: the size rule's log line is not capped at 8") + + shader = (ROOT / "gpu/vk/vk_shader.cpp").read_text(encoding="utf-8") + out_mark, in_mark = "// ---- (Triforce) Outside the translation stamp", "// ---- (Triforce) Inside the translation stamp again" + lo = shader.index(out_mark) + hi = shader.index(in_mark, lo) + between = shader[lo:hi] + for needle in ("void size_note(VkShaderModule m, const std::vector& code) {", + "if (!m || fallback_size_driver() == fallback_size::Driver::Other) return;", + "void size_move(VkShaderModule from, VkShaderModule to) {", + "size_move(st.module, (VkShaderModule)(uintptr_t)code.get());", + "fallback_size::Driver fallback_size_driver() {", + "fallback_size::driver_of(dev.radv_gcn, dev.driver_id == VK_DRIVER_ID_INTEL_OPEN_SOURCE_MESA);", + "fallback_size::Size stage_size(const Stage& st) {", + "const auto it = size_of.find(stage_exact_key(st));", + "size_of.erase(m);"): + if needle not in between: + raise SystemExit("vk_shader.cpp: not between the first pair of translation stamp marks: " + needle) + if shader.count("fallback_size") != between.count("fallback_size") or shader.count("size_of") != between.count("size_of"): + raise SystemExit("vk_shader.cpp names the size rule outside the marks: the translation stamp would move") + make = between[between.index("VkShaderModule make_module("):] + if make.count("size_note(m, code);") != 2: + raise SystemExit("make_module does not count each module it makes") + pack = between[between.index("void program_pack(Program& p) {"):] + pack = pack[:pack.index("\n}\n")] + if pack.index("exact_move(st.module,") > pack.index("size_move(st.module,") or \ + pack.index("size_move(st.module,") > pack.index("vkDestroyShaderModule(dev.device, st.module, nullptr);"): + raise SystemExit("program_pack does not move a stage's counts with its key before the module goes") + destroy = between[between.index("void stage_module_destroy(VkShaderModule m) {"):] + if "size_of.erase(m);" not in destroy[:destroy.index("\n}\n")]: + raise SystemExit("stage_module_destroy does not forget a module's counts") + header = (ROOT / "gpu/vk/vk_shader.h").read_text(encoding="utf-8") + for needle in ('#include "fallback_size_policy.h"', "fallback_size::Driver fallback_size_driver();", + "fallback_size::Size stage_size(const Stage& st);"): + if needle not in header: + raise SystemExit("vk_shader.h: no " + needle) + docs = (ROOT / "docs/development.md").read_text(encoding="utf-8") + if "| `RX_VK_FALLBACK_PHI_LIMIT=` |" not in docs or "| `RX_VK_FALLBACK_CALL_LIMIT=` |" not in docs or \ + "`gpu/vk/fallback_size_policy.h`" not in docs: + raise SystemExit("docs/development.md: the switches table lacks the two rows for the fallback's size limits") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="g++") + args = parser.parse_args() + check_source() + with tempfile.TemporaryDirectory(prefix="fallback-size-") as tmp: + cpp, exe = Path(tmp) / "fallback_size_test.cpp", Path(tmp) / "fallback-size.exe" + cpp.write_text(DRIVER, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-Wall", "-Wextra", "-Werror", "-I" + str(ROOT / "gpu/vk"), str(cpp), "-o", str(exe)], + check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + print(r.stdout.rstrip()) + if r.returncode: + raise SystemExit("FAIL: the fallback size policy's checks failed") + + +if __name__ == "__main__": + main() diff --git a/tests/test_fg_motion.py b/tests/test_fg_motion.py new file mode 100644 index 0000000..3539522 --- /dev/null +++ b/tests/test_fg_motion.py @@ -0,0 +1,96 @@ +"""NVIDIA DLSS frame generation is given the camera's motion alone, not the upscaler chain's (gpu/vk/fg_motion.h, +vk_fsr3_inputs.cpp chain_motion; no game, no GPU). + +(Triforce, 11-22, the owner's DLSS frame generation flicker.) With the chain's motion copied over FSR's inputs (FG-4) the +whole picture flickered again while the camera moved, with the clip-space fix of 11-20 in the build. The owner's runs: +RX_VK_FG_OWN_MOTION=0 (the camera's alone) had no flicker; the chain's motion with the upscaler audit's commits all set back +had it. + +This compiles fg_motion.h alone and checks the rule: with DLSS frame generation the chain's motion is not given, unless +RX_VK_DLSSG_OWN_MOTION is set and not 0; without it (AMD's frame generation, FSR 3.1's interpolation) it is, as before; +RX_VK_FG_OWN_MOTION=0 keeps the camera's alone for every generator, over RX_VK_DLSSG_OWN_MOTION too; the switches read as +the renderer reads them (unset, empty, 0). + +And reads the source: chain_motion asks the rule with dlssg::active() and both switches before it takes the lock or copies +anything, and returns false then; the old switch's line stays (tests/test_framegen_amd.py reads it); and the switch is in +the switches table. + + python -I -B tests/test_fg_motion.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +PROGRAM = r""" +#include +#include "fg_motion.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +using namespace fg_motion; +int main() { + CHECK(!set_on(nullptr) && !set_on("") && !set_on("0") && set_on("1") && set_on("2")); + CHECK(!camera_alone(nullptr) && !camera_alone("") && camera_alone("0") && !camera_alone("1")); + // (DLSS frame generation: the camera's alone by default; the chain's with RX_VK_DLSSG_OWN_MOTION=1) + CHECK(!chain(true, nullptr, nullptr)); + CHECK(!chain(true, nullptr, "0") && !chain(true, nullptr, "")); + CHECK(chain(true, nullptr, "1")); + // (AMD's frame generation, FSR 3.1's interpolation: the chain's, as before; the DLSS switch says nothing to them) + CHECK(chain(false, nullptr, nullptr) && chain(false, nullptr, "0") && chain(false, nullptr, "1")); + CHECK(chain(false, "1", nullptr)); + // (RX_VK_FG_OWN_MOTION=0: the camera's alone for every generator, over the DLSS switch too) + CHECK(!chain(false, "0", nullptr) && !chain(true, "0", nullptr) && !chain(true, "0", "1") && !chain(false, "0", "1")); + return fails ? 1 : 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + src = (VK / "vk_fsr3_inputs.cpp").read_text(encoding="utf-8") + + check('#include "fg_motion.h"' in src, "vk_fsr3_inputs.cpp does not use the rule") + motion = block(src, "bool chain_motion(VkCommandBuffer cmd, VkImage motion, uint32_t width, uint32_t height) {") + ask = 'if (!fg_motion::chain(dlssg::active(), getenv("RX_VK_FG_OWN_MOTION"), getenv("RX_VK_DLSSG_OWN_MOTION"))) return false;' + check(ask in motion, "chain_motion does not ask the rule with DLSS frame generation's state and both switches") + check(motion.index(ask) < motion.index("std::lock_guard lk(s.mu);") < motion.index("vkCmdCopyImage(cmd, motion, VK_IMAGE_LAYOUT_GENERAL, s.motion.image"), + "chain_motion copies or takes its lock before the rule says whether the chain's motion is given") + check(motion.count("vkCmdCopyImage(") == 1 and "s.frame.chain_motion = true;" in motion, + "chain_motion copies more than once, or no longer notes the chain's motion") + check('env_switch("RX_VK_FG_OWN_MOTION") == 0' in motion, "the old switch's line is gone (tests/test_framegen_amd.py reads it)") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`RX_VK_DLSSG_OWN_MOTION=1`" in dev, "docs/development.md: RX_VK_DLSSG_OWN_MOTION=1 is not in the switches table") + + with tempfile.TemporaryDirectory(prefix="fg-motion-") as tmp: + c, exe = Path(tmp) / "motion.cpp", Path(tmp) / "motion.exe" + c.write_text(PROGRAM, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(VK), str(c), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + check(r.returncode == 0, "the rule as made:\n" + r.stdout) + print("PASS: NVIDIA DLSS frame generation is given the camera's motion alone, the chain's only with RX_VK_DLSSG_OWN_MOTION=1") + + +if __name__ == "__main__": + main() diff --git a/tests/test_finalized_obsolete.py b/tests/test_finalized_obsolete.py new file mode 100644 index 0000000..0e0533b --- /dev/null +++ b/tests/test_finalized_obsolete.py @@ -0,0 +1,144 @@ +"""A finalized texture the GPU drew goes once the game has rewritten its storage (gpu/vk/finalized_obsolete.h, +vk_texture.cpp texture_sweep; no game, no GPU). + +(Triforce, 11-21, the performance audit's TEX-43, after Source A's.) A texture the GPU drew in one frame is kept after +the game finalizes it, for being made again over that storage. Within the budget only a draw over one let it go: the +game's own CPU writes to that memory kept each for good, about 200 render target images an hour in players' logs. + +This compiles finalized_obsolete.h alone: which textures can be one (finalized, GPU-written, in no virtual pool, 32 KB +or more), three quarters of the storage as the line, the look again every 60 presents, the switch; and 30 laps of 60 +tile-sized targets whose storage the game then reuses: 1,800 alive without the rule, 120 with it. + +And reads the source: the sweep's look uses the texture's own fields and write generations since its release, keeps +its answer with the texture, and a release starts it afresh; keep_finalized refuses an obsolete one; it counts as dead +in the hold's walk and as overwritten in the pass, so it is never written back; what goes is counted; the switch is in +the switches table. + + python -I -B tests/test_finalized_obsolete.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def in_order(text, parts, why): + at = 0 + for p in parts: + at = text.find(p, at) + check(at >= 0, why + " (missing, or out of order: " + p + ")") + at += len(p) + + +PROGRAM = r""" +#include +#include +#include +#include "finalized_obsolete.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +using namespace finalized_obsolete; +struct Tex { bool released, gpu_written; uint64_t gpu, guest_size, written; bool obsolete; uint64_t checked; }; +// the sweep's look, as vk_texture.cpp makes it of the rule +static bool look(Tex& t, uint64_t frame, bool keep_all) { + if (keep_all || !candidate(t.released, t.gpu_written, t.gpu, t.guest_size)) return false; + if (!t.obsolete && due(frame, t.checked)) t.checked = frame, t.obsolete = rewritten(t.written, t.guest_size); + return t.obsolete; +} +int main() { + CHECK(!keep(nullptr) && !keep("") && !keep("0") && keep("1")); + CHECK(candidate(true, true, 0x8000000000ull, 32u << 10) && candidate(true, true, 1, 135200)); + CHECK(!candidate(false, true, 1, 135200)); // (not finalized: the game still has it) + CHECK(!candidate(true, false, 1, 135200)); // (uploaded from memory: its own rules) + CHECK(!candidate(true, true, 1ull << 40, 135200)); // (a virtual pool: its pages are mapped, not written) + CHECK(!candidate(true, true, 1, (32u << 10) - 1)); // (a small one shares its pages) + CHECK(rewritten(3, 4) && rewritten(4, 4) && rewritten(101400, 135200) && !rewritten(101399, 135200) && !rewritten(0, 135200)); + CHECK(!rewritten(0, 0)); + CHECK(due(100, 0) && !due(100, 41) && due(100, 40) && due(1000, 1) && !due(5, 5)); + + // 30 laps, each makes 60 tile-sized targets (260 x 260 RG8) the GPU draws and the game finalizes; a lap later the game + // has put other data in the storage of all but the newest two laps' + for (const bool keep_all : {true, false}) { + std::vector alive; + uint64_t frame = 1; + for (int lap = 0; lap < 30; lap++) { + for (int i = 0; i < 60; i++) alive.push_back({true, true, 0x100000ull + alive.size() * 0x40000, 135200, 0, false, 0}); + for (size_t i = 0; i + 120 < alive.size() + 0 && i < alive.size(); i++) alive[i].written = 135200; // (older than two laps: rewritten) + frame += 600; + std::vector kept; + for (Tex& t : alive) + if (!look(t, frame, keep_all)) kept.push_back(t); + alive.swap(kept); + } + if (keep_all) CHECK(alive.size() == 1800); + else CHECK(alive.size() == 120); + } + { // one not rewritten yet is not looked at again before 60 presents; once rewritten it stays so + Tex t{true, true, 1, 135200, 0, false, 0}; + CHECK(!look(t, 100, false) && t.checked == 100); + t.written = 135200; + CHECK(!look(t, 159, false) && t.checked == 100); + CHECK(look(t, 160, false) && t.checked == 160); + t.written = 0; + CHECK(look(t, 400, false) && t.checked == 160); + } + return fails ? 1 : 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + tex = (VK / "vk_texture.cpp").read_text(encoding="utf-8").replace("\r\n", "\n") + start = tex.rindex("void texture_sweep(") + sweep = tex[start:] + check('#include "finalized_obsolete.h"' in tex, "vk_texture.cpp does not use the rule") + in_order(sweep, ['static const bool keep_obsolete = finalized_obsolete::keep(getenv("RX_VK_KEEP_OBSOLETE"));', + "const auto obsolete = [&](const Texture* tex) {", + "if (keep_obsolete || !finalized_obsolete::candidate(tex->released, tex->gpu_written, tex->gpu, tex->guest_size)) return false;", + "if (!tex->obsolete && finalized_obsolete::due(frame_no, tex->obsolete_checked)) {", + "tex->obsolete_checked = frame_no;", + "memory_written_runs(tex->guest, tex->guest_size, tex->release_gen, written, true);", + "tex->obsolete = finalized_obsolete::rewritten(sum, tex->guest_size);", "return tex->obsolete;", + "const auto keep_finalized = [&](const Texture* tex) {", + "tex->bytes > (4ull << 20) || obsolete(tex))", + "if (shifted || drawn_over(tex) || obsolete(tex)) hold_dead += tex->bytes + tex->pad;", + "const bool stale = obsolete(tex);", "const bool overwritten = shifted || drawn_over(tex) || stale;", + "if (sweep && !overwritten && !gone_early) continue;", + "!no_flush && !overwritten && tex->gpu_written && !tex->released && flushable(*tex)", + "if (stale && !shifted && !drawn_over(tex)) {", "st_obsolete_n"], + "texture_sweep does not look at a finalized texture's storage by the rule, keep its answer, refuse it among the kept, " + "count it dead in the hold's walk and overwritten in the pass, or count what goes") + check("it->second->obsolete = false, it->second->obsolete_checked = 0;" in tex and + # ((Triforce, 11-22, the performance audit's MEM-85) the release's generation is texture_released_at's gen) + tex.index("it->second->released = true, it->second->release_gen = gen;") < + tex.index("it->second->obsolete = false, it->second->obsolete_checked = 0;"), + "a release does not start the look afresh: a texture rewritten before it stays obsolete after it") + head = (VK / "vk_texture.h").read_text(encoding="utf-8") + check("mutable bool obsolete = false;" in head and "mutable uint64_t obsolete_checked = 0;" in head, + "the texture does not keep the look's answer") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`RX_VK_KEEP_OBSOLETE=1`" in dev, "docs/development.md: RX_VK_KEEP_OBSOLETE is not in the switches table") + + with tempfile.TemporaryDirectory(prefix="finalized-obsolete-") as tmp: + c, exe = Path(tmp) / "obsolete.cpp", Path(tmp) / "obsolete.exe" + c.write_text(PROGRAM, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(VK), str(c), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + check(r.returncode == 0, "the rule as made:\n" + r.stdout) + print("PASS: a finalized texture the GPU drew goes once the game has rewritten its storage") + + +if __name__ == "__main__": + main() diff --git a/tests/test_finish_installed.py b/tests/test_finish_installed.py new file mode 100644 index 0000000..fa49c5c --- /dev/null +++ b/tests/test_finish_installed.py @@ -0,0 +1,380 @@ +"""The finish passes over files the game folder already has from the same source (nx2/make_game.py INSTALLED_NOTE). + +(Triforce, 11-22, the performance audit's IO-45, after Source A's.) Every build copied every file into the game folder +as .new and swapped it in: the exe, the renderer, the prebuilt DLLs and image.bin, changed or not. The finish now +notes, per installed file, its source's SHA-1 and the installed copy's size, time, file number and mode, and the next +finish passes over a file whose source hashes the same and whose installed copy is still as noted. The installed file +is never read. HOOKSHOT_FINISH_COPY_ALL=1 copies everything and keeps no note, as before. + +Runs the production finish statements themselves (tests/test_finish_output.py's finish_statements cuts them out of +main()) on made-up files in a temporary folder: + - a first finish copies everything and writes the note; a second with the same sources calls neither shutil.copy2 nor + os.replace, opens no installed file, and leaves the files and the note as they were; + - a changed source is copied, alone, also when its size and time are the old ones; a source only touched is not; + - an installed file changed by hand, a note that is broken or of another format, a file the note does not have, and a + file system that gives no file number (st_ino 0) all copy; + - the launcher layout (the game in bin, the launcher beside it) is passed over file by file; + - a .new left beside a file that is passed over is removed, and a read-only one is left locked; + - a swap that is refused leaves no note saying the new file is there, and the retry installs it; + - the switch copies everything and removes the note; + - after every finish the installed files are the sources byte for byte. + +The exe and the renderer are linked again at every build, and lld writes the link's time into them, so they are still +copied. HOOKSHOT_LINK_REPRO=1 (nx2/build.py repro_link, off unless asked: it changes the linked files' bytes) adds +/Brepro to those two links. Checked here: the rule and its two hooks, and, with --cc, that two links of one tiny object +a second apart are the same bytes with the flag and are not without it. + + python -I -B tests/test_finish_installed.py [--cc clang] +""" +import ast +import contextlib +import hashlib +import io +import json +import os +from pathlib import Path +import shutil +import stat +import subprocess +import sys +import tempfile +import time +import types +import unittest +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +RUN_ALL_ARGS = ["--cc", "clang"] +sys.path.insert(0, str(ROOT / "tests")) +from test_finish_output import FINISH, make_game # (the production statements, and the module they came from) + + +class NoNumber: + """An os.stat result on a file system that gives no file number""" + def __init__(self, result): + self._result = result + + st_ino = 0 + + def __getattr__(self, name): + return getattr(self._result, name) + + +class FinishInstalled(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory(prefix="triforce-finish-installed-") + self.addCleanup(self.directory.cleanup) + self.root = Path(self.directory.name).resolve() + self.addCleanup(self.unlock) + self.work, self.out = self.root / "work", self.root / "game" + self.built = self.work / "sa" / "build" + self.built.mkdir(parents=True) + self.out.mkdir() + (self.work / "sa" / "image").mkdir() + self.exe, self.dll = self.built / "Triforce.exe", self.built / "triforce_vk.dll" + self.image = self.work / "sa" / "image" / "image.bin" + self.exe.write_bytes(b"a made-up executable") + self.dll.write_bytes(b"a made-up renderer") + self.image.write_bytes(b"a made-up image") + self.sources = {"Triforce.exe": self.exe, "triforce_vk.dll": self.dll, "image.bin": self.image} + self.note = self.out / make_game.INSTALLED_NOTE + environ = {k: v for k, v in os.environ.items() if k != "HOOKSHOT_FINISH_COPY_ALL"} + patched = mock.patch.dict(os.environ, environ, clear=True) + patched.start() + self.addCleanup(patched.stop) + + def unlock(self): + for path in self.root.rglob("*"): + if path.is_file(): + path.chmod(stat.S_IREAD | stat.S_IWRITE) + + def finish(self, launcher=False): + """The production finish; returns (the sources copied, the targets swapped in, what it printed)""" + if launcher: + (self.root / "HookshotBuilder-Setup.exe").write_bytes(b"a made-up launcher") + copied, swapped, said = [], [], io.StringIO() + copy2, replace = make_game.shutil.copy2, os.replace + + def copy(src, dst): + copied.append(Path(src).name) + return copy2(src, dst) + + def swap(src, dst): + swapped.append(Path(dst).name) + return replace(src, dst) + + with mock.patch.object(make_game, "BESIDE", ["Triforce.exe", "triforce_vk.dll"]), \ + mock.patch.object(make_game.shutil, "copy2", side_effect=copy), \ + mock.patch.object(make_game.os, "replace", side_effect=swap), contextlib.redirect_stdout(said): + namespace = dict(make_game.__dict__, built=str(self.built), out=str(self.out), work=str(self.work), + ROOT=str(self.root / "tools"), LINUX=False if launcher else make_game.LINUX) + exec(FINISH, namespace) + return copied, swapped, said.getvalue() + + def assert_installed(self, folder=None): + for name, source in self.sources.items(): + where = self.out / "image.bin" if name == "image.bin" else (folder or self.out) / name + self.assertEqual(where.read_bytes(), source.read_bytes(), name) + + def test_second_finish_copies_and_swaps_nothing(self): + copied, swapped, said = self.finish() + self.assertEqual(copied, ["Triforce.exe", "triforce_vk.dll", "image.bin"]) + self.assertEqual(swapped, ["Triforce.exe", "triforce_vk.dll", "image.bin", make_game.INSTALLED_NOTE]) + self.assert_installed() + note = json.loads(self.note.read_text(encoding="utf-8")) + self.assertEqual(sorted(note["files"]), ["Triforce.exe", "image.bin", "triforce_vk.dll"]) + self.assertEqual(note["files"]["Triforce.exe"]["sha1"], make_game.file_sha1(str(self.exe))) + before = {p.name: (p.read_bytes(), p.stat().st_mtime_ns, p.stat().st_ino) for p in self.out.iterdir()} + opened = [] + real_open = open + + def watch(path, *a, **k): + opened.append(os.path.normcase(os.path.abspath(path))) + return real_open(path, *a, **k) + + with mock.patch("builtins.open", side_effect=watch): + copied, swapped, said = self.finish() + self.assertEqual((copied, swapped), ([], [])) + self.assertIn("the game folder has 3 of this build's 3 files already", said) + # (the installed files are never read: only the sources, for their hashes, and the note) + inside = [p for p in opened if p.startswith(os.path.normcase(str(self.out)) + os.sep)] + self.assertEqual(inside, [os.path.normcase(str(self.note))]) + self.assertEqual(before, {p.name: (p.read_bytes(), p.stat().st_mtime_ns, p.stat().st_ino) for p in self.out.iterdir()}) + + def test_changed_source_is_copied_alone(self): + self.finish() + self.dll.write_bytes(b"a made-up renderer, built again") + copied, swapped, said = self.finish() + self.assertEqual(copied, ["triforce_vk.dll"]) + self.assertEqual(swapped, ["triforce_vk.dll", make_game.INSTALLED_NOTE]) + self.assertIn("2 of this build's 3 files already (Triforce.exe, image.bin)", said) + self.assert_installed() + self.assertEqual(self.finish()[:2], ([], [])) + + def test_changed_source_of_the_same_size_and_time_is_copied(self): + self.finish() + times = self.exe.stat() + self.exe.write_bytes(b"A MADE-UP EXECUTABLE") # (the same length) + os.utime(self.exe, ns=(times.st_atime_ns, times.st_mtime_ns)) + self.assertEqual(self.finish()[0], ["Triforce.exe"]) + self.assert_installed() + + def test_touched_source_is_not_copied(self): + self.finish() + times = self.image.stat() + os.utime(self.image, ns=(times.st_atime_ns, times.st_mtime_ns + 5_000_000_000)) + self.assertEqual(self.finish()[:2], ([], [])) + + def test_installed_file_changed_by_hand_is_copied(self): + self.finish() + (self.out / "image.bin").write_bytes(b"another image, longer than the one installed") + self.assertEqual(self.finish()[0], ["image.bin"]) + self.assert_installed() + # the same length, written later: its time is not the noted one + installed = self.out / "Triforce.exe" + times = installed.stat() + installed.write_bytes(b"x" * times.st_size) + os.utime(installed, ns=(times.st_atime_ns, times.st_mtime_ns + 2_000_000_000)) + self.assertEqual(self.finish()[0], ["Triforce.exe"]) + self.assert_installed() + # deleted + (self.out / "triforce_vk.dll").unlink() + self.assertEqual(self.finish()[0], ["triforce_vk.dll"]) + self.assert_installed() + + def test_broken_or_foreign_note_copies_everything(self): + self.finish() + good = self.note.read_text(encoding="utf-8") + for text in ("", "{ not json", "[1, 2]", json.dumps({"format": 2, "files": json.loads(good)["files"]}), + json.dumps({"format": 1, "files": [1]}), json.dumps({"format": 1, "files": {"Triforce.exe": "x"}})): + with self.subTest(note=text[:24]): + self.note.write_text(text, encoding="utf-8") + self.assertEqual(self.finish()[0], ["Triforce.exe", "triforce_vk.dll", "image.bin"]) + self.assert_installed() + self.assertEqual(self.finish()[:2], ([], [])) + self.note.unlink() # (deleting the note copies everything) + self.assertEqual(self.finish()[0], ["Triforce.exe", "triforce_vk.dll", "image.bin"]) + + def test_file_the_note_does_not_have_is_copied(self): + self.finish() + note = json.loads(self.note.read_text(encoding="utf-8")) + del note["files"]["image.bin"] + self.note.write_text(json.dumps(note), encoding="utf-8") + self.assertEqual(self.finish()[0], ["image.bin"]) + self.assertEqual(sorted(json.loads(self.note.read_text(encoding="utf-8"))["files"]), + ["Triforce.exe", "image.bin", "triforce_vk.dll"]) + + def test_no_file_number_always_copies(self): + real = os.stat + with mock.patch.object(make_game.os, "stat", side_effect=lambda p, *a, **k: NoNumber(real(p, *a, **k))): + self.assertIsNone(make_game.installed_stat(str(self.exe))) + for _ in range(2): + self.assertEqual(self.finish()[0], ["Triforce.exe", "triforce_vk.dll", "image.bin"]) + self.assert_installed() + self.assertFalse(self.note.exists() and json.loads(self.note.read_text(encoding="utf-8"))["files"]) + + def test_launcher_layout_is_passed_over_file_by_file(self): + copied, swapped, said = self.finish(launcher=True) + self.assertEqual(copied, ["Triforce.exe", "triforce_vk.dll", "image.bin", "HookshotBuilder-Setup.exe"]) + self.assertEqual((self.out / "Triforce.exe").read_bytes(), b"a made-up launcher") + self.assert_installed(self.out / "bin") + note = json.loads(self.note.read_text(encoding="utf-8"))["files"] + self.assertEqual(sorted(note), ["Triforce.exe", "bin/Triforce.exe", "bin/triforce_vk.dll", "image.bin"]) + self.assertEqual(self.finish(launcher=True)[:2], ([], [])) + self.exe.write_bytes(b"a made-up executable, built again") + copied, swapped, said = self.finish(launcher=True) + self.assertEqual(copied, ["Triforce.exe"]) + self.assertEqual((self.out / "Triforce.exe").read_bytes(), b"a made-up launcher") # (the launcher: passed over) + self.assert_installed(self.out / "bin") + + def test_leftover_new_beside_a_passed_over_file_goes_unless_locked(self): + self.finish() + left = self.out / "image.bin.new" + left.write_bytes(b"an interrupted build's copy") + self.assertEqual(self.finish()[:2], ([], [])) + self.assertFalse(left.exists()) + if os.name == "nt": # (a read-only file is not deleted there, and the finish never unlocks one) + left.write_bytes(b"an interrupted build's copy") + left.chmod(stat.S_IREAD) + with mock.patch.object(make_game.os, "chmod", wraps=os.chmod) as chmod: + self.assertEqual(self.finish()[:2], ([], [])) + chmod.assert_not_called() + self.assertEqual(left.read_bytes(), b"an interrupted build's copy") + self.assertFalse(left.stat().st_mode & stat.S_IWRITE) + + def test_refused_swap_notes_nothing_and_the_retry_installs(self): + self.finish() + old_note = self.note.read_text(encoding="utf-8") + self.exe.write_bytes(b"a made-up executable, built again") + said = io.StringIO() + with mock.patch.object(make_game, "BESIDE", ["Triforce.exe", "triforce_vk.dll"]), \ + mock.patch.object(make_game.os, "replace", side_effect=PermissionError("a made-up refusal")), \ + contextlib.redirect_stdout(said), self.assertRaises(SystemExit): + exec(FINISH, dict(make_game.__dict__, built=str(self.built), out=str(self.out), work=str(self.work), + ROOT=str(self.root / "tools"))) + self.assertIn("@@error Could not replace", said.getvalue()) + self.assertEqual(self.note.read_text(encoding="utf-8"), old_note) + self.assertEqual((self.out / "Triforce.exe").read_bytes(), b"a made-up executable") + copied, swapped, said = self.finish() + self.assertEqual(copied, ["Triforce.exe"]) + self.assert_installed() + + def test_switch_copies_everything_and_keeps_no_note(self): + self.finish() + self.assertTrue(self.note.exists()) + with mock.patch.dict(os.environ, {"HOOKSHOT_FINISH_COPY_ALL": "1"}): + for _ in range(2): + copied, swapped, said = self.finish() + self.assertEqual(copied, ["Triforce.exe", "triforce_vk.dll", "image.bin"]) + self.assertEqual(swapped, ["Triforce.exe", "triforce_vk.dll", "image.bin"]) + self.assertFalse(self.note.exists()) + self.assertNotIn("not copied again", said) + self.assert_installed() + with mock.patch.dict(os.environ, {"HOOKSHOT_FINISH_COPY_ALL": "0"}): + self.assertEqual(len(self.finish()[0]), 3) + self.assertEqual(self.finish()[:2], ([], [])) + + def test_helpers(self): + self.assertEqual(make_game.file_sha1(str(self.exe)), hashlib.sha1(b"a made-up executable").hexdigest()) + self.assertIsNone(make_game.file_sha1(str(self.root / "missing"))) + self.assertIsNone(make_game.installed_stat(str(self.root / "missing"))) + self.assertIsNone(make_game.installed_stat(str(self.out))) # (a folder is no installed file) + st = self.exe.stat() + self.assertEqual(make_game.installed_stat(str(self.exe)), [st.st_size, st.st_mtime_ns, st.st_ino, st.st_mode]) + self.assertEqual(make_game.installed_name(str(self.out), str(self.out / "bin" / "Triforce.exe")), "bin/Triforce.exe") + self.assertEqual(make_game.read_installed(str(self.out)), {}) + note = {"Triforce.exe": {"sha1": "abc", "stat": make_game.installed_stat(str(self.exe))}} + self.assertFalse(make_game.installed_same(note, str(self.built), str(self.exe), None)) + self.assertFalse(make_game.installed_same(note, str(self.built), str(self.exe), "abd")) + self.assertTrue(make_game.installed_same(note, str(self.built), str(self.exe), "abc")) + self.assertFalse(make_game.installed_same({}, str(self.built), str(self.exe), "abc")) + + def test_source_and_docs(self): + source = (ROOT / "nx2" / "make_game.py").read_text(encoding="utf-8") + main = source[source.index("def main(a=None):"):] + for needle in ('copy_all = os.environ.get("HOOKSHOT_FINISH_COPY_ALL", "0") not in ("", "0")', + "installed_note = {} if copy_all else read_installed(out)", + "if installed_same(installed_note, out, dst, digest):", + "write_installed(out, installed_now)"): + self.assertTrue(needle in main, needle) + self.assertTrue(main.index("if installed_same(installed_note, out, dst, digest):") < + main.index("shutil.copy2(src, dst + \".new\")") < main.index("write_installed(out, installed_now)")) + self.assertEqual(make_game.INSTALLED_NOTE, ".triforce_installed.json") + docs = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + rows = [line for line in docs.splitlines() if line.startswith("| `HOOKSHOT_FINISH_COPY_ALL=1` |")] + self.assertEqual(len(rows), 1) + self.assertTrue("IO-45" in rows[0] and ".triforce_installed.json" in rows[0]) + + +class ReproLink(unittest.TestCase): + """nx2/build.py repro_link: HOOKSHOT_LINK_REPRO=1 takes the link's time out of the exe and the renderer (off unless + asked: it changes the linked files' bytes), so that the finish above can pass an unchanged exe over""" + + def rule(self, link_opt=lambda o: "<" + o + ">"): + source = (ROOT / "nx2" / "build.py").read_text(encoding="utf-8") + tree = ast.parse(source) + function = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "repro_link"] + scope = {"os": os, "LINUX": False, "toolchain": types.SimpleNamespace(link_opt=link_opt)} + exec(compile(ast.Module(body=function, type_ignores=[]), "build.py", "exec"), scope) + return scope["repro_link"], source + + def test_rule_and_hooks(self): + repro, source = self.rule() + self.assertEqual(repro(""), []) + self.assertEqual(repro("0"), []) + self.assertEqual(repro("yes"), []) + self.assertEqual(repro("1"), [""]) + self.assertEqual(repro("1", linux=True), []) + with mock.patch.dict(os.environ, {"HOOKSHOT_LINK_REPRO": "1"}): + self.assertEqual(repro(), [""]) + environ = {k: v for k, v in os.environ.items() if k != "HOOKSHOT_LINK_REPRO"} + with mock.patch.dict(os.environ, environ, clear=True): + self.assertEqual(repro(), []) # (the default: the link is the command it was) + # the two Windows links that the finish installs ask it: the exe's and the renderer's + self.assertEqual(source.count("repro_link()"), 2) + exe = source[source.index('exe = os.path.join(out, "Triforce.exe")'):] + self.assertTrue("repro_link() +" in exe[:exe.index("capture_output=True")]) + dll = source[source.index('["-shared", "-o", vdll, toolchain.link_opt("/map:" + vdll[:-4] + ".map")'):] + self.assertTrue("+ repro_link()" in dll[:dll.index("\n")]) + docs = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + self.assertEqual(len([l for l in docs.splitlines() if l.startswith("| `HOOKSHOT_LINK_REPRO=1` |") and "IO-45" in l]), 1) + + def test_two_links_of_one_object_are_the_same_bytes_only_with_the_flag(self): + cc = CC and shutil.which(CC) + if os.name != "nt" or not cc: + self.skipTest("needs --cc on Windows") + target = subprocess.run([cc, "-dumpmachine"], capture_output=True, text=True).stdout.strip() + if "windows-gnu" not in target and "mingw" not in target: + self.skipTest("not llvm-mingw: %s" % target) + repro, _ = self.rule(lambda o: "-Wl,-Xlink=" + o) # (toolchain.link_opt's form for llvm-mingw) + with tempfile.TemporaryDirectory(prefix="triforce-repro-link-") as folder: + source = os.path.join(folder, "tiny.c") + with open(source, "w") as f: + f.write("int main(void) { return 0; }\n") + subprocess.run([cc, "-c", source, "-o", os.path.join(folder, "tiny.o")], check=True) + + def link(flags): + digests = [] + for i in range(2): + exe = os.path.join(folder, "tiny.exe") + subprocess.run([cc, os.path.join(folder, "tiny.o"), "-o", exe] + flags, check=True) + digests.append(make_game.file_sha1(exe)) + os.remove(exe) + if i == 0: + time.sleep(1.2) # (the link's time is in whole seconds) + return digests + + first, second = link(repro("1")) + self.assertEqual(first, second) + first, second = link(repro("0")) + self.assertNotEqual(first, second, "this linker writes no link time: the switch has nothing to do") + + +CC = None + +if __name__ == "__main__": + if "--cc" in sys.argv: + at = sys.argv.index("--cc") + CC = sys.argv[at + 1] + del sys.argv[at:at + 2] + unittest.main() diff --git a/tests/test_finish_output.py b/tests/test_finish_output.py index b1d9ef5..73559c4 100644 --- a/tests/test_finish_output.py +++ b/tests/test_finish_output.py @@ -1,6 +1,9 @@ """Exercise actual production stage-all/copy/swap statements with authored temporary files. python -I -B tests/test_finish_output.py + +(The fixture's program is Triforce.exe, the name the build gives it: under its old name the finish's own cleanup of old +programs removed it, and five of these tests stopped on a missing file.) """ import ast import contextlib @@ -49,7 +52,7 @@ class FinishOutputTests(unittest.TestCase): self.built = self.work / "sa" / "build" self.built.mkdir(parents=True) self.out.mkdir() - self.source, self.destination = self.built / "hookshot.exe", self.out / "hookshot.exe" + self.source, self.destination = self.built / "Triforce.exe", self.out / "Triforce.exe" self.source.write_bytes(b"new synthetic executable") self.destination.write_bytes(b"existing synthetic executable") (self.work / "sa" / "image").mkdir() @@ -67,7 +70,7 @@ class FinishOutputTests(unittest.TestCase): def finish(self, launcher=False): if launcher: (self.root / "HookshotBuilder-Setup.exe").write_bytes(b"new synthetic launcher") - with mock.patch.object(make_game, "BESIDE", ["hookshot.exe"]): + with mock.patch.object(make_game, "BESIDE", ["Triforce.exe"]): # Comprehensions use globals. Keep one namespace, as in production, and discover only authored setup files. namespace = dict(make_game.__dict__, built=str(self.built), out=str(self.out), work=str(self.work), ROOT=str(self.root / "tools"), LINUX=False if launcher else make_game.LINUX) @@ -78,7 +81,10 @@ class FinishOutputTests(unittest.TestCase): self.assertEqual((self.out / "image.bin").read_bytes(), b"existing synthetic image") def assert_no_staging_files(self): - self.assertEqual(sorted(path.name for path in self.out.iterdir()), ["hookshot.exe", "image.bin"]) + # (the note of what was installed is the finish's own, not a staging file: the performance audit's IO-45, + # tests/test_finish_installed.py) + self.assertEqual(sorted(path.name for path in self.out.iterdir() if path.name != make_game.INSTALLED_NOTE), + ["Triforce.exe", "image.bin"]) def stopped_output(self): output = io.StringIO() @@ -124,7 +130,7 @@ class FinishOutputTests(unittest.TestCase): self.assert_no_staging_files() def test_launcher_stages_all_three_files_before_replacement(self): - binary = self.out / "bin" / "hookshot.exe" + binary = self.out / "bin" / "Triforce.exe" binary.parent.mkdir() binary.write_bytes(b"existing synthetic game in bin") copy = make_game.shutil.copy2 @@ -148,7 +154,7 @@ class FinishOutputTests(unittest.TestCase): self.assertEqual(list(self.out.rglob("*.new")), []) def test_partial_launcher_copy_preserves_game_image_and_launcher(self): - binary = self.out / "bin" / "hookshot.exe" + binary = self.out / "bin" / "Triforce.exe" binary.parent.mkdir() binary.write_bytes(b"existing synthetic game in bin") copy = make_game.shutil.copy2 @@ -164,9 +170,9 @@ class FinishOutputTests(unittest.TestCase): self.finish(launcher=True) self.assert_installed_unchanged() self.assertEqual(binary.read_bytes(), b"existing synthetic game in bin") - self.assertEqual(binary.with_name("hookshot.exe.new").read_bytes(), self.source.read_bytes()) + self.assertEqual(binary.with_name("Triforce.exe.new").read_bytes(), self.source.read_bytes()) self.assertEqual((self.out / "image.bin.new").read_bytes(), self.image.read_bytes()) - self.assertEqual((self.out / "hookshot.exe.new").read_bytes(), b"partial launcher") + self.assertEqual((self.out / "Triforce.exe.new").read_bytes(), b"partial launcher") def test_partial_executable_copy_preserves_both_installed_files(self): def partial_copy(source, destination): @@ -177,7 +183,7 @@ class FinishOutputTests(unittest.TestCase): with self.assertRaises(OSError): self.finish() self.assert_installed_unchanged() - self.assertEqual((self.out / "hookshot.exe.new").read_bytes(), b"incomplete replacement") + self.assertEqual((self.out / "Triforce.exe.new").read_bytes(), b"incomplete replacement") self.assertFalse((self.out / "image.bin.new").exists()) def test_partial_image_copy_preserves_both_installed_files(self): @@ -194,7 +200,7 @@ class FinishOutputTests(unittest.TestCase): with self.assertRaises(OSError): self.finish() self.assert_installed_unchanged() - self.assertEqual((self.out / "hookshot.exe.new").read_bytes(), self.source.read_bytes()) + self.assertEqual((self.out / "Triforce.exe.new").read_bytes(), self.source.read_bytes()) self.assertEqual((self.out / "image.bin.new").read_bytes(), b"incomplete image") def test_readonly_source_mode_is_preserved_on_success(self): @@ -212,8 +218,8 @@ class FinishOutputTests(unittest.TestCase): self.assertIn("writable", message) self.assertNotIn("the game is running", message) self.assert_installed_unchanged() - self.assertEqual((self.out / "hookshot.exe.new").read_bytes(), self.source.read_bytes()) - self.assertFalse((self.out / "hookshot.exe.new").stat().st_mode & stat.S_IWRITE) + self.assertEqual((self.out / "Triforce.exe.new").read_bytes(), self.source.read_bytes()) + self.assertFalse((self.out / "Triforce.exe.new").stat().st_mode & stat.S_IWRITE) self.assertEqual((self.out / "image.bin.new").read_bytes(), self.image.read_bytes()) def test_stage_copy_permission_failure_has_actionable_paths(self): @@ -224,13 +230,13 @@ class FinishOutputTests(unittest.TestCase): with mock.patch.object(make_game.shutil, "copy2", side_effect=denied_copy): message = self.stopped_output() self.assertIn("stage", message) - self.assertIn(str(self.out / "hookshot.exe.new"), message) + self.assertIn(str(self.out / "Triforce.exe.new"), message) self.assertIn(str(self.source), message) self.assertIn("readable", message) self.assertIn("writable", message) self.assertIn("No installed files were replaced", message) self.assert_installed_unchanged() - self.assertEqual((self.out / "hookshot.exe.new").read_bytes(), b"partial staged copy") + self.assertEqual((self.out / "Triforce.exe.new").read_bytes(), b"partial staged copy") @unittest.skipUnless(os.name == "nt", "Windows file-sharing contract") def test_locked_destination_reports_action_and_retains_staged_build(self): @@ -250,7 +256,7 @@ class FinishOutputTests(unittest.TestCase): self.assertIn(str(self.destination), message) self.assertIn(str(self.source), message) self.assert_installed_unchanged() - self.assertEqual((self.out / "hookshot.exe.new").read_bytes(), self.source.read_bytes()) + self.assertEqual((self.out / "Triforce.exe.new").read_bytes(), self.source.read_bytes()) self.assertEqual((self.out / "image.bin.new").read_bytes(), self.image.read_bytes()) finally: self.assertTrue(kernel.CloseHandle(handle)) @@ -270,7 +276,7 @@ class FinishOutputTests(unittest.TestCase): @unittest.skipUnless(os.name == "nt", "Windows read-only staging-file contract") def test_readonly_existing_stage_is_not_removed_or_unlocked(self): - staged = self.out / "hookshot.exe.new" + staged = self.out / "Triforce.exe.new" staged.write_bytes(b"pre-existing staged content") staged.chmod(stat.S_IREAD) with mock.patch.object(make_game.os, "chmod", wraps=make_game.os.chmod) as chmod: diff --git a/tests/test_first_start.py b/tests/test_first_start.py new file mode 100644 index 0000000..12dfcb6 --- /dev/null +++ b/tests/test_first_start.py @@ -0,0 +1,165 @@ +"""A first start sized to the card: with RX_VK_FIRST_START_AUTO, the Render resolution row set once on a weak GPU +(gpu/vk/first_start.h, vk_menu.cpp menu_first_start, vk_main.cpp mkwvk_init; no game, no GPU). + +(Triforce, 11-22, the performance audit's PRESENT-46, after Source A's.) With no options file every card started with +the Render resolution at Auto. Source A's tier 1 (an integrated GPU, a card without mesh shaders, 4.5 GB of video memory +or less) starts with the 3D at 50%, or 33% for an integrated GPU beside 4 threads or fewer. Here that is behind a +switch that is off unless set: what a first start on a weak card gets is the owner's to decide. + +Compiles first_start.h and checks the rule for many cards: + integrated GPUs (a handheld's, a laptop's, a desktop's with 2 to 32 threads) 50%, and 33% at 4 threads or fewer + discrete cards without mesh shaders (2 to 12 GB) 50% + discrete cards with them: 2, 3, 4 and 4.5 GB 50% + 4.5 GB and a megabyte, 6, 8, 11, 12, 16, 24, 32 GB no change + a card that reports no memory, or no thread count never tier 1 for that alone + and that nothing in the rule reads the SteamDeck variable (a desktop card started from that launcher stays as it is); +the switch (off when unset or 0, on at 1, a larger number plays that many MB); and that the choice is made only with +the switch on, no options file, RX_VK_LOW_SPEC not asked and HOS_DYNRES not set. + +Then the source: menu_init only notes that there was no file; menu_first_start reads the switch and the card, sets the +row through the upscaler's own steps, moves the restart row's start value with it, saves nothing and logs one line; +mkwvk_init calls it after the device is made and its line logged; and the switch's row is in the development guide. + + python -I -B tests/test_first_start.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +PROGRAM = r""" +#include +#include +#include +#include +#include "first_start.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +using namespace first_start; +static Choice card(bool integrated, bool mesh, uint64_t mb, unsigned threads) { + Card c; + c.integrated = integrated, c.mesh_shaders = mesh, c.vram_mb = mb, c.threads = threads; + return choose(c); +} +int main() { + CHECK(kRowHalf == 4 && kRowThird == 6 && kSmallVramMb == 4608 && kFewThreads == 4); + // integrated GPUs: tier 1 whatever they report; 33% beside 4 threads or fewer + for (uint64_t mb : {0ull, 512ull, 2048ull, 3840ull, 8192ull, 16384ull, 32768ull}) + for (int mesh = 0; mesh < 2; mesh++) { + for (unsigned t : {1u, 2u, 4u}) CHECK(card(true, mesh, mb, t).row == kRowThird && card(true, mesh, mb, t).why == kIntegrated); + for (unsigned t : {0u, 5u, 6u, 8u, 12u, 16u, 32u}) + CHECK(card(true, mesh, mb, t).row == kRowHalf && card(true, mesh, mb, t).why == kIntegrated); + } + CHECK(card(true, true, 3840, 8).row == kRowHalf); // (a handheld: 4 cores, 8 threads, its own GPU) + CHECK(card(true, true, 8192, 4).row == kRowThird); // (Source A's note: a 2-core, 4-thread laptop's own GPU) + // discrete cards without mesh shaders: 50%, whatever their memory and threads + for (uint64_t mb : {0ull, 2048ull, 4096ull, 8192ull, 11264ull, 12288ull}) + for (unsigned t : {0u, 2u, 4u, 8u, 16u}) CHECK(card(false, false, mb, t).row == kRowHalf && card(false, false, mb, t).why == kNoMeshShaders); + // discrete cards with mesh shaders: by their video memory alone, 4.5 GB the last of tier 1 + for (unsigned t : {0u, 2u, 4u, 6u, 8u, 16u, 32u}) { + for (uint64_t mb : {1024ull, 2048ull, 3072ull, 4018ull, 4096ull, 4607ull, 4608ull}) + CHECK(card(false, true, mb, t).row == kRowHalf && card(false, true, mb, t).why == kSmallVram); + for (uint64_t mb : {4609ull, 5892ull, 6144ull, 8188ull, 8192ull, 10240ull, 11264ull, 12288ull, 16376ull, 24564ull, 32607ull}) + CHECK(card(false, true, mb, t).row == 0 && card(false, true, mb, t).why == kKeep); + CHECK(card(false, true, 0, t).row == 0); // (no memory reported: not tier 1 for that) + } + // the switch + CHECK(switch_mb(nullptr) == -1 && switch_mb("") == -1 && switch_mb("0") == -1 && switch_mb("-5") == -1 && switch_mb("on") == -1); + CHECK(switch_mb("1") == 0 && switch_mb("2") == 2 && switch_mb("4096") == 4096 && switch_mb("24564") == 24564); + // when the choice is made at all: (has_options, low_spec, resolution_env, switch) + CHECK(applies(false, false, false, 0) && applies(false, false, false, 4096)); + CHECK(!applies(false, false, false, -1)); // (the switch off: the default) + CHECK(!applies(true, false, false, 0) && !applies(true, false, false, 4096)); // (an options file: the player's own) + CHECK(!applies(false, true, false, 0)); // (RX_VK_LOW_SPEC sets the row itself) + CHECK(!applies(false, false, true, 0)); // (HOS_DYNRES sets it for the run) + for (int m = 0; m < 8; m++) + for (int64_t sw : {(int64_t)-1, (int64_t)0, (int64_t)4096}) + CHECK(applies(m & 1, m & 2, m & 4, sw) == (m == 0 && sw >= 0)); + // the lab's number plays a small card on a large one + CHECK(card(false, true, (uint64_t)switch_mb("4096"), 32).row == kRowHalf && card(false, true, (uint64_t)switch_mb("6144"), 32).row == 0); + CHECK(std::strlen(why_text(kIntegrated)) && std::strlen(why_text(kNoMeshShaders)) && std::strlen(why_text(kSmallVram)) && + std::strlen(why_text(kKeep))); + std::puts("first start: tier 1 for integrated GPUs, cards without mesh shaders and 4.5 GB or less; off unless asked"); + return fails ? 1 : 0; +} +""" + + +def check_source(): + header = (VK / "first_start.h").read_text(encoding="utf-8") + code = "\n".join(l for l in header.splitlines() if not l.lstrip().startswith("//")) + check("getenv" not in code and "SteamDeck" not in code and "#include 0 ? (uint64_t)asked : device_vram_bytes() >> 20;", + "card.threads = cpu_count();", # ((Triforce, 11-22, OS-49) cpu_count.h + "opts.resolution = ur::snap(upscaler(), choice.row);", + "restart_state.restart_value(restart_slot_of(r), restart_value_of(r));", + 'logf("menu: first start: tier 1 ('): + check(part in first, "menu_first_start: missing " + part) + check(first.index("first_start::applies(") < first.index("first_start::choose(card)") < first.index("opts.resolution ="), + "menu_first_start sets the row before it has asked whether it applies") + check(first.count("opts.") == first.count("opts.resolution") and "SteamDeck" not in first and "fopen" not in first + and "save" not in first.replace("nothing is saved", ""), + "menu_first_start sets another row than the Render resolution, reads SteamDeck, or saves the options") + check("void menu_first_start();" in (VK / "vk_menu.h").read_text(encoding="utf-8"), "vk_menu.h does not declare menu_first_start") + main = (VK / "vk_main.cpp").read_text(encoding="utf-8") + start = main[main.index("const MkwVkRenderer* mkwvk_init(const MkwVkHost* h) {"):] + order = [start.index(p) for p in ("if (!create_instance() || !pick_device() || !create_device()) return nullptr;", + 'logf("vk: %s, Vulkan %u.%u, driver %#x; mesh shaders %s', " menu_first_start();\n", + "table.start = start;")] + check(order == sorted(order) and main.count("menu_first_start()") == 1, + "mkwvk_init does not call menu_first_start once, after the device is made and before the renderer's table is given out") + docs = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("| `RX_VK_FIRST_START_AUTO=1`, `RX_VK_FIRST_START_AUTO=` |" in docs, "docs/development.md has no row for RX_VK_FIRST_START_AUTO") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + check_source() + with tempfile.TemporaryDirectory(prefix="first-start-") as tmp: + src, exe = Path(tmp) / "first.cpp", Path(tmp) / "first.exe" + src.write_text(PROGRAM, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I", str(VK), str(src), "-o", str(exe)], check=True) + out = subprocess.run([str(exe)], stdout=subprocess.PIPE, text=True, timeout=60) + print(out.stdout.strip()) + if out.returncode: + raise SystemExit("FAIL: the rule") + print("PASS: a first start on a weak card takes its Render resolution only with RX_VK_FIRST_START_AUTO, no options file " + "and no RX_VK_LOW_SPEC; the hook is after the device") + + +if __name__ == "__main__": + main() diff --git a/tests/test_flush_nothing_held.py b/tests/test_flush_nothing_held.py new file mode 100644 index 0000000..1a41714 --- /dev/null +++ b/tests/test_flush_nothing_held.py @@ -0,0 +1,223 @@ +"""Flushes with nothing held skip the queue's lock (gpu/vk/vk_replay.cpp pend_held, RX_VK_PEND_HELD). + +queue_flush_for and queue_flush_poll took dev.present_mu to find nothing held, at every fence wait and progress poll (with +batching off, always), behind a present or a late page's fix waiting on the GPU. pend_held is raised under the lock when a +submit is held and lowered when the batch goes out; both flushes look at it first. A held submit's recording is registered +for the waiters only after it is in pend with the flag raised, so a waiter that finds the registration finds it held (or +already sent) and flushes it. + +Checked, compiled (pend_held, pend_held_on, queue_flush_for and queue_flush_poll cut out of vk_replay.cpp unchanged, with a +counting mutex for dev.present_mu): nothing held, no lock; held, the lock and the flush; RX_VK_PEND_HELD=0 and the lab knob +nopendheld, the lock every time; four waiters against a submitter that holds, raises and then registers: every waiter's +flush after it found a registration leaves that recording sent. In the source: the lowering after pend.cbs.clear(); in +submit_recording the registration before the lock only when not registered late, after the push and the raise when held, +before the direct submit otherwise; wait_recorded's counter read before its flush; vk_timeline_wait's second flush once its +lookup finds the recording; the docs row. + + python -I -B tests/test_flush_nothing_held.py [--cxx clang++] +""" +import argparse +import os +import subprocess +import tempfile +from pathlib import Path + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def fail(message): + raise SystemExit("FAIL: " + message) + + +def in_order(text, parts, where): + at = 0 + for part in parts: + found = text.find(part, at) + if found < 0: + fail("%s: %r is missing, or not after the part before it" % (where, part)) + at = found + len(part) + + +FIXTURE = r""" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace lab_ab { +bool knob = false; +bool on(const char* name) { return knob && !strcmp(name, "nopendheld"); } +} +struct CountingMutex { + std::mutex m; + std::atomic locks{0}; + void lock() { locks.fetch_add(1); m.lock(); } + void unlock() { m.unlock(); } +}; +struct { CountingMutex present_mu; } dev; +struct { std::vector cbs; } pend; +int flush_reason; +enum { FR_OTHER, FR_POLL, FR_WAIT, FR_VALUE }; +std::chrono::steady_clock::time_point pend_since; + +DECL + +uint64_t held_seq; // (the newest recording in pend; dev.present_mu) +std::atomic sent_seq{0}; // (the newest one sent) +long flushes; +void queue_flush_locked() { // (the real one's end: the batch out, then the flag lowered) + if (pend.cbs.empty()) return; + flushes++; + sent_seq.store(held_seq, std::memory_order_relaxed); + pend.cbs.clear(); + pend_held.store(false, std::memory_order_release); +} +void hold(uint64_t seq) { // (submit_recording's held path) + std::lock_guard lk(dev.present_mu); + pend.cbs.push_back(1); + held_seq = seq; + pend_held.store(true, std::memory_order_release); +} + +FLUSH + +int main(int argc, char** argv) { + const char* mode = argc > 1 ? argv[1] : ""; + int bad = 0; + auto check = [&](bool ok, const char* what) { if (!ok) { std::printf("FAIL %s\n", what); bad++; } }; + if (!strcmp(mode, "basic") || !strcmp(mode, "off") || !strcmp(mode, "knob")) { + lab_ab::knob = !strcmp(mode, "knob"); + const bool skip = !strcmp(mode, "basic"); + const long l0 = dev.present_mu.locks; + for (int i = 0; i < 1000; i++) queue_flush_for(FR_WAIT), queue_flush_poll(); + check(skip ? dev.present_mu.locks == l0 : dev.present_mu.locks == l0 + 2000, "nothing held: the lock as the mode says"); + check(flushes == 0, "nothing held: no flush"); + hold(7); + const long l1 = dev.present_mu.locks; + queue_flush_for(FR_WAIT); + check(dev.present_mu.locks == l1 + 1 && flushes == 1 && sent_seq == 7 && !pend_held.load(), "held: the lock and the flush"); + hold(8); + queue_flush_poll(); + check(flushes == 2 && sent_seq == 8 && pend.cbs.empty(), "held: the poll flushes (RX_VK_BATCH_POLL_US=0)"); + const long l2 = dev.present_mu.locks; + queue_flush_for(FR_WAIT), queue_flush_poll(); + check(skip ? dev.present_mu.locks == l2 : dev.present_mu.locks == l2 + 2, "after the flush: nothing held again"); + std::printf("%s: %ld locks, %ld flushes\n", mode, (long)dev.present_mu.locks, flushes); + return bad; + } + if (!strcmp(mode, "race")) { + std::mutex map_mu; // (rec_map_mu) + uint64_t registered = 0; + std::atomic stop{false}; + std::atomic late{0}, looks{0}; + std::thread submitter([&] { + for (uint64_t s = 1; s <= 200000; s++) { + { + std::lock_guard lk(dev.present_mu); + pend.cbs.push_back(1); + held_seq = s; + pend_held.store(true, std::memory_order_release); + std::lock_guard mk(map_mu); // (registered after the raise, under the lock) + registered = s; + } + if (s % 64 == 0) { std::lock_guard lk(dev.present_mu); queue_flush_locked(); } // (the cap) + } + stop = true; + }); + std::vector waiters; + for (int w = 0; w < 4; w++) + waiters.emplace_back([&] { + while (!stop) { + uint64_t s; + { + std::lock_guard mk(map_mu); + s = registered; + } + if (!s) continue; + queue_flush_for(FR_WAIT); + looks++; + if (sent_seq.load(std::memory_order_relaxed) < s) late++; + } + }); + submitter.join(); + for (auto& t : waiters) t.join(); + std::printf("race: %ld looks, %ld left a found recording held\n", (long)looks, (long)late); + return late != 0; + } + return 1; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + src = (ROOT / "gpu" / "vk" / "vk_replay.cpp").read_text(encoding="utf-8") + if "std::atomic pend_held{false};" not in src: + fail("vk_replay.cpp has no pend_held") + decl = src[src.index("std::atomic pend_held{false};"):] + decl = decl[:decl.index(block(decl, "bool pend_held_on() {")) + len(block(decl, "bool pend_held_on() {"))] + flush = block(src, "void queue_flush_for(int why) {") + "\n" + block(src, "void queue_flush_poll() {") + for name in ("queue_flush_for", "queue_flush_poll"): + body = block(src, "void %s(" % name) + if body.index("pend_held.load(std::memory_order_acquire)") > body.index("std::lock_guard lk(dev.present_mu);"): + fail("%s: the look is not before the lock" % name) + locked = block(src, "void queue_flush_locked() {") + in_order(locked, ["pend.cbs.clear()", "pend_held.store(false, std::memory_order_release);"], "queue_flush_locked") + sub = block(src, "void submit_recording(Recording& r, bool signal_value = true, uint64_t signal_at = 0, bool part = false) {") + in_order(sub, ["seq = ++rec_seq;", "const auto register_recording = [&] {", "std::lock_guard mk(rec_map_mu);", + "fin_rec.submitted(rec_open_values, seq, !part);", "};", + "const bool batching = batch_on(), registered_late = batching && pend_held_on();", + "if (!registered_late) register_recording();", "std::lock_guard lk(dev.present_mu);", "if (batch_on()) {", + "for (uint32_t i = 0; i < n; i++) pend.cbs.push_back(cb[i]);", "pend.recs++;", + "pend_held.store(true, std::memory_order_release);", "if (registered_late) register_recording();", + "queue_flush_locked();", "return;", "if (registered_late) register_recording();", + "vkQueueSubmit2(dev.queue, 1, &si, r.fence);"], "submit_recording") + if sub.count("register_recording();") != 3: + fail("submit_recording: the registration is made somewhere this test does not know") + wr = block(src, "bool wait_recorded(uint64_t f, uint64_t timeout_ns, int* ok) {") + in_order(wr, ["seq = it->second;", "pend_held_on() && vkGetSemaphoreCounterValue(dev.device, rec_timeline, &rec_done) == VK_SUCCESS && rec_done >= seq", + "*ok = 1;", "return true;", "queue_flush_for(FR_WAIT);"], "wait_recorded") + tw = block(src, "void vk_timeline_wait(uint64_t v) {") + in_order(tw, ["queue_flush_for(FR_VALUE);", "seq = fin_rec.of(v);", "if (seq && pend_held_on()) queue_flush_for(FR_VALUE);"], + "vk_timeline_wait") + docs = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + if "`RX_VK_PEND_HELD=0`" not in docs: + fail("docs/development.md: no RX_VK_PEND_HELD row") + print("ok the source: the look before the lock, the lowering after the clear, the registration after the raise, the " + "counter read, the second flush") + with tempfile.TemporaryDirectory(prefix="flush-nothing-held-") as tmp: + c, exe = Path(tmp) / "fixture.cpp", Path(tmp) / "fixture.exe" + c.write_text(FIXTURE.replace("DECL", decl).replace("FLUSH", flush), encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-pthread", "-Wall", "-Wextra", "-Werror", str(c), "-o", str(exe)], check=True) + for mode, env in (("basic", {}), ("off", {"RX_VK_PEND_HELD": "0"}), ("knob", {}), ("race", {})): + e = dict(os.environ, RX_VK_BATCH_POLL_US="0") + e.pop("RX_VK_PEND_HELD", None) + e.update(env) + r = subprocess.run([str(exe), mode], capture_output=True, text=True, env=e, timeout=300) + print(("ok " if r.returncode == 0 else "FAIL ") + r.stdout.strip().replace("\n", "; ")) + if r.returncode: + fail("the fixture's %s mode" % mode) + print("PASS: flushes with nothing held skip the queue's lock") + + +if __name__ == "__main__": + main() diff --git a/tests/test_fma_soft.py b/tests/test_fma_soft.py new file mode 100644 index 0000000..d80674a --- /dev/null +++ b/tests/test_fma_soft.py @@ -0,0 +1,220 @@ +"""The exact fused multiply-add bodies' short way out on a CPU without FMA3, bit-exact with the bodies (game-free). + +(Triforce, 11-22, the performance audit's LIFT-06, after Source A's.) Where the build has no hardware FMA (x86-64-v2) +every double-precision FMADD, and each single-precision one its binary64 fast path cannot prove, runs the kit's exact +body: integer code of many words. nx2/fma_soft.py puts a short way out at the head of recomp_fpe_fma64 and +recomp_fpe_fma32 for the plain case (default FPCR with or without FZ, three normal operands, a normal result): an exact +product and sum in binary64, rounded to odd, then rounded once, FPSR.IXC set when the result is inexact. Everything +else goes on into the body. + +The text: the kit's runtime C from a64_rtgen.runtime_c() under fma_soft.install() must be the kit's own text with +three pieces put in (the header after `#include "a64.h"`, one group of lines after each body's head) and nothing else +changed; with A64_NO_FMA_SOFT=1 it must be the kit's text byte for byte. + +The run: tests/fma_soft_test.c calls both bodies with the way out and without it (the same body past its first lines, +so the kit's code) from the same Ctx, and compares the value and every byte of the Ctx: + - random: 10 million operand triples a width (zeros, subnormals, infinities, NaNs of both kinds with payloads, the + extreme normals, exponents everywhere, at the way out's limits and matched to the product; addends that cancel the + product within a few ulps, sit at its half ulp, or have the product at their half ulp), under random FPCR (FZ, DN, + every rounding mode, FIZ/AH/NEP, the trap enables, FZ16, the host's kill switch) and FPSR states; + - sweep: about 7 million: edge exponents and fractions for both factors, each with 71 addends placed against the + product, under FZ on and off with IXC set and clear. +Built at -march=x86-64-v2 and -march=x86-64 (no FMA: the way out is compiled, and must answer calls, set IXC on some, +leave it clear on exact ones, and answer under FZ), and at -march=x86-64-v3 and -march=native (with FMA3 it is compiled +out: no call may be answered). A build without the test's hook checks HOS_FMA_FAST as the game reads it: unset and 1 +answer, 0 answers none. + +And reads the source: nx2/lift.py installs it, A64_NO_FMA_SOFT is an A64_ name (in nx2/make_game.py's lift key, as +nx2/fma_soft.py is), and both switches are in the switches table. + + python -I -B tests/test_fma_soft.py [--cc clang] [--cxx clang++] [--random 10000000] [--time] + +Skipped (exit 0) on a host that is not x86-64 or has no C or C++ compiler. +""" +import argparse +import os +import platform +import re +import subprocess +import sys +import tempfile +import time + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(ROOT, "tests")) +sys.path.insert(0, os.path.join(ROOT, "nx2")) +import test_fpx_sticky as fs # noqa: E402 (the build helpers the kit's other lifted-code tests use) + +RUN_ALL_ARGS = ["--cc", "clang", "--cxx", "{cxx}"] +SEED = 20261122 +MARCHES = ("x86-64-v2", "x86-64", "x86-64-v3", "native") +HOOK = """/* tests/test_fma_soft.py: the way out switched by the driver, and its answers counted */ +extern int fma_soft_test_on; +extern long long fma_soft_test_exits; +#define RECOMP_FMA_SOFT_PROBE() (fma_soft_test_exits++) +#ifndef FMA_SOFT_ENV +#define RECOMP_FMA_SOFT_ON() fma_soft_test_on +#endif +""" + + +def generate(cxx, tmp, flags): + """The kit's runtime header and C (the C with the way out), a64_fpx.h and the hook header in tmp. Returns the C file""" + dll = os.path.join(tmp, "a64emit.dll" if os.name == "nt" else "liba64emit.so") + fs.build([cxx, "-std=c++20", "-O0", "-shared", "-Wno-everything", "-o", dll, os.path.join(fs.KIT, "lift", "a64emit.cpp")] + + flags + ([] if os.name == "nt" else ["-fPIC"]), "a64emit") + import kit # noqa: F401 (a64-kit on the path) + import build_emit + build_emit.build = lambda force=False: dll # (not a64emit.dll beside the source) + import a64_lift + import a64_rtgen + import a64emit + import fma_soft + try: + kit_text = a64_rtgen.runtime_c() + fma_soft.install() # (the wrapper under test, around the kit's runtime_c) + now = a64_rtgen.runtime_c() + os.environ["A64_NO_FMA_SOFT"] = "1" + try: + off = a64_rtgen.runtime_c() + finally: + del os.environ["A64_NO_FMA_SOFT"] + before = set(os.listdir(tmp)) + a64_rtgen.write(tmp) + finally: + if a64emit._lib is not None: # (unloaded, so the folder can go) + import _ctypes + (_ctypes.FreeLibrary if os.name == "nt" else _ctypes.dlclose)(a64emit._lib._handle) + a64emit._lib = None + if off != kit_text: + raise SystemExit("fma soft: with A64_NO_FMA_SOFT=1 the runtime C is not the kit's text") + pieces = [fma_soft.HEADER] + [fma_soft.EARLY % n for n in (32, 64)] + stripped = now + for piece in pieces: + if stripped.count(piece) != 1: + raise SystemExit("fma soft: the runtime C does not hold this piece once: %s" % piece.strip()[:80]) + stripped = stripped.replace(piece, "") + if stripped != kit_text: + raise SystemExit("fma soft: the runtime C differs from the kit's by more than the header and the two ways out") + for n in (32, 64): # (the way out is the body's first lines: right after its head) + if (fma_soft.HEAD % n) + (fma_soft.EARLY % n) not in now: + raise SystemExit("fma soft: recomp_fpe_fma%d does not begin with the way out" % n) + rt_c = [f for f in sorted(set(os.listdir(tmp)) - before) if f.endswith(".c")] + if len(rt_c) != 1: + raise SystemExit("fma soft: a64_rtgen wrote %s, not one C file" % rt_c) + with open(os.path.join(tmp, rt_c[0]), encoding="utf-8") as f: + if f.read() != now: + raise SystemExit("fma soft: a64_rtgen.write did not write the runtime C with the way out") + with open(os.path.join(tmp, "a64_fpx.h"), "w", newline="\n") as f: + f.write(a64_lift.fpx_header(set())) + with open(os.path.join(tmp, "fma_soft_hook.h"), "w", newline="\n") as f: + f.write(HOOK) + return os.path.join(tmp, rt_c[0]) + + +def check_sources(): + with open(os.path.join(ROOT, "nx2", "lift.py"), encoding="utf-8") as f: + lift = f.read() + at = [lift.find(p) for p in ("import fma_soft", "import a64_rtgen", "fma_soft.install()", "def main():")] + if min(at) < 0 or not at[1] < at[2] < at[3]: + raise SystemExit("fma soft: nx2/lift.py does not install nx2/fma_soft.py at module level") + with open(os.path.join(ROOT, "nx2", "make_game.py"), encoding="utf-8") as f: + make_game = f.read() + if 'if k.startswith(("A64_", "OOT_", "RX_"))' not in make_game: + raise SystemExit("fma soft: nx2/make_game.py's lift_key no longer takes the A64_ variables: A64_NO_FMA_SOFT is not in the key") + not_keyed = re.search(r"^LIFT_KEY_NOT = \((.*?)\) #", make_game, re.S | re.M) + if not not_keyed or '"fma_soft.py"' in not_keyed.group(1): + raise SystemExit("fma soft: nx2/fma_soft.py must stay a file of the lift key") + with open(os.path.join(ROOT, "docs", "development.md"), encoding="utf-8") as f: + if "| `HOS_FMA_FAST=0`, `A64_NO_FMA_SOFT=1` |" not in f.read(): + raise SystemExit("fma soft: docs/development.md: HOS_FMA_FAST and A64_NO_FMA_SOFT are not in the switches table") + + +def counts(text, mode): + """{(mode, width): {name: number}} from the driver's lines""" + out = {} + for line in text.splitlines(): + m = re.match(r"%s fma(\d+) (.*)" % mode, line) + if m: + out[int(m.group(1))] = {k: int(v) for k, v in (kv.split("=") for kv in m.group(2).split())} + return out + + +def main(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--cc") + p.add_argument("--cxx") + p.add_argument("--random", type=int, default=10000000) + p.add_argument("--time", action="store_true", help="also time a plain call with the way out and without, at x86-64-v2") + a = p.parse_args() + check_sources() + if platform.machine().lower() not in ("x86_64", "amd64"): + fs.skip("not an x86-64 host: %s" % platform.machine()) + cc = fs.compiler(a.cc, "CC", ("clang", "gcc", "cc"), "clang") + cxx = fs.compiler(a.cxx, "CXX", ("clang++", "g++", "c++"), "clang++") + if not cc or not cxx: + fs.skip("no C compiler" if not cc else "no C++ compiler") + t0 = time.time() + failed, total = [], 0 + with tempfile.TemporaryDirectory(prefix="triforce-fma-soft-") as tmp: + rt_c = generate(cxx, tmp, fs.target_flags(cxx, tmp)) + + def build(march, extra, name): + exe = os.path.join(tmp, "%s_%s.exe" % (name, march)) + fs.build([cc] + fs.target_flags(cc, tmp) + ["-march=" + march] + fs.FLAGS + extra + + ["-D_CRT_SECURE_NO_WARNINGS", "-include", os.path.join(tmp, "fma_soft_hook.h"), + "-I" + os.path.join(fs.KIT, "rt"), "-I" + tmp, os.path.join(ROOT, "tests", "fma_soft_test.c"), rt_c, + "-o", exe], "the driver at " + march) + return exe + + for march in MARCHES: + exe = build(march, [], "fma_soft_test") + for mode, args in (("random", ["random", str(a.random), str(SEED)]), ("sweep", ["sweep"])): + r = fs.run([exe] + args, timeout=3600) + print("%-10s %s" % (march, r.stdout.rstrip().replace("\n", "\n "))) + if r.returncode == 3: + print("fma soft: %s skipped (this CPU cannot run it)" % march) + break + got = counts(r.stdout, mode) + if r.returncode or sorted(got) != [32, 64]: + failed.append("%s %s (exit %d)" % (march, mode, r.returncode)) + continue + for width, c in sorted(got.items()): + total += c["checks"] + if c["fma64"]: # (FMA3: compiled out) + if c["exits"]: + failed.append("%s %s fma%d: the way out answered %d calls in a build with FMA3" % (march, mode, width, c["exits"])) + elif not (c["exits"] and c["new_ixc"] and c["exact"] and c["fz"]): + failed.append("%s %s fma%d: the way out is not exercised: %s" % (march, mode, width, c)) + if not c["fma64"] and march in ("x86-64-v3",): + failed.append("%s: built without FMA3" % march) + if c["fma64"] and march in ("x86-64-v2", "x86-64"): + failed.append("%s: built with FMA3" % march) + # HOS_FMA_FAST as the game reads it (the build without the hook's switch): unset and 1 answer, 0 answers none + exe = build("x86-64-v2", ["-DFMA_SOFT_ENV"], "fma_soft_env") + sums = set() + for value, want in ((None, True), ("1", True), ("0", False)): + env = {k: v for k, v in os.environ.items() if k != "HOS_FMA_FAST"} + if value is not None: + env["HOS_FMA_FAST"] = value + r = subprocess.run([exe, "env"], stdout=subprocess.PIPE, text=True, env=env, timeout=120) + m = re.match(r"fma64=0 exits=(\d+) sum=(\w+)", r.stdout) + if r.returncode == 3: + break + if not m or (int(m.group(1)) == 2000) != want or (int(m.group(1)) == 0) == want: + failed.append("HOS_FMA_FAST=%s: %s" % (value, r.stdout.strip())) + elif m: + sums.add(m.group(2)) + if len(sums) > 1: + failed.append("HOS_FMA_FAST changes a result: %s" % sorted(sums)) + if a.time and not failed: + print("timing at -O2 -march=x86-64-v2:") + print(fs.run([build("x86-64-v2", [], "fma_soft_test"), "time"], timeout=1200).stdout.rstrip()) + if failed: + raise SystemExit("FAIL: fma soft: %s" % "; ".join(failed)) + print("PASS: %d calls of the exact FMA bodies give the same value and context with the short way out as without (%.0f s)" + % (total, time.time() - t0)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_fma_steps.py b/tests/test_fma_steps.py new file mode 100644 index 0000000..7240fbe --- /dev/null +++ b/tests/test_fma_steps.py @@ -0,0 +1,246 @@ +"""The single-precision FRECPS and FRSQRTS steps as one fused instruction on x86-64-v3, bit-exact with the binary64 form +and with the exact bodies (game-free). + +(Triforce, 11-22, the performance audit's LIFT-05, after Source A's.) The kit's fast path for the Newton-Raphson steps +(recomp_fpx_recps32: 2 - a*b; recomp_fpx_rsqrts32: (3 - a*b) / 2) computes them in binary64 on x86-64 and declines a +binary32 midpoint, because the baseline has no FMA. nx2/fma_steps.py gives both the guard the kit's fused multiply-add +already has: where the build has FMA3, one fused instruction, the step rounded once. + +The text: the kit's runtime header from a64_rtgen.runtime_h() under fma_steps.install() must be the kit's own with those +two #if lines changed and nothing else, all three functions (fma32, recps32, rsqrts32) under the same guard; with +A64_NO_FMA_STEPS=1 it must be the kit's text byte for byte. + +The run: guest words (FRECPS and FRSQRTS: scalar S, 2S and 4S, with Vd apart, Vd = Vn, Vd = Vm, one register for all +and registers above 15; and the D forms, which do not change) go through a64emit.translate and a64_lift.op_text once, +and that one text is compiled twice, against the header as written now and against the kit's (A64_NO_FMA_STEPS=1), +tests/fma_steps_side.c. tests/fma_steps_test.c runs each word as lifted and as a hot function has it on both sides, +and the exact body inline (the word translated with the fast path off), from the same random Ctx: operands biased to +zeros, infinities, NaNs of both kinds with payloads, subnormals, the extreme normals, steps that cancel and steps that are +exactly a binary32 midpoint; random FPCR (FZ, DN, the rounding modes, the other control bits) and FPSR (the gate open and +closed), every flag combination. The whole Ctx, FPSR in it, must be the same in all five. Then the two fast paths by +themselves: what both sides keep must be the same bits, what either keeps must be the exact body's value with FPSR +untouched, and the kit's side must keep nothing the new one declines, with one exception that the driver checks by its +size: an FRSQRTS step of 2^127 or more, whose fused 3 - a*b is past binary32's range before the halving, so the new side +declines it and the exact body answers (the same value). + +Built at -march=x86-64-v3 and -march=native (with FMA3 the new side must keep midpoints the kit's declines, the audit's +two named pairs among them), at -march=x86-64-v2 and -march=x86-64 (no FMA3: both sides must decide alike on every pair), +and at -march=x86-64-v3 with -DA64_FPX_NO_FMA32 (the binary64 form again: alike too). + +And reads the source: nx2/lift.py installs it, A64_NO_FMA_STEPS is an A64_ name (in nx2/make_game.py's lift key, as +nx2/fma_steps.py is), and the switch is in the switches table. + + python -I -B tests/test_fma_steps.py [--cc clang] [--cxx clang++] [--random 20000] [--direct 4000000] + +Skipped (exit 0) on a host that is not x86-64 or has no C or C++ compiler. +""" +import argparse +import os +import platform +import re +import sys +import tempfile +import time + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(ROOT, "tests")) +sys.path.insert(0, os.path.join(ROOT, "nx2")) +import test_fpx_sticky as fs # noqa: E402 (the build helpers the kit's other lifted-code tests use) + +RUN_ALL_ARGS = ["--cc", "clang", "--cxx", "{cxx}"] +SEED = 20261122 +# (-march, more flags, whether the steps are the fused instruction there; None: as this CPU has it) +BUILDS = (("x86-64-v3", [], True), ("x86-64-v2", [], False), ("x86-64", [], False), ("native", [], None), + ("x86-64-v3", ["-DA64_FPX_NO_FMA32"], False)) +REGS = ((0, 5, 1), (5, 5, 1), (1, 5, 1), (2, 2, 2), (3, 30, 17)) # (rd, rn, rm) + + +def cases(): + """(name, guest word, rn, rm, double)""" + out = [] + for name, base, dbl in (("FRECPS s", 0x5E20FC00, 0), ("FRSQRTS s", 0x5EA0FC00, 0), ("FRECPS .2s", 0x0E20FC00, 0), + ("FRSQRTS .2s", 0x0EA0FC00, 0), ("FRECPS .4s", 0x4E20FC00, 0), ("FRSQRTS .4s", 0x4EA0FC00, 0), + ("FRECPS d", 0x5E60FC00, 1), ("FRSQRTS d", 0x5EE0FC00, 1), ("FRECPS .2d", 0x4E60FC00, 1), + ("FRSQRTS .2d", 0x4EE0FC00, 1)): + for rd, rn, rm in (REGS[:2] if dbl else REGS): + out.append(("%s v%d, v%d, v%d" % (name, rd, rn, rm), base | (rm << 16) | (rn << 5) | rd, rn, rm, dbl)) + return out + + +def lifted(a64emit, a64_lift, used): + """fma_steps_lifted.h: each guest word as lifted, as a hot function has it and with the exact body inline""" + all_cases = cases() + words = [(0x1000 + 4 * i, c[1]) for i, c in enumerate(all_cases)] + texts, flags = a64emit.translate(words) + if any(flags): + raise SystemExit("fma steps: a64emit did not translate the guest words: %s" % flags) + texts = [a64_lift.op_text(t, used) for t in texts] + a64emit.lib().a64_config(1, 0) # (the fast path off: the exact body inline, the reference) + try: + exact, _ = a64emit.translate(words) + finally: + a64emit.lib().a64_config(1, 1) + out = ["/* Generated by tests/test_fma_steps.py from guest words, as a64_lift.py lifts them. */"] + for i, (t, ex, c) in enumerate(zip(texts, exact, all_cases)): + helper = "a64_fpx_%s%d_" % ("rsqrts" if "FRSQRTS" in c[0] else "recps", 64 if c[4] else 32) + if helper not in t or "recomp_fpx_" in ex: + raise SystemExit("fma steps: %s is not lifted through %s, or its exact form has a fast path" % (c[0], helper)) + out.append("static void g_lifted%d(Ctx *__restrict c) { %s }" % (i, t)) + out.append("static void g_hot%d(Ctx *__restrict c) { %s }" % (i, a64_lift.hot_text(t))) + out.append("static void g_exact%d(Ctx *__restrict c) { %s }" % (i, ex)) + out.append("static const struct { const char *name; int rn, rm, dbl; void (*lifted)(Ctx *), (*hot)(Ctx *), (*exact)(Ctx *); }" + " CASES[] = {%s};" % ", ".join('{"%s", %d, %d, %d, g_lifted%d, g_hot%d, g_exact%d}' % (c[0], c[2], c[3], c[4], i, i, i) + for i, c in enumerate(all_cases))) + return "\n".join(out) + "\n", len(all_cases) + + +def generate(cxx, tmp, flags): + """tmp/now and tmp/old: the kit's runtime header and C as written now and with A64_NO_FMA_STEPS=1; tmp: a64_fpx.h and + fma_steps_lifted.h. Returns (the shapes, the runtime C file)""" + dll = os.path.join(tmp, "a64emit.dll" if os.name == "nt" else "liba64emit.so") + fs.build([cxx, "-std=c++20", "-O0", "-shared", "-Wno-everything", "-o", dll, os.path.join(fs.KIT, "lift", "a64emit.cpp")] + + flags + ([] if os.name == "nt" else ["-fPIC"]), "a64emit") + import kit # noqa: F401 (a64-kit on the path) + import build_emit + build_emit.build = lambda force=False: dll # (not a64emit.dll beside the source) + import a64_lift + import a64_rtgen + import a64emit + import fma_steps + now_dir, old_dir = os.path.join(tmp, "now"), os.path.join(tmp, "old") + used = set() + try: + kit_h = a64_rtgen.runtime_h() + fma_steps.install() # (the wrapper under test, around the kit's runtime_h) + now_h = a64_rtgen.runtime_h() + a64_rtgen.write(now_dir) + os.environ["A64_NO_FMA_STEPS"] = "1" + try: + old_h = a64_rtgen.runtime_h() + a64_rtgen.write(old_dir) + finally: + del os.environ["A64_NO_FMA_STEPS"] + text, n = lifted(a64emit, a64_lift, used) + finally: + if a64emit._lib is not None: # (unloaded, so the folder can go) + import _ctypes + (_ctypes.FreeLibrary if os.name == "nt" else _ctypes.dlclose)(a64emit._lib._handle) + a64emit._lib = None + if old_h != kit_h: + raise SystemExit("fma steps: with A64_NO_FMA_STEPS=1 the runtime header is not the kit's text") + want = kit_h + for old, new in ((fma_steps.RECPS32_OLD, fma_steps.RECPS32_NEW), (fma_steps.RSQRTS32_OLD, fma_steps.RSQRTS32_NEW)): + if want.count(old) != 1: + raise SystemExit("fma steps: the kit's header no longer has %r once" % old.split("\n")[0]) + want = want.replace(old, new) + if now_h != want or now_h == kit_h: + raise SystemExit("fma steps: the runtime header is not the kit's with the two steps' guards changed and nothing else") + for name in ("fma32", "recps32", "rsqrts32"): # (all three under one guard, so -DA64_FPX_NO_FMA32 returns all three) + if not re.search(r"static RECOMP_INLINE int recomp_fpx_%s\([^\n]*\{\n%s" % (name, re.escape(fma_steps.GUARD_NEW)), now_h): + raise SystemExit("fma steps: recomp_fpx_%s is not under the FMA3 guard" % name) + if now_h.count(fma_steps.GUARD_NEW) != 3 or kit_h.count(fma_steps.GUARD_NEW) != 1: + raise SystemExit("fma steps: the FMA3 guard is not on three functions now and one in the kit's text") + files = {} + for side, text_h in ((now_dir, now_h), (old_dir, old_h)): + names = sorted(os.listdir(side)) + heads, srcs = [f for f in names if f.endswith(".h")], [f for f in names if f.endswith(".c")] + if len(heads) != 1 or len(srcs) != 1: + raise SystemExit("fma steps: a64_rtgen wrote %s" % names) + with open(os.path.join(side, heads[0]), encoding="utf-8") as f: + if f.read() != text_h: + raise SystemExit("fma steps: a64_rtgen.write did not write the header under test into %s" % side) + with open(os.path.join(side, srcs[0]), encoding="utf-8") as f: + files[side] = (srcs[0], f.read()) + if files[now_dir] != files[old_dir]: + raise SystemExit("fma steps: the runtime C differs between the two sides (only the header should)") + with open(os.path.join(tmp, "fma_steps_lifted.h"), "w", newline="\n") as f: + f.write(text) + with open(os.path.join(tmp, "a64_fpx.h"), "w", newline="\n") as f: + f.write(a64_lift.fpx_header(used)) + return n, os.path.join(now_dir, files[now_dir][0]) + + +def check_sources(): + with open(os.path.join(ROOT, "nx2", "lift.py"), encoding="utf-8") as f: + lift = f.read() + at = [lift.find(p) for p in ("import fma_steps", "import a64_rtgen", "fma_steps.install()", "def main():")] + if min(at) < 0 or not at[1] < at[2] < at[3]: + raise SystemExit("fma steps: nx2/lift.py does not install nx2/fma_steps.py at module level") + with open(os.path.join(ROOT, "nx2", "make_game.py"), encoding="utf-8") as f: + make_game = f.read() + if 'if k.startswith(("A64_", "OOT_", "RX_"))' not in make_game: + raise SystemExit("fma steps: nx2/make_game.py's lift_key no longer takes the A64_ variables: A64_NO_FMA_STEPS is not in the key") + not_keyed = re.search(r"^LIFT_KEY_NOT = \((.*?)\) #", make_game, re.S | re.M) + if not not_keyed or '"fma_steps.py"' in not_keyed.group(1): + raise SystemExit("fma steps: nx2/fma_steps.py must stay a file of the lift key") + with open(os.path.join(ROOT, "docs", "development.md"), encoding="utf-8") as f: + if "| `A64_NO_FMA_STEPS=1` |" not in f.read(): + raise SystemExit("fma steps: docs/development.md: A64_NO_FMA_STEPS is not in the switches table") + + +def fields(line): + return {k: int(v) for k, v in (kv.split("=") for kv in line.split()[1:])} + + +def main(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--cc") + p.add_argument("--cxx") + p.add_argument("--random", type=int, default=20000) + p.add_argument("--direct", type=int, default=4000000) + a = p.parse_args() + check_sources() + if platform.machine().lower() not in ("x86_64", "amd64"): + fs.skip("not an x86-64 host: %s" % platform.machine()) + cc = fs.compiler(a.cc, "CC", ("clang", "gcc", "cc"), "clang") + cxx = fs.compiler(a.cxx, "CXX", ("clang++", "g++", "c++"), "clang++") + if not cc or not cxx: + fs.skip("no C compiler" if not cc else "no C++ compiler") + t0 = time.time() + failed, states, pairs, shapes = [], 0, 0, 0 + with tempfile.TemporaryDirectory(prefix="triforce-fma-steps-") as tmp: + shapes, rt_c = generate(cxx, tmp, fs.target_flags(cxx, tmp)) + for at, (march, more, fused) in enumerate(BUILDS): + what = "%s%s" % (march, " " + " ".join(more) if more else "") + common = [cc] + fs.target_flags(cc, tmp) + ["-march=" + march] + fs.FLAGS + more + \ + ["-D_CRT_SECURE_NO_WARNINGS", "-Wno-everything", "-I" + os.path.join(fs.KIT, "rt"), "-I" + tmp] + objs = [] + for side in ("now", "old"): # (the same lifted text against each side's runtime header) + obj = os.path.join(tmp, "side_%s_%d.o" % (side, at)) + fs.build(common + ["-DSIDE=" + side, "-I" + os.path.join(tmp, side), "-c", + os.path.join(ROOT, "tests", "fma_steps_side.c"), "-o", obj], "the %s side at %s" % (side, what)) + objs.append(obj) + exe = os.path.join(tmp, "fma_steps_test_%d.exe" % at) + fs.build(common + ["-I" + os.path.join(tmp, "now"), os.path.join(ROOT, "tests", "fma_steps_test.c"), rt_c] + objs + + ["-o", exe], "the driver at " + what) + for mode, n in (("lifted", a.random), ("direct", a.direct)): + r = fs.run([exe, mode, str(n), str(SEED)], timeout=3600) + print("%-28s %s" % (what, r.stdout.rstrip())) + if r.returncode == 3: + print("fma steps: %s skipped (this CPU cannot run it)" % what) + break + line = [l for l in r.stdout.splitlines() if l.startswith(mode + " ")] + if r.returncode or len(line) != 1: + failed.append("%s %s (exit %d)" % (what, mode, r.returncode)) + continue + c = fields(line[0]) + if fused is not None and bool(c["fused"]) != fused: + failed.append("%s: the steps are %sfused there" % (what, "" if c["fused"] else "not ")) + if mode == "lifted": + states += c["checks"] + continue + pairs += c["checks"] + if not c["named"] or not c["kept_old"]: + failed.append("%s direct: %s" % (what, c)) + elif c["fused"] and not c["now_only"]: + failed.append("%s direct: the fused form kept no pair the binary64 form declines" % what) + elif not c["fused"] and (c["now_only"] or c["kept_now"] != c["kept_old"]): + failed.append("%s direct: the sides decide differently without FMA3: %s" % (what, c)) + if failed: + raise SystemExit("FAIL: fma steps: %s" % "; ".join(failed)) + print("PASS: %d guest words, %d random states (each five ways) and %d operand pairs: the fused steps leave what the " + "kit's and the exact bodies leave (%.0f s)" % (shapes, states, pairs, time.time() - t0)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_fps_cap_monitor.py b/tests/test_fps_cap_monitor.py index 523d074..4b5fcd2 100644 --- a/tests/test_fps_cap_monitor.py +++ b/tests/test_fps_cap_monitor.py @@ -162,7 +162,9 @@ def main(): loop = vsync[vsync.index("for (;;)"):] check("1000000000LL / fps_max()" in loop, "the vsync event's period is taken once, before Monitor's rate is known") presented = block(vi, "void hos_vi_native_presented(void) {") - check("const int interval = fps_max() < 60 ? 1 : present_interval;" in presented and "fps_max() * interval" in presented, + # ((Triforce, 11-22, PRESENT-30) the period is pace_fps()'s, which is the cap unless a movie plays at a cap of 60) + check("const int interval = fps_max() < 60 ? 1 : present_interval;" in presented and "pace_fps() * interval" in presented + and "return movie_pace_fps(movie_raise, fps_max());" in block(vi, "static int pace_fps(void) {"), "a cap under 60 that settles late is paced with the game's own interval on top") readme = (ROOT / "readme.md").read_text(encoding="utf-8") check("| Frame rate cap | 30, 40, 45, 60, 90, 120, 144, 165, 240, Monitor's rate" in readme, diff --git a/tests/test_fps_overlay_order.py b/tests/test_fps_overlay_order.py index e4982e9..90aaf17 100644 --- a/tests/test_fps_overlay_order.py +++ b/tests/test_fps_overlay_order.py @@ -89,6 +89,10 @@ int page = 0; // ((Triforce) the menu's page: not t constexpr int kModsRowPage = 9; void draw_mods_row_shut(ui::Ctx&) { calls += "M"; } void draw_notice(ui::Ctx&) { calls += "n"; } +int ach_notice = -1; // ((Triforce) no achievement earned here: its notice is tests/test_achievements.py's) +int64_t ach_notice_at = 0; +constexpr int64_t kAchNoticeMs = 4000; +void draw_ach_notice(ui::Ctx&) { calls += "a"; } ''' + fn + r''' int main() { for (bool startup : {false, true}) for (int fps : {0, 1, 2}) for (bool menu : {false, true}) for (bool can : {true, false}) { diff --git a/tests/test_frame_rate_cap_step.py b/tests/test_frame_rate_cap_step.py index eb72e81..4d12504 100644 --- a/tests/test_frame_rate_cap_step.py +++ b/tests/test_frame_rate_cap_step.py @@ -52,8 +52,10 @@ FIXTURE = r'''#define _CRT_SECURE_NO_WARNINGS #include #include #include "step_steady.h" +#include "movie_pace.h" /* ((Triforce, 11-22, PRESENT-30) realtime_step goes by pace_fps: the cap here, no movie plays) */ #define CHECK(x) do { if (!(x)) { fprintf(stderr, "FAIL:%d %s\n", __LINE__, #x); exit(1); } } while (0) static int64_t pace_period_ns; /* (the period the presents are paced by: hos_vi_native_presented) */ +static int movie_raise; /* (nn_vi.c's, set once a present by movie_pace_decide: tests/test_movie_pace.py) */ static uint8_t guest[0x4000]; #define GP(a) ((void *)(guest + (a))) #define MOD 0x1000u @@ -181,8 +183,8 @@ def main(): "a second run left one divide's hook from the first") vi = (ROOT / "host" / "nn_vi.c").read_text(encoding="utf-8") - host = "\n".join(block(vi, m) for m in ("static float text_scale(", "static int fps_cap_of(", "static int fps_max(void) {", "static int realtime_rate(", - "static void realtime_step(")) + host = "\n".join(block(vi, m) for m in ("static float text_scale(", "static int fps_cap_of(", "static int fps_max(void) {", "static int pace_fps(void) {", + "static int realtime_rate(", "static void realtime_step(")) fixture = FIXTURE.replace("@HOST@", host).replace("@HOOK@", "\n".join(oot_patches.STEP_HOOK.split("\n")[:2])) with tempfile.TemporaryDirectory(prefix="cap-step-") as tmp: c = Path(tmp) / "fixture.c" diff --git a/tests/test_frame_trace.cpp b/tests/test_frame_trace.cpp index 027a349..362e362 100644 --- a/tests/test_frame_trace.cpp +++ b/tests/test_frame_trace.cpp @@ -23,15 +23,15 @@ int main() { // as many values as columns, in their order; outside: the interval less the replay and the present ft::Split s; s.interval = 16.0, s.replay = 9.5, s.present = 0.5, s.dispatch = 0.75, s.qsubmit = 0.25, s.par_wait = 1.5, s.game_wait = 2.0; - s.captures = 1.25, s.wb_wait = 3.0, s.wb_held = 2.5, s.qs_bg = 0.1; + s.captures = 1.25, s.wb_wait = 3.0, s.wb_held = 2.5, s.qs_bg = 0.1, s.gpu_waits = 1.75; CHECK("outside", ft::outside(s) == 6.0); char l[512]; const int n = ft::line(s, l, sizeof l); CHECK("a line", n > 0 && (size_t)n < sizeof l); CHECK("as many values as columns", commas(l) == commas(ft::kColumns)); - CHECK("the values in the columns' order", !std::strcmp(l, "9.50,0.50,0.75,0.25,1.50,6.00,2.00,1.25,3.00,2.50,0.10")); + CHECK("the values in the columns' order", !std::strcmp(l, "9.50,0.50,0.75,0.25,1.50,6.00,2.00,1.25,3.00,2.50,0.10,1.75")); CHECK("the columns", std::string(ft::kColumns) == "replay_ms,present_ms,dispatch_ms,qsubmit_ms,par_wait_ms,outside_ms," - "game_wait_ms,captures_ms,wb_wait_ms,wb_held_ms,qs_bg_ms"); + "game_wait_ms,captures_ms,wb_wait_ms,wb_held_ms,qs_bg_ms,gpu_waits_ms"); // ((Triforce, 11-22, SYNC-27) the last one) // (a replay that began before the present before, a submit across it: outside below 0, written as it is) ft::Split o; o.interval = 10.0, o.replay = 11.0, o.present = 0.5; diff --git a/tests/test_framegen_idle.py b/tests/test_framegen_idle.py new file mode 100644 index 0000000..9cb329e --- /dev/null +++ b/tests/test_framegen_idle.py @@ -0,0 +1,147 @@ +"""Frame generation's images are let go 120 presents after it stops (gpu/vk/framegen_idle.h, vk_framegen.cpp +framegen_idle, vk_replay.cpp's present; no game, no GPU). + +(Triforce, 11-21, the performance audit's PRESENT-18, after Source A's.) The built-in interpolation's frames and +made-frame sets were kept until a resize: turned off, or left for NVIDIA's own frame generation, they stayed in VRAM. + +This compiles framegen_idle.h alone and checks the rule: 119 presents off let nothing go and the 120th does; a present +with it on, or one that recorded made frames while the row already reads off, starts the count again; nothing goes with +no images; the images of the picture without the HUD have a count of their own, which only AMD FSR 4's row holds at +zero; RX_VK_FG_KEEP keeps everything while the counts go on. + +And reads the source: framegen_idle asks the rule with the row read again, what this present recorded and what there +is; before anything is dropped the display thread is done with its sets and every queue is idle; the present calls it +right after framegen_queue, with what it recorded; the switch is in the switches table. + + python -I -B tests/test_framegen_idle.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +PROGRAM = r""" +#include +#include +#include "framegen_idle.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +using namespace framegen_idle; +int main() { + CHECK(kPresents == 120); + { // off for 119 presents: nothing; the 120th lets them go + Counts c; + Release r{}; + for (int i = 0; i < 119; i++) { r = step(c, {false, false, false, true, false}, false); CHECK(!r.shared && !r.hudless); } + r = step(c, {false, false, false, true, false}, false); + CHECK(r.shared && !r.hudless && c.off == 120); + } + { // 119 off, then a present that recorded made frames though the row reads off by then: the count starts again + Counts c; + for (int i = 0; i < 119; i++) step(c, {false, false, false, true, false}, false); + Release r = step(c, {false, true, false, true, false}, false); + CHECK(!r.shared && c.off == 0); + for (int i = 0; i < 119; i++) { r = step(c, {false, false, false, true, false}, false); CHECK(!r.shared); } + CHECK(step(c, {false, false, false, true, false}, false).shared); + } + { // on: never, however long + Counts c; + for (int i = 0; i < 1000; i++) CHECK(!step(c, {true, i % 2 == 0, false, true, false}, false).shared); + CHECK(c.off == 0); + } + { // no images: nothing to let go, and the count goes on for when there are + Counts c; + for (int i = 0; i < 500; i++) CHECK(!step(c, {false, false, false, false, false}, false).shared); + CHECK(step(c, {false, false, false, true, false}, false).shared); + } + { // the picture without the HUD: held by AMD FSR 4's row alone; another kind of frame generation lets it go + Counts c; + for (int i = 0; i < 300; i++) CHECK(!step(c, {true, true, true, true, true}, false).hudless); + Release r{}; + for (int i = 0; i < 119; i++) { r = step(c, {true, true, false, true, true}, false); CHECK(!r.hudless && !r.shared); } + r = step(c, {true, true, false, true, true}, false); + CHECK(r.hudless && !r.shared); + CHECK(!step(c, {true, true, false, true, false}, false).hudless); // (gone: nothing more to let go) + Counts d; // (wanted by the row, but frame generation off) + for (int i = 0; i < 119; i++) step(d, {false, false, true, true, true}, false); + r = step(d, {false, false, true, true, true}, false); + CHECK(r.hudless && r.shared); + } + { // RX_VK_FG_KEEP: kept, the counts going on + Counts c; + for (int i = 0; i < 400; i++) { + const Release r = step(c, {false, false, false, true, true}, true); + CHECK(!r.shared && !r.hudless); + } + CHECK(c.off == 400 && step(c, {false, false, false, true, true}, false).shared); + } + CHECK(next_off(UINT32_MAX, false, false) == UINT32_MAX); // (a long run off does not wrap round to on) + return fails ? 1 : 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + fg = (VK / "vk_framegen.cpp").read_text(encoding="utf-8") + replay = (VK / "vk_replay.cpp").read_text(encoding="utf-8") + + check('#include "framegen_idle.h"' in fg, "vk_framegen.cpp does not use the rule") + idle = block(fg, "void framegen_idle(bool recorded) {") + check("const bool on = framegen_on();" in idle and + "framegen_idle::step(counts, {on, recorded, fsr4fg::frames_wanted(), fg.w != 0, have_hudless}, keep);" in idle, + "framegen_idle does not ask the rule with the row read again, what this present recorded and what there is") + check('getenv("RX_VK_FG_KEEP") && atoi(getenv("RX_VK_FG_KEEP")) != 0' in idle, "the switch is not RX_VK_FG_KEEP") + at = [idle.index(part) for part in ( + "if (!go.shared && !go.hudless) return;", + "fg.cv.wait(lk, [] { return fg.jobs.empty() && !fg.sets[0].busy && !fg.sets[1].busy && !fg.sets[2].busy; });", + "device_wait_idle_locked();", "drop_size();", "drop_img(hl.srgb), drop_img(hl.out)")] + check(at == sorted(at), "an image is dropped before the display thread is done with its sets and every queue is idle") + check("if (go.shared) {" in idle and "if (go.hudless) {" in idle and idle.index("if (go.hudless) {") < idle.index("std::lock_guard lk(hl.mu);", idle.index("if (go.hudless) {")), + "the two kinds of images are not let go each by its own count, the HUD-less ones under their lock") + start = replay.index("if (interpolated) framegen_queue(value);") + after = replay[start:start + 900] + # (after the queue and the else that belongs to it, which counts the frame for the FPS line) + check("framegen_idle(interpolated);" in after and replay.count("framegen_idle(") == 1 and + after.index("else if (!held) menu_note_frame(") < after.index("framegen_idle(interpolated);"), + "the present does not count after framegen_queue and its else, with what it recorded") + check("void framegen_idle(bool recorded);" in (VK / "vk_framegen.h").read_text(encoding="utf-8"), "vk_framegen.h does not declare it") + # (unchanged, as the tests that compile or read them need: the remake after a resize and AMD's context's own release) + check("void drop_size() {" in fg and "bool ensure_size(VkCommandBuffer cmd, uint32_t w, uint32_t h) {" in fg and + "void framegen_present_tick() { fsr4fg::present_tick(); }" in fg, "ensure_size, drop_size or framegen_present_tick changed") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`RX_VK_FG_KEEP=1`" in dev, "docs/development.md: RX_VK_FG_KEEP=1 is not in the switches table") + + with tempfile.TemporaryDirectory(prefix="framegen-idle-") as tmp: + c, exe = Path(tmp) / "idle.cpp", Path(tmp) / "idle.exe" + c.write_text(PROGRAM, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(VK), str(c), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + check(r.returncode == 0, "the rule as made:\n" + r.stdout) + print("PASS: frame generation's images are let go 120 presents after it stops, the GPU idle first") + + +if __name__ == "__main__": + main() diff --git a/tests/test_framegen_refresh.py b/tests/test_framegen_refresh.py new file mode 100644 index 0000000..1362f2e --- /dev/null +++ b/tests/test_framegen_refresh.py @@ -0,0 +1,119 @@ +"""With RX_VK_FRAMEGEN_REFRESH=1 a fixed frame generation multiplier makes no more frames than the display can show +(gpu/vk/framegen_refresh.h, vk_framegen.cpp framegen_record; no game, no GPU). + +(Triforce, 11-21, the upscaler audit's UPS-05.) The Frame generation row's 2x to 8x made that many frames for each of the +game's whatever the display's refresh rate: the owner's log has 5x on a 165 Hz display with the game near 66 frames a +second, 331 frames a second presented, half of them never on the screen, each present call 2.2 to 3.0 ms. + +This compiles framegen_refresh.h alone and checks the rule: the frames are the display's refreshes in a frame of the game, +rounded up, a tenth of a refresh forgiven; never more than the row's number, never fewer than the game's own frame; an +unknown frame time or refresh rate leaves the row's number; and the switch is off unless set. + +And reads the source: framegen_record takes the row's multiplier through set_multiplier as before, and only then, with +the switch on and the row at 2x to 8x, through the rule, with the game's recent frame time and the display's period; the +row's Auto and AMD FSR's are not touched; and the switch is in the switches table. + + python -I -B tests/test_framegen_refresh.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +PROGRAM = r""" +#include +#include +#include +#include "framegen_refresh.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +using namespace framegen_refresh; +static int64_t ns(double fps) { return (int64_t)(1e9 / fps); } +int main() { + CHECK(!on(nullptr) && !on("") && !on("0") && on("1")); + CHECK(period_ns(165000) == 6060606 && period_ns(60000) == 16666666 && period_ns(0) == 0 && period_ns(-1) == 0); + const int64_t hz165 = period_ns(165000), hz120 = period_ns(120000), hz60 = period_ns(60000), hz240 = period_ns(240000); + // (the owner's log: 5x, the game near 66 frames a second, 165 Hz: 2.5 refreshes a frame, 3 frames, not 5) + CHECK(frames(5, ns(66), hz165) == 3); + CHECK(frames(2, ns(66), hz165) == 2 && frames(3, ns(66), hz165) == 3 && frames(8, ns(66), hz165) == 3); + // (a game at a whole fraction of the refresh rate is not given one more for its frames' jitter) + CHECK(frames(4, ns(60), hz120) == 2 && frames(4, ns(59.5), hz120) == 2 && frames(4, ns(60.5), hz120) == 2); + CHECK(frames(4, ns(30), hz120) == 4 && frames(8, ns(30), hz120) == 4 && frames(8, ns(30), hz240) == 8); + CHECK(frames(4, ns(60), hz60) == 1 && frames(2, ns(60), hz60) == 1); // (at the display's rate: the game's frames alone) + CHECK(frames(4, ns(200), hz60) == 1 && frames(4, ns(165), hz165) == 1); // (faster than the display: never under 1) + CHECK(frames(2, ns(40), hz60) == 2 && frames(4, ns(40), hz60) == 2 && frames(4, ns(45), hz60) == 2); + CHECK(frames(8, ns(20), hz165) == 8 && frames(8, ns(24), hz165) == 7); + // (nothing known, or nothing to cap: the row's number) + CHECK(frames(5, 0, hz165) == 5 && frames(5, ns(66), 0) == 5 && frames(5, -1, hz165) == 5); + CHECK(frames(1, ns(66), hz165) == 1 && frames(0, ns(66), hz165) == 0); + // (never more than the row asks, never fewer than one, over every row and a sweep of rates) + for (int m = 1; m <= 8; m++) + for (int fps = 10; fps <= 400; fps += 7) + for (int64_t p : {hz60, hz120, hz165, hz240}) { + const int f = frames(m, ns(fps), p); + CHECK(f >= 1 && f <= m); + // (what is shown never falls short of the display where the row's number would have reached it) + if (m > 1 && (double)m * fps >= 1e9 / (double)p) CHECK((double)f * fps >= 0.9 * 1e9 / (double)p || f == m); + } + return fails ? 1 : 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + fg = (VK / "vk_framegen.cpp").read_text(encoding="utf-8") + + check('#include "framegen_refresh.h"' in fg, "vk_framegen.cpp does not use the rule") + rec = block(fg, "bool framegen_record(VkCommandBuffer cmd, VkImageView src, uint32_t sw, uint32_t sh, const int crop[4]) {") + at = [rec.index(part) for part in ( + "fg.cv.wait(lk, [] { return !fg.sets[fg.next_set].busy && (!pace::display_yields() || fg.jobs.empty()); });", + "int m = framegen_rows::set_multiplier(multiplier(), fg.mult);", + 'static const bool refresh_cap = framegen_refresh::on(getenv("RX_VK_FRAMEGEN_REFRESH"));', + "if (refresh_cap && opts.framegen >= 2 && opts.framegen <= kMaxMult) {", + "m = framegen_refresh::frames(m, fg.game_ns, framegen_refresh::period_ns(display_refresh_mhz()));", + "const bool blend = fg.have_prev && !gap;", + "if (opts.framegen == 1) {", + "if (set->made == 0 && !none && blend && m > 1) {", + "for (int k = 1; k < m; k++) make_frame(cmd, prev, cur, set->frame[k - 1], (float)k / (float)m, uw, uh);")] + check(at == sorted(at), "framegen_record does not take the row's multiplier through set_multiplier and then, with the switch, the rule") + check(rec.count("framegen_refresh::frames(") == 1 and "framegen_refresh::" not in fg.replace(rec, "").replace('#include "framegen_refresh.h"', ""), + "the rule is asked somewhere else than framegen_record's fixed multipliers") + auto = rec[rec.index("if (opts.framegen == 1) {"):rec.index("bool none = false;")] + check("framegen_refresh" not in auto, "the row's Auto goes through the rule: it has its own, by the display's refreshes") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`RX_VK_FRAMEGEN_REFRESH=1`" in dev, "docs/development.md: RX_VK_FRAMEGEN_REFRESH=1 is not in the switches table") + + with tempfile.TemporaryDirectory(prefix="framegen-refresh-") as tmp: + c, exe = Path(tmp) / "refresh.cpp", Path(tmp) / "refresh.exe" + c.write_text(PROGRAM, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(VK), str(c), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + check(r.returncode == 0, "the rule as made:\n" + r.stdout) + print("PASS: with RX_VK_FRAMEGEN_REFRESH=1 a fixed multiplier makes no more frames than the display can show") + + +if __name__ == "__main__": + main() diff --git a/tests/test_framegen_set_bounds.py b/tests/test_framegen_set_bounds.py new file mode 100644 index 0000000..77aaba8 --- /dev/null +++ b/tests/test_framegen_set_bounds.py @@ -0,0 +1,116 @@ +"""Frame interpolation never writes past the frames its sets were made with, whatever the Frame generation row does while +a present waits (gpu/vk/framegen_rows.h set_multiplier and game_slot, vk_framegen.cpp framegen_record and ensure_size; no +game). + +(Triforce, 11-20) The owner's "upscaling crash": three runs of 11-20 ended in an access violation reading 0x590 inside +NVIDIA's driver, in a vkCmdCopyImage run by the recorder thread while a present waited for it. Each had frame +interpolation on, and the Frame generation row had been stepped up in the options (Auto, then 3x at the next start, then +5x). The sets' frames are made for the row's multiplier when a present begins; the present then waits for the display +thread to let a set go, and while frames are made that thread runs the menu, so a step of the row lands in the wait. The +multiplier read again after it was more than the frames made: the made frames went into the frames there were and the +game's own into an image that was never made. + +This compiles framegen_rows.h alone and checks the two rules: the multiplier a present records with is the row's and no +more than the frames made; the game's frame goes after the made ones and into the set's last frame at most. Then, for +every number of frames a set can be made with and every multiplier the row can go to in the wait, that every frame a +present writes is one the set has. + +And reads the source for what only the game shows: the sets made with as many frames as the multiplier read there, and +that number kept; framegen_record taking its multiplier through set_multiplier with the frames made, after the wait, and +no bare multiplier(); the game's frame's place through game_slot before the copy; Auto's frames counted from the frames +made. + + python -I -B tests/test_framegen_set_bounds.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +PROGRAM = r""" +#include +#include +#include "framegen_rows.h" +using namespace mkwvk::framegen_rows; +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +int main() { + // the multiplier a present records with + CHECK(set_multiplier(3, 3) == 3 && set_multiplier(2, 3) == 2 && set_multiplier(8, 8) == 8); + CHECK(set_multiplier(4, 3) == 3 && set_multiplier(5, 3) == 3 && set_multiplier(8, 4) == 4); // (stepped up in the wait) + CHECK(set_multiplier(0, 3) == 1 && set_multiplier(-2, 3) == 1); // (Off in the wait: the game's frame) + CHECK(set_multiplier(5, 0) == 1 && set_multiplier(5, 1) == 1); + // the game's frame's place + CHECK(game_slot(2, 3) == 2 && game_slot(0, 3) == 0 && game_slot(7, 8) == 7); + CHECK(game_slot(3, 3) == 2); // (the crash: three made frames over sets of three, the game's into a fourth) + CHECK(game_slot(1, 1) == 0); // (AMD FSR 4 in the wait, over sets made for the game's frames alone) + CHECK(game_slot(0, 0) == 0 && game_slot(-1, 3) == 0); + // a present, as framegen_record writes it: the made frames into 0 to m - 2, the game's after them + for (int made_with = 1; made_with <= kMaxMult; made_with++) + for (int row = 0; row <= kMaxMult; row++) { + const int m = set_multiplier(row, made_with); + for (int k = 1; k < m; k++) CHECK(k - 1 < made_with); + const int game = game_slot(m - 1, made_with); + CHECK(game >= 0 && game < made_with); + CHECK(row > made_with || row < 1 || (m == row && game == row - 1)); // (a row the sets were made for: as it asks) + } + CHECK(game_slot(1, 2) == 1 && game_slot(1, 8) == 1); // (AMD's one made frame, sets of two or more: the game's second) + return fails ? 1 : 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + fg = (VK / "vk_framegen.cpp").read_text(encoding="utf-8") + + size = block(fg, "bool ensure_size(VkCommandBuffer cmd, uint32_t w, uint32_t h) {") + check("const int mult = multiplier();" in size and "for (int k = 0; k < mult; k++)" in size and "fg.mult = mult;" in size, + "ensure_size: the sets' frames are not made for one reading of the multiplier, kept as fg.mult") + + record = block(fg, "bool framegen_record(VkCommandBuffer cmd, VkImageView src, uint32_t sw, uint32_t sh, const int crop[4]) {") + taken = "m = framegen_rows::set_multiplier(multiplier(), fg.mult);" + check(taken in record, "framegen_record: the multiplier is not held to the frames the sets were made with") + check(record.count("multiplier()") == 1, "framegen_record reads the row's multiplier again, past the frames made") + wait = record.index("fg.cv.wait(lk,") + check(record.index("ensure_size(cmd,") < wait < record.index(taken), + "framegen_record: the multiplier is taken before the wait for a set, where the menu may still step the row") + slot, copy = "set->made = framegen_rows::game_slot(set->made, fg.mult);", "set->frame[set->made].image" + check(slot in record and record.index(slot) < record.index(copy), "framegen_record: the game's frame may go past the set's last frame") + check(record.count(copy) == 1, "framegen_record copies the game's frame into a set in a second place") + check("n = auto_slots(before, now, period, fg.slot_next, t, at, std::min(fg.mult, kMaxMult));" in record, + "Auto's frames are not counted from the frames the sets were made with") + + with tempfile.TemporaryDirectory(prefix="framegen-set-bounds-") as tmp: + src, exe = Path(tmp) / "bounds.cpp", Path(tmp) / "bounds.exe" + src.write_text(PROGRAM, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(VK), str(src), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + check(r.returncode == 0, "the frames a present writes:\n" + r.stdout) + print("PASS: a present writes only frames its sets were made with, whatever the Frame generation row does in its wait") + + +if __name__ == "__main__": + main() diff --git a/tests/test_gpl_library_cache.py b/tests/test_gpl_library_cache.py index c16f05e..ec6a3c8 100644 --- a/tests/test_gpl_library_cache.py +++ b/tests/test_gpl_library_cache.py @@ -25,7 +25,7 @@ def main(): build = between(draw, "VkPrimitiveTopology topo, int part, bool* cold) {", "\n}\n") for line, why in ( ("const bool lib_cached = (part == 1 || part == 2) && gpl_libs_cached();", "only the two shader libraries are kept"), - ("const VkPipelineCache wanted = lib_cached || (part < 0 && !driver_cache_only) ? pcache : VK_NULL_HANDLE;", + ("const VkPipelineCache wanted = lib_cached || (part < 0 && !driver_cache_only && !pcache_libs_only) ? pcache : VK_NULL_HANDLE;", # ((Triforce, 11-22, PIPE-78) whole pipelines out when lean) "the libraries are not built into the pipeline cache"), # ((Triforce) through the cache's gate, held over the creation: none once it is let go for the RAM, pcache_gate.h) ("const PcacheUse use(pcache_door, wanted);", "the creation does not hold the cache's gate"), diff --git a/tests/test_gpu_choice.cpp b/tests/test_gpu_choice.cpp index 05174af..8a2b5d1 100644 --- a/tests/test_gpu_choice.cpp +++ b/tests/test_gpu_choice.cpp @@ -26,5 +26,25 @@ int main() { const std::vector twins = {"NVIDIA GeForce RTX 3090", "NVIDIA GeForce RTX 3090"}; CHECK("two cards of one name: the numbered one when it has that name", pick("NVIDIA GeForce RTX 3090", twins, 2) == 1); CHECK("a number past the list with a name: the first of that name", pick("NVIDIA GeForce RTX 3090", twins, 5) == 0); + + // ((Triforce) 11-22) a number alone on the processor graphics while a card of its own is there: Auto (RX 6600 + 5600G: the + // launcher's "1: AMD Radeon RX 6600" in Windows' order saved 2, the game's 2 is the 5600G's graphics) + using mkwvk::gpu_choice::unnamed_integrated; + const std::vector apu = {"AMD Radeon RX 6600", "AMD Radeon(TM) Graphics"}; + const std::vector apu_types = {2, 1}; + CHECK("no name, the number on the processor graphics: Auto", unnamed_integrated("", apu, apu_types, 2)); + CHECK("a name no card has, the number on the processor graphics: Auto", + unnamed_integrated("AMD Radeon RX 6600 XT", apu, apu_types, 2)); + CHECK("the processor graphics chosen by its name: taken", !unnamed_integrated("AMD Radeon(TM) Graphics", apu, apu_types, 2)); + CHECK("the name finds the card under another number: pick's", !unnamed_integrated("AMD Radeon RX 6600", apu, apu_types, 2) && + pick("AMD Radeon RX 6600", apu, 2) == 0); + CHECK("no name, the number on the card: taken", !unnamed_integrated("", apu, apu_types, 1)); + CHECK("Auto: Auto as before", !unnamed_integrated("", apu, apu_types, 0)); + CHECK("processor graphics alone (no card of its own): the number", !unnamed_integrated("", {"Intel(R) Iris(R) Xe Graphics"}, {1}, 1)); + const std::vector twice = {"AMD Radeon RX 7900 XTX", "AMD Radeon RX 7900 XTX", "AMD Radeon(TM) Graphics"}; + CHECK("the card listed twice, the number on the processor graphics: Auto", unnamed_integrated("", twice, {2, 2, 1}, 3)); + CHECK("two processor graphics and no card (a software device, type 4): the number", + !unnamed_integrated("", {"AMD Radeon(TM) Graphics", "llvmpipe (LLVM 22.1.8, 256 bits)"}, {1, 4}, 1)); + CHECK("types shorter than names: the number", !unnamed_integrated("", apu, {2}, 2)); std::puts("gpu choice: all checks passed"); } diff --git a/tests/test_gpu_choice.py b/tests/test_gpu_choice.py index 0050763..31797c7 100644 --- a/tests/test_gpu_choice.py +++ b/tests/test_gpu_choice.py @@ -38,6 +38,18 @@ def main(): raise SystemExit("pick_device reads the GPU row before menu_init") if pick.index('getenv("RX_VK_DEVICE")') > pick.index("gpu_choice::pick("): raise SystemExit("pick_device takes the GPU row before RX_VK_DEVICE") + # ((Triforce) 11-22) a number saved without a name on the processor graphics: Auto, after RX_VK_DEVICE and before pick, the + # types beside the names, and the row set to the card Auto took + unnamed = "gpu_choice::unnamed_integrated(menu_gpu_name(), gpu_device_names, valid_types, opts.gpu_device)" + if unnamed not in pick: + raise SystemExit("pick_device: a number without a name on the processor graphics is taken (no unnamed_integrated)") + if not pick.index('getenv("RX_VK_DEVICE")') < pick.index(unnamed) < pick.index("gpu_choice::pick("): + raise SystemExit("pick_device: unnamed_integrated is not between RX_VK_DEVICE and the row's pick") + if "valid_types.push_back((int)p.deviceType)" not in pick or "gpu_device_names.push_back" not in pick: + raise SystemExit("pick_device: the device types are not listed beside the names") + fix = pick.index("if (unnamed_integrated) {") + if not pick.index("best = (int)i") < fix < pick.index("dev.physical = all[best]") or "opts.gpu_device = (int)k + 1" not in pick[fix:]: + raise SystemExit("pick_device: the row does not follow the card Auto took for an unnamed number") menu = (vk / "vk_menu.cpp").read_text(encoding="utf-8") if not preset_exempt(menu, "gpu_device"): raise SystemExit("vk_menu.cpp: changing the GPU row resets the quality preset (and its VRAM budget) to Custom") diff --git a/tests/test_gpu_clock.py b/tests/test_gpu_clock.py new file mode 100644 index 0000000..564ae58 --- /dev/null +++ b/tests/test_gpu_clock.py @@ -0,0 +1,146 @@ +"""The Steam Deck's GPU held at amdgpu's peak stable power state while the game runs (gpu/vk/gpu_clock.h, vk_gpu_clock.cpp, +vk_main.cpp create_device; RX_VK_GPU_CLOCK; Linux; no game/GPU). + +The rule is compiled from the header and run on made-up devices: unset, the peak state on the Deck's APU (its LCD and OLED Vulkan +names, or SteamDeck=1 on an integrated GPU) only while the level is "auto"; a level set by hand, or one not read, left alone; +SteamDeck=1 on a discrete GPU, another AMD GPU and any other vendor left alone; =0 nothing; =peak, =1 and =standard on any AMD GPU +at any level, never on NVIDIA. The ioctl's numbers against what amdgpu_drm.h gives them (written out here, as it was before and +since Linux 5.18; on Linux the system's own header too, when it is there). The source: create_device calls it after vkCreateDevice, +on Linux alone, with VK_EXT_physical_device_drm's presence; vk_gpu_clock.cpp is Linux's alone, checks the node it opens is the +device Vulkan names, sets the state on a context of its own and keeps the file open; the docs row is there. + + python -I -B tests/test_gpu_clock.py [--cxx clang++] +""" +import argparse +import os +from pathlib import Path +import subprocess +import sys +import tempfile + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] + +FIXTURE = r'''#include +#include "gpu_clock.h" +#ifdef CHECK_SYSTEM_HEADER +#include +static_assert(mkwvk::gpu_clock::kCtxIoctl == DRM_IOCTL_AMDGPU_CTX, "DRM_IOCTL_AMDGPU_CTX"); +static_assert(mkwvk::gpu_clock::kCtxArgBytes == sizeof(union drm_amdgpu_ctx), "union drm_amdgpu_ctx"); +static_assert(mkwvk::gpu_clock::kCtxOpAlloc == AMDGPU_CTX_OP_ALLOC_CTX, "AMDGPU_CTX_OP_ALLOC_CTX"); +#ifdef AMDGPU_CTX_OP_SET_STABLE_PSTATE +static_assert(mkwvk::gpu_clock::kCtxOpSetStablePstate == AMDGPU_CTX_OP_SET_STABLE_PSTATE, "AMDGPU_CTX_OP_SET_STABLE_PSTATE"); +static_assert(mkwvk::gpu_clock::kPstatePeak == AMDGPU_CTX_STABLE_PSTATE_PEAK, "AMDGPU_CTX_STABLE_PSTATE_PEAK"); +static_assert(mkwvk::gpu_clock::kPstateStandard == AMDGPU_CTX_STABLE_PSTATE_STANDARD, "AMDGPU_CTX_STABLE_PSTATE_STANDARD"); +#endif +#endif +using namespace mkwvk::gpu_clock; +// (amdgpu_drm.h's numbers, written out: DRM_COMMAND_BASE 0x40, DRM_AMDGPU_CTX 0x02, _IOC_READ | _IOC_WRITE 3, a 16-byte union; +// the ops and the states of Linux 5.18's) +static_assert(kCtxIoctl == 0xC0106442ul, "DRM_IOCTL_AMDGPU_CTX"); +static_assert(kCtxOpAlloc == 1 && kCtxOpSetStablePstate == 6 && kPstateStandard == 1 && kPstatePeak == 4, "the ops and states"); +static int bad = 0; +static void expect(Want got, Want want, const char* what) { + if (got != want) { std::printf("FAIL %s: %d, wanted %d\n", what, (int)got, (int)want); bad++; } +} +int main() { + const char* lcd = "AMD Custom GPU 0405 (RADV VANGOGH)"; + const char* oled = "AMD Custom GPU 0932 (RADV VANGOGH)"; + expect(choose(nullptr, 0x1002, true, lcd, nullptr, "auto").want, kPeak, "Deck LCD, auto"); + expect(choose(nullptr, 0x1002, true, oled, nullptr, "auto").want, kPeak, "Deck OLED, auto"); + expect(choose(nullptr, 0x1002, true, "AMD Radeon Graphics (RADV VANGOGH)", "1", "auto").want, kPeak, "SteamDeck=1, integrated"); + expect(choose(nullptr, 0x1002, false, "AMD Radeon RX 6800 XT (RADV NAVI21)", "1", "auto").want, kNone, "SteamDeck=1, discrete"); + expect(choose(nullptr, 0x1002, true, "AMD Radeon 780M (RADV PHOENIX)", nullptr, "auto").want, kNone, "another AMD APU"); + expect(choose(nullptr, 0x1002, true, "AMD Radeon 780M (RADV PHOENIX)", "0", "auto").want, kNone, "SteamDeck=0"); + expect(choose(nullptr, 0x1002, true, oled, nullptr, "manual").want, kNone, "Deck, level manual"); + expect(choose(nullptr, 0x1002, true, oled, nullptr, "profile_peak").want, kNone, "Deck, level profile_peak"); + expect(choose(nullptr, 0x1002, true, oled, nullptr, nullptr).want, kNone, "Deck, level unread"); + expect(choose("0", 0x1002, true, oled, nullptr, "auto").want, kNone, "=0 on the Deck"); + expect(choose("peak", 0x1002, false, "AMD Radeon RX 7900 XTX (RADV NAVI31)", nullptr, "manual").want, kPeak, "=peak, any AMD"); + expect(choose("1", 0x1002, false, "AMD Radeon RX 7900 XTX", nullptr, nullptr).want, kPeak, "=1, any AMD"); + expect(choose("standard", 0x1002, true, oled, nullptr, "auto").want, kStandard, "=standard"); + expect(choose("peak", 0x10DE, false, "NVIDIA GeForce RTX 4070", nullptr, "auto").want, kNone, "=peak on NVIDIA"); + expect(choose(nullptr, 0x10DE, true, lcd, "1", "auto").want, kNone, "NVIDIA, a Deck's name"); + expect(choose(nullptr, 0x8086, true, "Intel(R) Graphics", "1", "auto").want, kNone, "Intel"); + if (bad) return 1; + std::printf("OK\n"); + return 0; +} +''' + + +def fail(message): + raise SystemExit("FAIL: " + message) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def check_source(): + main = (ROOT / "gpu/vk/vk_main.cpp").read_text(encoding="utf-8") + clock = (ROOT / "gpu/vk/vk_gpu_clock.cpp").read_text(encoding="utf-8") + dev = (ROOT / "docs/development.md").read_text(encoding="utf-8") + create = block(main, "bool create_device() {") + call = " gpu_clock_device_created(has_extension(exts, VK_EXT_PHYSICAL_DEVICE_DRM_EXTENSION_NAME));\n" + at = create.index(call) + if create.index("vkCreateDevice(") > at: + fail("create_device calls gpu_clock_device_created before vkCreateDevice") + if "#ifdef __linux__" not in "\n".join(create[:at].splitlines()[-2:]) or not create[at + len(call):].lstrip().startswith("#endif"): + fail("the call is not Linux's alone") + body = clock[clock.index('#include "vk_gpu_clock.h"') + len('#include "vk_gpu_clock.h"'):].strip() + if not body.startswith("#ifdef __linux__") or not body.endswith("#endif"): + fail("vk_gpu_clock.cpp is not Linux's alone") + fn = block(clock, "void gpu_clock_device_created(bool drm_ext) {") + for needle in ("major(st.st_rdev) != (unsigned)drm.renderMajor", "minor(st.st_rdev) != (unsigned)drm.renderMinor", + "a.in.op = gpu_clock::kCtxOpAlloc;", "s.in.op = gpu_clock::kCtxOpSetStablePstate;", "s.in.ctx_id = a.alloc.ctx_id;", + "held_fd = fd;", "gpu_clock::choose(sw, dev.props.vendorID, integrated, dev.props.deviceName, getenv(\"SteamDeck\")"): + if needle not in fn: + fail("gpu_clock_device_created no longer has: " + needle) + tail = fn[fn.index("held_fd = fd;"):] + if "close(fd)" in tail: + fail("gpu_clock_device_created closes the file it holds the state with") + if fn.index("if (dev.props.vendorID != 0x1002) return;") > fn.index("logf("): + fail("gpu_clock_device_created logs on GPUs that are not AMD's") + if "`RX_VK_GPU_CLOCK=0`" not in dev: + fail("docs/development.md has no RX_VK_GPU_CLOCK row") + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--cxx", default="g++") + a = p.parse_args() + try: + check_source() + except ValueError as e: + fail("a text this test reads moved (%s)" % e) + flags = subprocess.BELOW_NORMAL_PRIORITY_CLASS if os.name == "nt" else 0 + system_header = sys.platform.startswith("linux") and any( + Path(d, "drm/amdgpu_drm.h").is_file() for d in ("/usr/include", "/usr/include/libdrm")) + with tempfile.TemporaryDirectory(prefix="gpu-clock-") as tmp: + cpp = Path(tmp) / "fixture.cpp" + exe = Path(tmp) / ("fixture" + (".exe" if os.name == "nt" else "")) + cpp.write_text(FIXTURE, encoding="utf-8") + extra = ["-DCHECK_SYSTEM_HEADER", "-I/usr/include/libdrm"] if system_header else [] + built = subprocess.run([a.cxx, "-std=c++20", "-O1", "-Wall", "-Werror", "-I" + str(ROOT / "gpu/vk")] + extra + + [str(cpp), "-o", str(exe)], capture_output=True, text=True, creationflags=flags) + if built.returncode: + print(built.stdout + built.stderr) + fail("the fixture did not build") + done = subprocess.run([str(exe)], capture_output=True, text=True, timeout=60, creationflags=flags) + if done.returncode or done.stdout.strip() != "OK": + fail((done.stdout + done.stderr).strip()) + print("PASS: the GPU's stable power state: peak on the Steam Deck's APU at level auto, the switch's states on any AMD GPU, " + "nothing elsewhere; the ioctl's numbers%s; create_device calls it on Linux after vkCreateDevice and the file stays open" + % (" (and the system's amdgpu_drm.h)" if system_header else "")) + + +if __name__ == "__main__": + main() diff --git a/tests/test_gpu_waits.py b/tests/test_gpu_waits.py new file mode 100644 index 0000000..a9b18ae --- /dev/null +++ b/tests/test_gpu_waits.py @@ -0,0 +1,113 @@ +"""The host's time in calls that block on the GPU, counted by kind at the Vulkan wait wrappers (gpu/vk/gpu_wait_kinds.h, +vk_work.h's five wrappers, vk_replay.cpp's line and the frame trace's column; no game, no GPU). + +(Triforce, 11-22, the performance audit's SYNC-27, after Source A's.) The log's "GPU-bound wait" counts the waits for a +recording slot alone and reads about 0 while threads wait on the GPU elsewhere. Every blocking wait is now timed in one +place, vk_work.h, and summed by kind: fences, timelines, idle, queries. + +tests/gpu_waits_test.cpp gets vk_work.h's wrappers cut from the source and runs them over stand-ins for the driver: + - each wrapper passes its arguments and the driver's result through; + - a fence or semaphore wait with timeout 0 and a query read without the wait bit count nothing; with a timeout above 0, + the wait bit, or an idle, the time lands under its own kind and no other; + - a wait inside a wait counts once, as the outer one; a wait inside an Exclude scope counts nothing, and the count + comes back after the scope; + - two readers each see every nanosecond and every wait, whenever they read; + - four threads' 2,000 waits are all counted while a fifth sits in an Exclude scope; + - RX_VK_GPU_WAITS=0: the calls are made as before and nothing is counted. + +And reads the source: the five macros call their wrapper after the recorder's boundary and no other macro changed; the +low-latency sleep's wait is in an Exclude scope; the line after the 120-present line and the trace's column each have +their own reader; `lab` stays the trace's last column; the switches table has the row. + + python -I -B tests/test_gpu_waits.py [--cxx clang++] +""" +import argparse +import os +import re +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] +WAITS = {"vkQueueWaitIdle": "queue_idle", "vkDeviceWaitIdle": "device_idle", "vkWaitForFences": "fences", + "vkWaitSemaphores": "semaphores", "vkGetQueryPoolResults": "query_results"} + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + check((VK / "gpu_wait_kinds.h").exists(), "gpu/vk/gpu_wait_kinds.h is missing") + work = (VK / "vk_work.h").read_text(encoding="utf-8") + check('#include "gpu_wait_kinds.h"' in work, "vk_work.h does not include gpu_wait_kinds.h") + + # the five macros, and only they + macros = [] + for call, wrapper in WAITS.items(): + line = "#define %s(...) (::mkwvk::rec::boundary(::mkwvk::rec::B_%s), ::mkwvk::gpu_waits_vk::%s(::vkx::%s, __VA_ARGS__))" % (call, call, wrapper, call) + check(line in work, "vk_work.h: %s does not call its timed wrapper, with the driver's call, after the recorder's boundary" % call) + macros.append(line) + timed_macros = re.findall(r"^#define (\w+)\(\.\.\.\) .*gpu_waits_vk::", work, re.M) + check(sorted(timed_macros) == sorted(WAITS), "vk_work.h: the timed macros are not the five waits: %s" % timed_macros) + start, end = work.index("namespace mkwvk::gpu_waits_vk {"), work.index("} // namespace mkwvk::gpu_waits_vk") + wrappers = work[start:end] + "}\n" + check(start < work.index("#define vkCmdPipelineBarrier2("), "vk_work.h: the wrappers are below the macros they would be changed by") + for name, kind, blocks, call in (("queue_idle", "IDLE", "true", "call(queue)"), ("device_idle", "IDLE", "true", "call(device)"), + ("fences", "FENCE", "timeout != 0", "call(device, count, list, all, timeout)"), + ("semaphores", "TIMELINE", "timeout != 0", "call(device, info, timeout)"), + ("query_results", "QUERY", "(flags & VK_QUERY_RESULT_WAIT_BIT) != 0", + "call(device, pool, first, count, size, data, stride, flags)")): + pattern = r"inline VkResult %s\(Call&& call, [^{]*\{\s*return ::gpu_wait_kinds::timed\(::gpu_wait_kinds::%s, %s,\s*\[&\] \{ return %s; \}\);" % ( + name, kind, re.escape(blocks), re.escape(call)) + check(re.search(pattern, wrappers), "vk_work.h: %s is not timed as %s when %s" % (name, kind, blocks)) + check("::vkx::" not in wrappers, "vk_work.h: a wrapper names the driver's call itself: a file that never waits that way would need it declared") + + reflex = (VK / "vk_reflex.cpp").read_text(encoding="utf-8") + at = reflex.index("vkWaitSemaphores(dev.device, &w, 100000000ull);") + check("const ::gpu_wait_kinds::Exclude pacing;\n vkWaitSemaphores(dev.device, &w, 100000000ull);" in reflex and + reflex[at:at + 200].split("\n")[1].strip() == "}", "vk_reflex.cpp: the low-latency sleep's wait is not alone in an Exclude scope") + check(reflex.count("vkWaitSemaphores(") == 1, "vk_reflex.cpp has another semaphore wait: excluded or not?") + + rep = (VK / "vk_replay.cpp").read_text(encoding="utf-8") + order = [rep.find(s) for s in ("GPU-bound wait %.1f ms; ", "static ::gpu_wait_kinds::Reader waits;", "const int64_t all = waits.take(ns, n);", + "if (::gpu_wait_kinds::on())", "vk: host waits for the GPU: %.2f ms a frame (fences %.2f, timelines %.2f, idle %.2f, queries %.2f;")] + check(min(order) >= 0 and order == sorted(order), "vk_replay.cpp: the host waits' line is not after the 120-present line, from a reader of its own") + tick = rep[rep.index("void status_tick() {"):] + tick = tick[:tick.index("\n}\n")] + order = [tick.find(s) for s in ("frame_trace::Split sp;", "static ::gpu_wait_kinds::Reader trace_waits;", + "sp.gpu_waits = trace_waits.take(nullptr, nullptr) / 1e6;", "frame_trace::line(sp, split, sizeof split);", + 'snprintf(split_lab, sizeof split_lab, "%s,%d", split, lab_col);')] + check(min(order) >= 0 and order == sorted(order), "status_tick: the trace's gpu_waits_ms is not this frame's, from a reader of its own, before lab") + check('if (f) fputs(",lab\\n", f);' in tick and tick.index("frame_trace::kColumns);") < tick.index('if (f) fputs(",lab\\n", f);'), + "status_tick: lab is no longer the trace's last column") + check(rep.count("gpu_wait_kinds::Reader") == 2, "vk_replay.cpp: the readers of the waits are not the line's and the trace's") + trace_h = (VK / "frame_trace.h").read_text(encoding="utf-8") + check('wb_held_ms,qs_bg_ms,"\n "gpu_waits_ms";' in trace_h and "s.qs_bg, s.gpu_waits);" in trace_h, "frame_trace.h: gpu_waits_ms is not the split's last column") + kinds = (VK / "gpu_wait_kinds.h").read_text(encoding="utf-8") + check('getenv("RX_VK_GPU_WAITS")' in kinds and "static const bool env" in kinds and "!(e && *e && atoi(e) == 0)" in kinds and + 'return env && !mkwvk::lab_ab::on("nogpuwaits");' in kinds, "RX_VK_GPU_WAITS is not read once, on unless 0, with the lab knob nogpuwaits") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`RX_VK_GPU_WAITS=0`" in dev and "`gpu_waits_ms`" in dev and "`nogpuwaits`" in dev, + "docs/development.md: RX_VK_GPU_WAITS=0, the column gpu_waits_ms or the knob is not described") + + base_env = {k: v for k, v in os.environ.items() if not k.startswith("RX_VK_")} + with tempfile.TemporaryDirectory(prefix="gpu-waits-") as tmp: + src, exe = Path(tmp) / "waits.cpp", Path(tmp) / "waits.exe" + src.write_text((ROOT / "tests" / "gpu_waits_test.cpp").read_text(encoding="utf-8").replace("/* WRAPPERS */", wrappers).replace("/* MACROS */", "\n".join(macros)), encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-pthread", "-I" + str(VK), str(src), "-o", str(exe)], check=True) + for mode, env in (("on", base_env), ("off", dict(base_env, RX_VK_GPU_WAITS="0"))): + r = subprocess.run([str(exe), mode], env=env, capture_output=True, text=True, timeout=120) + print(r.stdout.rstrip()) + check(r.returncode == 0, "RX_VK_GPU_WAITS %s: the waits' checks failed" % mode) + print("PASS: every blocking wait for the GPU is timed once under its kind; polls, pacing and RX_VK_GPU_WAITS=0 count nothing") + + +if __name__ == "__main__": + main() diff --git a/tests/test_guest_census.py b/tests/test_guest_census.py index a313ec5..763faf3 100644 --- a/tests/test_guest_census.py +++ b/tests/test_guest_census.py @@ -73,6 +73,14 @@ static unsigned char scratch[1 << 20]; static uint64_t window_lo; #define GP(a) ((void *)((uintptr_t)scratch + (uintptr_t)((a) - window_lo))) static uint64_t guest_of(const void *p) { return (uint64_t)((uintptr_t)p - (uintptr_t)scratch) + window_lo; } +/* ((Triforce, 11-22, the performance audit's MEM-13) host/hos.h's fill of guest memory: the bytes, and the range it would + mark written, counted (tests/test_host_free_marks.py has the helper itself and its switch) */ +static int fills; +static uint64_t fill_lo, fill_n; +static void hos_guest_fill(uint64_t a, int v, uint64_t n) { + memset(GP(a), v, (size_t)n); + fills++, fill_lo = a, fill_n = n; +} typedef size_t SIZE_T; #define MEM_DECOMMIT 0x4000 static int decommits; @@ -96,6 +104,13 @@ static void hos_guest_io_end(int io) { (void)io; } static int cores_away, cores_back, core_away_at_discard; static int hos_vcore_away(void) { cores_away++; return 1; } static void hos_vcore_back(void) { cores_back++; } +/* ((Triforce, OS-51) the host's part of a long frame, host/stall_host.h: counted around the discard and the zero-fill, not here) */ +enum { STALL_HOST_DISCARD, STALL_HOST_ZERO }; +static int64_t stall_host_begin(void) { return 0; } +static void stall_host_end(int cause, int64_t began, uint64_t bytes) { (void)cause, (void)began, (void)bytes; } +/* ((Triforce, OS-40) HOS_DISCARD_KEEP_CORE's line when it is set, and the time a freed region waits for its core back) */ +static void hos_log(const char *fmt, ...) { (void)fmt; } +static int64_t hos_now_ns(void) { return 0; } /* A discard as it happens: whether a listed region still holds its pages (so that hos_region_alloc would not hand them out meanwhile), and whether the regions' lock is held */ static int seen, seen_listed, seen_locked; @@ -463,7 +478,12 @@ static void test_region_free(void) { memset(&list[1], 0, sizeof list[1]); list[1].addr = r, list[1].size = 0x40000, nregions = 2, imported_now = 1, window_lo = r; hos_mem_note(HOS_MEM_COMMITTED, r, 0x40000); + memset(scratch, 0x5A, 0x40000), fills = 0; hos_region_free(r); + /* ((Triforce, 11-22, MEM-13) zeroed through hos_guest_fill, which marks the whole range it zeroed) */ + CHECK(fills == 1 && fill_lo == r && fill_n == 0x40000 && !scratch[0] && !scratch[0x3FFFF], + "Linux's zeroing under an import is not hos_guest_fill's of the region: %d fills, %#llx +%#llx", fills, + (unsigned long long)fill_lo, (unsigned long long)fill_n); CHECK(gc_count(&census, GC_KEPT, r, r + 0x40000) == 0x40000 && census_zeroed == zeroed + 0x40000 && census_discarded == discarded, "Linux's zeroing under an import: %llu kept, %llu counted zeroed, %llu discarded", (unsigned long long)gc_count(&census, GC_KEPT, r, r + 0x40000), (unsigned long long)(census_zeroed - zeroed), @@ -481,7 +501,12 @@ static void test_region_free(void) { list[1].addr = r, list[1].size = 0x40000, nregions = 2, imported_now = 1, window_lo = r; hos_mem_note(HOS_MEM_COMMITTED, r, 0x40000); seen = 0; + memset(scratch, 0x5A, 0x40000), fills = 0; hos_region_free(r); + /* ((Triforce, 11-22, MEM-13) zeroed through hos_guest_fill, which marks the whole range it zeroed) */ + CHECK(fills == 1 && fill_lo == r && fill_n == 0x40000 && !scratch[0] && !scratch[0x3FFFF], + "Windows' zeroing under an import is not hos_guest_fill's of the region: %d fills, %#llx +%#llx", fills, + (unsigned long long)fill_lo, (unsigned long long)fill_n); CHECK(seen == 1 && seen_listed && !seen_locked && nregions == 1, "Windows' zeroing under an import: %d seen, %s, %s", seen, seen_listed ? "listed" : "no longer listed", seen_locked ? "under the lock" : "not under the lock"); CHECK(gc_count(&census, GC_KEPT, r, r + 0x40000) == 0x40000 && census_zeroed == zeroed2 + 0x40000 && census_discarded == discarded2, @@ -598,9 +623,17 @@ MUTATIONS = [ ("hos_region_free", "game = regions[i].game;", "game = 0;", "the game's own regions are not known as such: every free a discard"), ("hos_region_free", "hos_region_discard_needed(game && hos_mem_limit(),", "hos_region_discard_needed(game,", "a bare region freed without its discard with the limit off: the default path is no longer upstream's"), - ("discard_pages", " const int away = hos_mem_limit() ? hos_vcore_away() : 0;\n", " const int away = hos_vcore_away();\n", + # ((Triforce, OS-40) the line asks HOS_DISCARD_KEEP_CORE's rule now; not set, it is the limit's, as these two say) + ("discard_pages", " const int away = discard_core_call_away(discard_core(), hos_mem_limit() != 0) ? hos_vcore_away() : 0;\n", + " const int away = hos_vcore_away();\n", "the discard gives its core up with the limit off: the default path is no longer upstream's"), - ("discard_pages", " const int away = hos_mem_limit() ? hos_vcore_away() : 0;\n", " const int away = 0;\n", + ("discard_core", "return mode == DISCARD_CORE_GIVEN_UP || (mode == DISCARD_CORE_AS_BEFORE && limit_on);", + "return mode == DISCARD_CORE_GIVEN_UP || mode == DISCARD_CORE_AS_BEFORE || limit_on;", + "with the switch not set a discard gives its core up without the limit: the default is no longer as before"), + ("hos_region_free", " if (discard_core_region_away(discard_core())) away = hos_vcore_away();\n", " away = hos_vcore_away();\n", + "with the switch not set a freed region gives its core up around its discard: the default is no longer as before"), + ("discard_pages", " const int away = discard_core_call_away(discard_core(), hos_mem_limit() != 0) ? hos_vcore_away() : 0;\n", + " const int away = 0;\n", "the discard made with the thread's virtual core held: the threads of its core wait out the kernel's milliseconds"), ("discard_pages", " if (away) hos_vcore_back();\n", " (void)away;\n", "the core given up for a discard and never taken back"), ("discard_pages", " if (hi <= lo || keep) return;\n", " if (hi <= lo || keep) return;\n hos_mem_note(HOS_MEM_DISCARDED, lo, hi - lo);\n", @@ -649,7 +682,7 @@ def check_source(): touch = block(glue, "static LONG CALLBACK commit_on_touch(", "host/mkw_glue.c") need(touch, "hos_mem_note(HOS_MEM_TOUCHED", "a first touch is not noted") free_pages = block(mem, "HLE(_ZN2nn2os15FreeMemoryPagesEmm)", "host/hos_mem.c") - need(free_pages, "memset(GP(lo), 0, (size_t)(hi - lo)), hos_mem_note(HOS_MEM_ZEROED, lo, hi - lo)", + need(free_pages, "hos_guest_fill(lo, 0, hi - lo), hos_mem_note(HOS_MEM_ZEROED, lo, hi - lo)", # ((Triforce, 11-22, MEM-13) the helper's) "FreeMemoryPages does not note its pages zeroed under an import as kept") need(free_pages, "MEM_DECOMMIT), hos_mem_note(HOS_MEM_DECOMMITTED, lo, hi - lo)", "FreeMemoryPages does not note its decommit") need(block(mem, "HLE(_ZN2nn2os19AllocateMemoryPagesEmm)", "host/hos_mem.c"), "hos_mem_note(HOS_MEM_PAGES, a, n)", @@ -692,10 +725,15 @@ def main(): "hos_mem_commit_end": block(mem, "void hos_mem_commit_end(void *had, uint64_t addr, uint64_t size, int committed)", "host/hos_mem.c"), "pages_mark_used_locked": block(mem, "static void pages_mark_used_locked(", "host/hos_mem.c"), + # ((Triforce, OS-40) HOS_DISCARD_KEEP_CORE's rule, which discard_pages and hos_region_free ask: not set here, so + # everything below is as before; tests/test_region_free_vcore.py has its other two answers) + "discard_core": mem[mem.index("enum { DISCARD_CORE_AS_BEFORE, DISCARD_CORE_GIVEN_UP, DISCARD_CORE_KEPT };"): + mem.index("static void discard_pages(")], "discard_pages": block(mem, "static void discard_pages(", "host/hos_mem.c"), "hos_region_discard_needed": block(mem, "int hos_region_discard_needed(int game_region, int census_on, uint64_t game_pages)", "host/hos_mem.c"), "region_frees_bare": "static int64_t region_frees_bare;", + "region_frees_away": "static int64_t region_frees_away, region_back_longest_ns;", "hos_region_free": block(mem, "void hos_region_free(uint64_t addr)", "host/hos_mem.c")} if mem.count("static int64_t region_frees_bare;") != 1: raise SystemExit("FAIL: host/hos_mem.c's region_frees_bare not found once") diff --git a/tests/test_guest_holes.py b/tests/test_guest_holes.py new file mode 100644 index 0000000..a0eaee6 --- /dev/null +++ b/tests/test_guest_holes.py @@ -0,0 +1,227 @@ +"""The all-zero upload check reads only the guest memfd's data, not its holes (gpu/vk/guest_holes.h, vk_texture.cpp +upload_all_zero, RX_VK_ZERO_HOLES; no game/GPU). + +On Linux the guest window is a memfd's where /dev/udmabuf is usable; a page the game never wrote is a hole, which reads as zero +but becomes RAM when read through the shared mapping. The check read the whole storage (the never-written 288 MB shadow map +became RAM at each Hyrule Field arrival). + +upload_all_zero's Linux block is cut out of vk_texture.cpp unchanged and compiled with guest_holes.h, stand-ins for the host's +window and dlsym, and a scripted file for lseek's SEEK_DATA and SEEK_HOLE (so it runs on Windows too). Bytes the file calls holes +are set non-zero, so a read of any of them shows. Checked: all holes: nothing read, all zero; data past holes: only the data read, +a set byte there found; the data clipped at the storage's end; a storage outside the window, no memfd, a seek that fails and +RX_VK_ZERO_HOLES=0 each read the whole storage; past 1,024 extents the rest is read whole. share: every byte of the extents in +exactly one thread's part, the tail of a size not a multiple of 8 or of the threads too. The source: upload_all_zero asks +any_set with the extents, the block is Linux's alone, the log lines say how much was read, and the switch has its docs row. + + python -I -B tests/test_guest_holes.py [--cxx clang++] +""" +import argparse +import os +from pathlib import Path +import subprocess +import tempfile + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] + +FIXTURE = r'''#include +#include +#include +#include +#include +#include +#include +#ifndef SEEK_DATA +#define SEEK_DATA 3 +#define SEEK_HOLE 4 +#endif +#ifndef ENXIO +#define ENXIO 6 +#endif +// (the scripted file: its data as [lo, hi) offsets from its start, the rest holes, kFileSize long) +static std::vector> g_data; +static bool g_fail = false; +static int g_seeks = 0; +constexpr uint64_t kFileSize = 8ull << 20; +static int64_t fake_lseek(int fd, int64_t pos, int whence) { + g_seeks++; + if (fd != 7 || g_fail || pos < 0) return errno = EBADF, -1; + if (whence == SEEK_DATA) { + for (const auto& d : g_data) + if (d.second > (uint64_t)pos) return d.first > (uint64_t)pos ? (int64_t)d.first : pos; + return errno = ENXIO, -1; + } + if (whence == SEEK_HOLE) { + for (const auto& d : g_data) + if (d.first <= (uint64_t)pos && (uint64_t)pos < d.second) return (int64_t)d.second; + return pos; + } + return errno = EINVAL, -1; +} +#define lseek fake_lseek +#define GUEST_HOLES_TEST 1 +#include "guest_holes.h" +using namespace mkwvk; + +static uint8_t g_window[kFileSize]; +static int g_memfd = 7; +static int hos_posix_guest_memfd() { return g_memfd; } +#define RTLD_DEFAULT nullptr +static void* dlsym(void*, const char* name) { return !strcmp(name, "hos_posix_guest_memfd") ? (void*)&hos_posix_guest_memfd : nullptr; } +struct Host { uint8_t* (*guest_base)(void); uint64_t (*guest_size)(void); }; +static uint8_t* base_fn() { return g_window; } +static uint64_t size_fn() { return sizeof g_window; } +static const Host host_obj{base_fn, size_fn}; +static const Host* host = &host_obj; + +struct Result { bool found; uint64_t read; }; +static Result check(const uint8_t* p, uint64_t total) { + const uint32_t threads = 4; +@CUT@ + (void)threads; + return {guest_holes::any_set(p, extents, threads), read_bytes}; +} + +static int bad = 0; +static void expect(bool ok, const char* what) { + if (!ok) { std::printf("FAIL %s\n", what); bad++; } +} +// (every byte the file calls a hole set to 0xAB, its data 0) +static void fill() { + memset(g_window, 0xAB, sizeof g_window); + for (const auto& d : g_data) memset(g_window + d.first, 0, d.second - d.first); +} + +int main(int argc, char** argv) { + const char* c = argv[1]; + const uint64_t off = 1ull << 20, total = 4ull << 20; // (the storage: 4 MB at 1 MB into the window) + uint8_t* p = g_window + off; + if (!strcmp(c, "share")) { + static const uint64_t sizes[] = {1, 7, 4095, 4096, 4097, 65536 * 3 + 5, (1ull << 20) + 3}; + for (uint32_t n = 1; n <= 9; n++) + for (uint64_t sz : sizes) { + std::vector ex{{3, 3 + sz / 3}, {sz, sz + sz / 2 + 1}, {5 * sz, 5 * sz + sz - sz / 3 - sz / 2 - 1}}; + std::vector seen(6 * sz + 8, 0); + for (uint32_t t = 0; t < n; t++) + for (const auto& e : guest_holes::share(ex, t, n)) + for (uint64_t k = e.lo; k < e.hi; k++) seen[k]++; + for (const auto& e : ex) + for (uint64_t k = e.lo; k < e.hi; k++) + if (seen[k] != 1) { std::printf("FAIL share n=%u size=%llu byte %llu read %d times\n", n, (unsigned long long)sz, (unsigned long long)k, seen[k]); return 1; } + uint64_t all = 0; + for (int v : seen) all += v; + expect(all == guest_holes::bytes(ex), "share: nothing outside the extents"); + } + expect(guest_holes::share({}, 0, 4).empty(), "share: no extents, no part"); + expect(!guest_holes::any_set(p, {}, 4), "any_set: no extents, nothing set"); + } else if (!strcmp(c, "holes")) { // (all holes) + g_data.clear(); + fill(); + const Result r = check(p, total); + expect(!r.found && r.read == 0, "all holes: all zero, nothing read"); + } else if (!strcmp(c, "data")) { // (data past holes, a byte set in it) + g_data = {{off + 65536, off + 2 * 65536}, {off + (2ull << 20), off + (2ull << 20) + 12288}}; + fill(); + const Result r = check(p, total); + expect(!r.found, "data past holes, all zero: all zero"); + expect(r.read == 65536 + 12288, "data past holes: only the data read"); + g_window[off + (2ull << 20) + 12287] = 1; + expect(check(p, total).found, "data past holes: its last byte set found"); + } else if (!strcmp(c, "tail")) { // (data running past the storage's end, and before its start) + g_data = {{off - 4096, off + 4096}, {off + total - 4096, off + total + 65536}}; + fill(); + const Result r = check(p, total); + expect(!r.found && r.read == 8192, "data past either end: clipped to the storage"); + } else if (!strcmp(c, "whole")) { // (a storage outside the window, no memfd, a failing seek, the switch: the whole read) + g_data = {{off + 65536, off + 2 * 65536}}; + fill(); + const char* how = argv[2]; + const uint8_t* q = p; + uint8_t* outside = nullptr; + if (!strcmp(how, "outside")) q = outside = (uint8_t*)calloc(total, 1); + if (!strcmp(how, "nomemfd")) g_memfd = -1; + if (!strcmp(how, "fail")) g_fail = true; + if (outside) outside[total - 1] = 1; + const Result r = check(q, total); + expect(r.found && r.read == total, "the whole storage read, the set bytes found"); + if (!strcmp(how, "switch") || !strcmp(how, "nomemfd") || !strcmp(how, "outside")) expect(g_seeks == 0, "no seek made"); + free(outside); + } else if (!strcmp(c, "many")) { // (2,000 extents: the first 1,023, then the rest whole) + g_data.clear(); + for (uint64_t i = 0; i < 2000; i++) g_data.push_back({off + i * 2048, off + i * 2048 + 1024}); + fill(); + const Result r = check(p, total); + expect(r.found, "past 1,024 extents: the rest read whole (its holes' set bytes found)"); + expect(r.read == total - 1022 * 1024, "past 1,024 extents: 1,023 of them, then the rest"); + } else { + return 2; + } + if (bad) return 1; + std::printf("OK %s\n", c); + return 0; +} +''' + +CASES = [(["share"], {}), (["holes"], {}), (["data"], {}), (["tail"], {}), (["whole", "outside"], {}), (["whole", "nomemfd"], {}), + (["whole", "fail"], {}), (["whole", "switch"], {"RX_VK_ZERO_HOLES": "0"}), (["many"], {}), (["data"], {"RX_VK_ZERO_HOLES": "1"})] + + +def fail(message): + raise SystemExit("FAIL: " + message) + + +def cut(tex): + body = tex[tex.index("bool upload_all_zero(Texture& tex, uint64_t total, const Ranges* dirty, bool widened_format) {"):] + body = body[:body.index("\n}\n")] + start = body.index(" std::vector extents{{0, total}};") + end = body.index(" const uint64_t read_bytes = guest_holes::bytes(extents);") + block = body[start:end] + if "#ifdef __linux__" not in block or "#endif" not in block: + fail("upload_all_zero's look at the memfd is not Linux's alone") + if "guest_holes::any_set(p, extents, std::clamp(cpu_count() / 2, 1u, 8u));" not in body: + fail("upload_all_zero does not read the extents through any_set") + for line in ("is not all zero (checked in %.1f ms, %llu MB read)", "is all zero: cleared on the GPU (checked in %.1f ms, %llu MB read)"): + if line not in body: + fail("upload_all_zero's log lines do not say how much was read: " + line) + block = block.replace("#ifdef __linux__\n", "").replace("#endif\n", "") + return block + " const uint64_t read_bytes = guest_holes::bytes(extents);\n" + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--cxx", default="g++") + a = p.parse_args() + tex = (ROOT / "gpu/vk/vk_texture.cpp").read_text(encoding="utf-8") + dev = (ROOT / "docs/development.md").read_text(encoding="utf-8") + try: + text = FIXTURE.replace("@CUT@", cut(tex)) + except ValueError as e: + fail("a text this test reads moved, or is not there (%s)" % e) + if "`RX_VK_ZERO_HOLES=0`" not in dev: + fail("docs/development.md has no RX_VK_ZERO_HOLES row") + flags = subprocess.BELOW_NORMAL_PRIORITY_CLASS if os.name == "nt" else 0 + problems = [] + with tempfile.TemporaryDirectory(prefix="guest-holes-") as tmp: + cpp = Path(tmp) / "fixture.cpp" + exe = Path(tmp) / ("fixture" + (".exe" if os.name == "nt" else "")) + cpp.write_text(text, encoding="utf-8") + built = subprocess.run([a.cxx, "-std=c++20", "-O1", "-pthread", "-D_FILE_OFFSET_BITS=64", "-Wall", "-Wno-unused-function", + "-I" + str(ROOT / "gpu/vk"), str(cpp), "-o", str(exe)], capture_output=True, text=True, creationflags=flags) + if built.returncode: + print(built.stdout + built.stderr) + fail("the fixture did not build") + base = {k: v for k, v in os.environ.items() if k != "RX_VK_ZERO_HOLES"} + for args, extra in CASES: + done = subprocess.run([str(exe)] + args, env={**base, **extra}, capture_output=True, text=True, timeout=60, + creationflags=flags) + if done.returncode or done.stdout.strip() != "OK " + args[0]: + problems.append("%s %s: %s" % (" ".join(args), extra, (done.stdout + done.stderr).strip())) + if problems: + fail("\n ".join([""] + problems)) + print("PASS: the all-zero check reads the memfd's data only, the whole storage otherwise or with RX_VK_ZERO_HOLES=0; every byte " + "in one thread's part (%d cases)" % len(CASES)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_heap_commit_lazy.py b/tests/test_heap_commit_lazy.py new file mode 100644 index 0000000..8f06000 --- /dev/null +++ b/tests/test_heap_commit_lazy.py @@ -0,0 +1,156 @@ +"""The Windows heap is committed as it is touched, with the three guards that needs (host/heap_commit.h, mkw_glue.c, +hos_mem.c, nn_fs.c; no game). + +(Triforce, 11-21, the performance audit's MEM-09, after Source A's.) The game asks for a ~6.9 GB heap and touches a +part of it; committed whole at its start, all of it counted against RAM plus the page file. Left reserved, the +first-touch handler commits it 64 KB at a touch. That needs, first: a fault whose page another thread has committed +meanwhile tried again; a commit refused at the commit limit waited for; and guest memory committed before a file read +or write hands it to the kernel, which faults nothing in. + +This compiles heap_commit.h alone: the heap whole on Linux, in large pages and with HOS_HEAP_COMMIT=1, and not +otherwise; a fault tried again only when committed, read-write and no execute fault; a touch's pages the reserved run +within its 64 KB; which touches get a log line. + +And reads the source: SetMemoryHeapSize commits by the rule; commit_on_touch tries again before it looks for reserved +pages, on Windows only, commits only the run, waits at the limit and notes the touch; hos_commit waits at the limit +inside its bracket; the file read and write read each page of their buffers once before the kernel's call, without +VirtualQuery; the scanner knows commit_wait; the switch is in the switches table. + + python -I -B tests/test_heap_commit_lazy.py [--cc clang] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +HOST = ROOT / "host" +RUN_ALL_ARGS = ["--cc", "clang"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, head): + """The function that starts at head, to its closing brace at the line's start""" + at = text.index(head) + return text[at:text.index("\n}\n", at)] + + +def in_order(text, parts, why): + at = [text.find(p) for p in parts] + check(all(a >= 0 for a in at), why + " (missing: " + repr(parts[[a >= 0 for a in at].index(False)]) + ")" if not all(a >= 0 for a in at) else why) + check(at == sorted(at), why) + + +PROGRAM = r""" +#include +#include +#include "heap_commit.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +int main(void) { + /* whole: Linux, large pages, the switch */ + CHECK(!heap_commit_whole(NULL, 0, 1) && !heap_commit_whole("", 0, 1) && !heap_commit_whole("0", 0, 1)); + CHECK(heap_commit_whole("1", 0, 1) && heap_commit_whole("2", 0, 1)); + CHECK(heap_commit_whole(NULL, 1, 1) && heap_commit_whole("0", 1, 1)); + CHECK(heap_commit_whole(NULL, 0, 0) && heap_commit_whole("0", 0, 0)); + /* tried again */ + CHECK(heap_touch_again(1, 1, 0) && heap_touch_again(1, 1, 1)); /* (a read, a write) */ + CHECK(!heap_touch_again(1, 1, 8)); /* (an execute fault: the crash report) */ + CHECK(!heap_touch_again(0, 1, 1) && !heap_touch_again(1, 0, 1)); /* (reserved; protected: another handler's) */ + /* the pages of a touch */ + uint64_t lo, hi; + heap_touch_range(0x12345678, 0x10000000, 0x20000000, &lo, &hi); /* (a wide reserved run: the whole 64 KB) */ + CHECK(lo == 0x12340000 && hi == 0x12350000); + heap_touch_range(0x12345678, 0x12343000, 0x12347000, &lo, &hi); /* (committed pages on both sides: the run alone) */ + CHECK(lo == 0x12343000 && hi == 0x12347000); + heap_touch_range(0x12345678, 0x12345000, 0x20000000, &lo, &hi); /* (committed below) */ + CHECK(lo == 0x12345000 && hi == 0x12350000); + heap_touch_range(0x12345678, 0x10000000, 0x12346000, &lo, &hi); /* (committed above) */ + CHECK(lo == 0x12340000 && hi == 0x12346000); + /* the log's lines */ + CHECK(heap_touch_said(1) && !heap_touch_said(2) && !heap_touch_said(1024) && heap_touch_said(4096) && !heap_touch_said(4097)); + CHECK(heap_touch_said(8192) && heap_touch_said(65536) && !heap_touch_said(12288)); + return fails ? 1 : 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cc", default="clang") + args = parser.parse_args() + glue = (HOST / "mkw_glue.c").read_text(encoding="utf-8") + mem = (HOST / "hos_mem.c").read_text(encoding="utf-8") + fs = (HOST / "nn_fs.c").read_text(encoding="utf-8") + + heap = block(mem, "HLE(_ZN2nn2os17SetMemoryHeapSizeEm)") + in_order(heap, ['const int whole = heap_commit_whole(getenv("HOS_HEAP_COMMIT"), hos_large_hi != 0, windows);', + "if (size > hos_heap_size && whole) hos_commit(HOS_HEAP_BASE + hos_heap_size, size - hos_heap_size);", + "hos_heap_size = size;"], "SetMemoryHeapSize does not commit the heap by heap_commit.h's rule") + check(heap.count("hos_commit(") == 1, "SetMemoryHeapSize commits the heap somewhere else too") + check("#if defined(_WIN32)\n const int windows = 1;\n#else\n const int windows = 0;" in heap.replace("\r\n", "\n"), + "the heap is left reserved on Linux too, or committed whole on Windows") + + touch = block(glue, "static LONG CALLBACK commit_on_touch(") + in_order(touch, ["if (!VirtualQuery(p, &mbi, sizeof mbi)) return EXCEPTION_CONTINUE_SEARCH;", "#ifdef _WIN32", + "if (heap_touch_again(mbi.State == MEM_COMMIT, mbi.Protect == PAGE_READWRITE, rec->ExceptionInformation[0]))", + "return EXCEPTION_CONTINUE_EXECUTION;", "#endif", + "if (mbi.State != MEM_RESERVE) return EXCEPTION_CONTINUE_SEARCH;", + "heap_touch_range(guest, run, run + mbi.RegionSize, &lo, &hi);", + "if (!VirtualAlloc(GP(lo), (SIZE_T)(hi - lo), MEM_COMMIT, PAGE_READWRITE) &&", + "!(GetLastError() == ERROR_COMMITMENT_LIMIT && commit_wait(GP(lo), (SIZE_T)(hi - lo))))", + "hos_mem_note(HOS_MEM_TOUCHED, lo, hi - lo);"], + "commit_on_touch: a fault another thread's commit answered is not tried again (on Windows only, before the look for " + "reserved pages), or it commits more than the reserved run, does not wait at the commit limit, or does not note the touch") + check("0x10000, MEM_COMMIT" not in touch, "commit_on_touch commits its whole 64 KB, over pages committed already") + in_order(touch, ["const int heap = guest >= HOS_HEAP_BASE && guest - HOS_HEAP_BASE < hos_heap_size;", + "if (!heap && InterlockedIncrement(&touched) <= 16)", "InterlockedIncrement(&heap_touched);"], + "the heap's first touches are among the 16 lines for pages nothing handed out, or not counted") + + wait = block(glue, "static int commit_wait(void *p, SIZE_T size) {") + in_order(wait, ["const int tries = host_alloc_tries(hos_commit_wait_ms());", "for (int i = 0; i < tries; i++) {", "Sleep(50);", "if (VirtualAlloc(p, size, MEM_COMMIT, PAGE_READWRITE)) {", + "return 1;", "if (GetLastError() != ERROR_COMMITMENT_LIMIT) return 0;", "return 0;"], + "commit_wait does not try a refused commit again for 5 s, or goes on after another error") + commit = block(glue, "int hos_commit(uint64_t addr, uint64_t size) {") + in_order(commit, ["const int io = hos_guest_io_begin(GP(lo), hi - lo);", "VirtualAlloc(GP(lo), (SIZE_T)(hi - lo), MEM_COMMIT, PAGE_READWRITE)", + "commit_err == ERROR_COMMITMENT_LIMIT && commit_wait(GP(lo), (SIZE_T)(hi - lo))", "hos_guest_io_end(io);", + "hos_mem_commit_end(census_had, lo, hi - lo, committed);"], + "hos_commit does not wait at the commit limit inside its bracket, before it tells the census") + touched = block(glue, "void hos_guest_touch(uint64_t addr, uint64_t size) {") + in_order(touched, ["if (!hos_heap_on_touch || !size || !hos_guest_range(addr, size)) return;", + "for (uint64_t a = addr; a < end; a = (a | 0xFFF) + 1) (void)*(volatile const uint8_t *)GP(a);"], + "hos_guest_touch does not read each page of a guest range once, or does with the heap committed whole") + check("VirtualQuery" not in touched, "hos_guest_touch asks VirtualQuery: it walks every page of a committed region, GBs of them, " + "in each file read") + check("hos_heap_on_touch = 1;" in heap and heap.index("if (size > hos_heap_size && !whole) {") < heap.index("hos_heap_on_touch = 1;"), + "SetMemoryHeapSize does not say the heap is committed as it is touched: the kernel calls' buffers are not touched") + + read = block(fs, "static uint64_t read_at(") + in_order(read, ["const int away = hos_vcore_away();", "hos_guest_touch(buf, size);", "const int io = hos_guest_io_begin(GP(buf), size);", + "ReadFile(fh->h, GP(buf + done)"], + "read_at does not touch its buffer before the kernel's read (with no virtual core held)") + write = block(fs, "HLE(_ZN2nn2fs9WriteFileENS0_10FileHandleElPKvmRKNS0_11WriteOptionE)") + in_order(write, ["hos_guest_touch(ARG(2), ARG(3));", "const int io = hos_guest_io_begin(GP(ARG(2)), ARG(3));", "WriteFile(fh->h, GP(ARG(2))"], + "WriteFile does not touch its buffer before the kernel's write") + check("void hos_guest_touch(uint64_t addr, uint64_t size);" in (HOST / "hos.h").read_text(encoding="utf-8"), + "hos.h does not declare hos_guest_touch") + check('("mkw_glue.c", "commit_wait", "VirtualAlloc")' in (ROOT / "tests" / "test_lazy_wb_host.py").read_text(encoding="utf-8"), + "tests/test_lazy_wb_host.py does not know commit_wait's commit") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`HOS_HEAP_COMMIT=1`" in dev, "docs/development.md: HOS_HEAP_COMMIT is not in the switches table") + + with tempfile.TemporaryDirectory(prefix="heap-commit-") as tmp: + c, exe = Path(tmp) / "heap.c", Path(tmp) / "heap.exe" + c.write_text(PROGRAM, encoding="utf-8") + subprocess.run([args.cc, "-std=c11", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(HOST), str(c), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + check(r.returncode == 0, "the rules as made:\n" + r.stdout) + print("PASS: the Windows heap is committed as it is touched, behind its three guards") + + +if __name__ == "__main__": + main() diff --git a/tests/test_helper_spin.py b/tests/test_helper_spin.py index 1a3891d..401033a 100644 --- a/tests/test_helper_spin.py +++ b/tests/test_helper_spin.py @@ -1,7 +1,7 @@ """The parallel replay's spins (the third performance audit's P6, after NX2navi's replay workers): a helper thread with no segment to record spins for the next before it sleeps on the pool's condition variable (gpu/vk/vk_replay.cpp par_worker_main), and the replay thread spins for a segment a helper records before it sleeps for it (replay_handle), both -bounded by RX_VK_HELPER_SPIN_US (100 us; 0: off, as before; gpu/vk/helper_spin.h). The spins only make a wait shorter: the +bounded by RX_VK_HELPER_SPIN_US (20 us since 11-21, 100 before; 0: off, as before; gpu/vk/helper_spin.h). The spins only make a wait shorter: the condition variable's wait with its predicate follows each as before. gpu/vk/helper_spin.h and vk_replay.cpp's ParPool and par_worker_main (cut out unchanged) run in tests/helper_spin_test.cpp: diff --git a/tests/test_host_free_marks.py b/tests/test_host_free_marks.py new file mode 100644 index 0000000..979a079 --- /dev/null +++ b/tests/test_host_free_marks.py @@ -0,0 +1,157 @@ +"""Freed guest pages zeroed under a Vulkan import go through hos_guest_fill, which can mark them written for the renderer +(host/hos.h hos_guest_fill, host/hos_mem.c hos_mem_zero_mark and its three sites, host/hos_svc.c's one; no game). + +(Triforce, 11-22, the performance audit's MEM-13, after Source A's.) Pages a Vulkan import holds are zeroed in place when +the game frees them, not discarded. A bare memset is invisible to the renderer (the soft write watch knows what guest +code and HOS_WRITTEN mark), so its mirror, captures and the write-back's shadow kept the freed bytes. hos_guest_fill +memsets and then, with HOS_MEM_ZERO_MARK=1, marks the range written. Off by default: the entry leaves the default to a +measurement at arrivals. + +This cuts hos_guest_fill out of hos.h and hos_mem_zero_mark out of hos_mem.c and runs them over stand-ins: + - HOS_MEM_ZERO_MARK unset, empty or 0: the bytes are filled and nothing is marked (a plain memset, as before); + - =1: the same bytes, and exactly the filled range is marked, once, AFTER the bytes are there (the mark's stand-in + reads the range: a capture takes the mark before it copies the page); + - the switch is read once however often the helper is called; bytes outside the range are not touched. + +And reads the source: the four sites (Windows' discard, Linux's discard, FreeMemoryPages, UnmapPhysicalMemory) call the +helper and keep their HOS_MEM_ZEROED note and their hos_guest_io bracket, none has a bare memset(GP( left, the helper +memsets before it marks, and the switches table has the row. + + python -I -B tests/test_host_free_marks.py [--cc clang] +""" +import argparse +import os +import re +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +RUN_ALL_ARGS = ["--cc", "clang"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker, what): + check(text.count(marker) == 1, "not found once in %s: %s" % (what, marker)) + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +FIXTURE = r""" +#include +#include +#include +#include + +static int fails; +#define CHECK(c) do { if (!(c)) { printf("FAIL: line %d: %s\n", __LINE__, #c); fails++; } } while (0) +static unsigned char guest[1 << 16]; +#define GP(a) ((void *)(guest + (a))) +/* (the mark: what it is given, and whether the fill's bytes were there already) */ +static int marks, mark_saw_fill; +static uint64_t mark_at, mark_n; +static int want_byte; +static void mark_written(const void *p, size_t n) { + marks++, mark_at = (uint64_t)((const unsigned char *)p - guest), mark_n = n, mark_saw_fill = 1; + for (size_t i = 0; i < n; i++) + if (((const unsigned char *)p)[i] != (unsigned char)want_byte) mark_saw_fill = 0; +} +#define HOS_WRITTEN(p, n) mark_written((p), (n)) +static int env_reads; +static const char *env_value; +static const char *fake_getenv(const char *name) { env_reads++; return strcmp(name, "HOS_MEM_ZERO_MARK") ? NULL : env_value; } +#define getenv fake_getenv +static int logged; +static void hos_log(const char *fmt, ...) { (void)fmt; logged++; } +int hos_mem_zero_mark(void); +@@HELPER@@ +@@SWITCH@@ + +int main(int argc, char **argv) { + env_value = argc > 1 && strcmp(argv[1], "unset") ? argv[1] : NULL; + const int on = argc > 2 && !strcmp(argv[2], "on"); + uint64_t state = 0x9E3779B97F4A7C15ULL; + for (int t = 0; t < 2000; t++) { + state ^= state << 13, state ^= state >> 7, state ^= state << 17; + const uint64_t a = 4096 + state % 40000, n = (state >> 20) % 9000; + const int v = t % 3 == 0 ? 0xA7 : 0; /* (the sites fill with 0; any byte is the helper's) */ + memset(guest, 0x5A, sizeof guest); + marks = 0, mark_saw_fill = 0, want_byte = v; + hos_guest_fill(a, v, n); + int ok = guest[a - 1] == 0x5A && guest[a + n] == 0x5A; /* (nothing outside the range) */ + for (uint64_t i = 0; i < n; i++) ok &= guest[a + i] == (unsigned char)v; + CHECK(ok); + if (on) CHECK(marks == 1 && mark_at == a && mark_n == n && mark_saw_fill); + else CHECK(marks == 0); + } + CHECK(env_reads == 1); /* (read once) */ + CHECK(logged == (on ? 1 : 0)); /* (and said once, when on) */ + printf("%s: 2000 fills, %s\n", argc > 1 ? argv[1] : "unset", on ? "each marked after its bytes" : "none marked"); + return fails != 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cc", default="clang") + args = parser.parse_args() + hos_h = (ROOT / "host/hos.h").read_text(encoding="utf-8") + mem = (ROOT / "host/hos_mem.c").read_text(encoding="utf-8") + svc = (ROOT / "host/hos_svc.c").read_text(encoding="utf-8") + + helper = block(hos_h, "static inline void hos_guest_fill(uint64_t a, int v, uint64_t n) {", "host/hos.h") + order = [helper.find(s) for s in ("memset(GP(a), v, (size_t)n);", "if (hos_mem_zero_mark()) HOS_WRITTEN(GP(a), (size_t)n);")] + check(min(order) >= 0 and order == sorted(order), "hos_guest_fill does not memset first and mark after, behind the switch") + check(hos_h.index("#define HOS_WRITTEN(p, n) ((void)0)") < hos_h.index("static inline void hos_guest_fill("), + "hos_guest_fill is defined before HOS_WRITTEN") + switch = block(mem, "int hos_mem_zero_mark(void) {", "host/hos_mem.c") + check('getenv("HOS_MEM_ZERO_MARK")' in switch and "static int on = -1;" in switch and "e && *e && atoi(e) != 0" in switch, + "HOS_MEM_ZERO_MARK is not off unless set to a number other than 0, or is read more than once") + + # the four sites + discard = block(mem, "static void discard_pages(", "host/hos_mem.c") + free_pages = block(mem, "HLE(_ZN2nn2os15FreeMemoryPagesEmm)", "host/hos_mem.c") + unmap = svc[svc.index("case 0x2D: /* UnmapPhysicalMemory"):] + unmap = unmap[:unmap.index("return;", unmap.index("c->x[0] = R_OK;"))] + win = "if (held) hos_guest_fill(lo, 0, hi - lo), hos_mem_note(HOS_MEM_ZEROED, lo, hi - lo);" + lin = "if (rx_guest_imported(lo, hi)) hos_guest_fill(lo, 0, hi - lo), hos_mem_note(HOS_MEM_ZEROED, lo, hi - lo);" + check(discard.count(win) == 1 and discard.index("hos_guest_io_begin(GP(lo), hi - lo);") < discard.index(win) < discard.index("hos_guest_io_end(io);"), + "Windows' discard does not zero imported pages through hos_guest_fill inside its hos_guest_io bracket") + check(discard.count(lin) == 1 and discard.index("#elif !defined(__EMSCRIPTEN__)") < discard.index(lin), "Linux's discard does not zero imported pages through hos_guest_fill") + fp = "if (hi > lo && rx_guest_imported(lo, hi)) hos_guest_fill(lo, 0, hi - lo), hos_mem_note(HOS_MEM_ZEROED, lo, hi - lo);" + check(free_pages.count(fp) == 1 and free_pages.index("hos_guest_io_begin(GP(lo), hi - lo)") < free_pages.index(fp) < free_pages.index("hos_guest_io_end(io);"), + "FreeMemoryPages does not zero imported pages through hos_guest_fill inside its hos_guest_io bracket") + um = "hos_guest_fill(c->x[0], 0, c->x[1]), hos_mem_note(HOS_MEM_ZEROED, c->x[0], c->x[1]);" + check(unmap.count(um) == 1 and unmap.index("hos_guest_io_begin(GP(c->x[0]), c->x[1]);") < unmap.index(um) < unmap.index("hos_guest_io_end(io);"), + "UnmapPhysicalMemory does not zero imported pages through hos_guest_fill inside its hos_guest_io bracket") + for name, text in (("discard_pages", discard), ("FreeMemoryPages", free_pages), ("UnmapPhysicalMemory", unmap)): + check(not re.search(r"memset\(GP\(", text), "%s still has a bare memset of guest memory" % name) + check(len(re.findall(r"hos_guest_fill\(", mem + svc)) == 4, "hos_guest_fill is not called at exactly the four sites") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`HOS_MEM_ZERO_MARK=1`" in dev, "docs/development.md: HOS_MEM_ZERO_MARK=1 is not in the switches table") + + with tempfile.TemporaryDirectory(prefix="host-free-marks-") as tmp: + src, exe = Path(tmp) / "marks.c", Path(tmp) / "marks.exe" + src.write_text(FIXTURE.replace("@@HELPER@@", helper).replace("@@SWITCH@@", switch), encoding="utf-8") + subprocess.run([args.cc, "-std=c11", "-O1", "-Wall", "-Wextra", "-Werror", str(src), "-o", str(exe)], check=True) + env = {k: v for k, v in os.environ.items() if k != "HOS_MEM_ZERO_MARK"} + for value, on in (("unset", False), ("", False), ("0", False), ("1", True), ("2", True), ("off", False)): + r = subprocess.run([str(exe), value, "on" if on else "off"], env=env, capture_output=True, text=True, timeout=120) + print(r.stdout.rstrip()) + check(r.returncode == 0, "HOS_MEM_ZERO_MARK %s: the fill's checks failed" % (value or "(empty)")) + print("PASS: freed pages zeroed under an import go through hos_guest_fill: a plain memset unless HOS_MEM_ZERO_MARK=1, " + "then marked written after the bytes") + + +if __name__ == "__main__": + main() diff --git a/tests/test_host_zstd.py b/tests/test_host_zstd.py index cce992b..afe3df9 100644 --- a/tests/test_host_zstd.py +++ b/tests/test_host_zstd.py @@ -554,6 +554,85 @@ int main(int argc, char **argv) { CHECK(r == 0xffffffffu && hz_stats.s_checked == sc0, "check: a stream left to the game was compared (%#llx)", (unsigned long long)r); ZSTD_freeDCtx(f2.d); + /* ====== ((Triforce, 11-21, the performance audit's IO-01) the content's own room, the canary, who gives the core up ====== */ + { + size_t LITN, LZN, LNN; + uint8_t *lit = load(argv[1], "lit.bin", &LITN), *LZ = load(argv[1], "lit.zst", &LZN), *LN = load(argv[1], "litnosize.zst", &LNN); + const uint64_t spare = 256u << 10; /* (room past the content, as the game's buffers have) */ + const uint64_t past = OUTA + ((LITN + 4095) & ~4095ull) + 4096; /* (a page wholly after the content) */ + const uint8_t *out = (const uint8_t *)HZ_PTR(OUTA); + size_t i; + /* one call: a frame of many literals into a room 256 KiB over its content. libzstd keeps a block's literals in + spare room after the block when it has any; given the content's own size it has none */ + put(COMPA, LZ, LZN); + memset(HZ_PTR(OUTA), 0xA5, LITN + spare); + clear_marks(); + r = once(OUTA, LITN + spare, COMPA, LZN, 0, 0, 0); + CHECK(r == LITN && !memcmp(out, lit, LITN), "literals, one call: %#llx or other bytes", (unsigned long long)r); + for (i = LITN; i < LITN + spare && out[i] == 0xA5; i++) {} + CHECK(i == LITN + spare, "literals, one call: byte %zu after the content was written (room %llu over it)", i - LITN, + (unsigned long long)spare); + CHECK(marked(OUTA, LITN) && !marked(past, 1), "literals, one call: the pages told to the write tracking are not the content's alone"); + /* the same frame without its content size: the caller's room is all libzstd can be given, and all of it is told */ + put(COMPA, LN, LNN); + memset(HZ_PTR(OUTA), 0xA5, LITN + spare); + clear_marks(); + r = once(OUTA, LITN + spare, COMPA, LNN, 0, 0, 0); + CHECK(r == LITN && !memcmp(out, lit, LITN), "literals without a content size: %#llx or other bytes", (unsigned long long)r); + CHECK(marked(OUTA, LITN + spare), "literals without a content size: the whole room was not told to the write tracking"); + /* a stream: the same frame, its state's room 256 KiB over the content */ + put(COMPA, LZ, LZN); + state(SA, OUTA, (uint32_t)(LITN + spare), WSA); + memset(HZ_PTR(OUTA), 0xA5, LITN + spare); + clear_marks(); + r = feed(SA, COMPA, (int32_t)LZN, 0); + CHECK(r == 0 && phase(SA) == 4 && !memcmp(out, lit, LITN), "literals, a stream: %#llx in phase %d or other bytes", + (unsigned long long)r, phase(SA)); + for (i = LITN; i < LITN + spare && out[i] == 0xA5; i++) {} + CHECK(i == LITN + spare, "literals, a stream: byte %zu after the content was written", i - LITN); + CHECK(marked(OUTA, LITN) && !marked(past, 1), "literals, a stream: the pages told to the write tracking are not the content's alone"); + /* check mode: the host's own output with the same room, and what it writes past the content counted */ + Fake1 l1 = {OUTA, LITN + spare, COMPA, LZN, NULL, 0, 0, 0, 0}; + uint64_t t0 = hz_stats.tail, e0 = hz_stats.differ; + r = hz_oneshot_check(OUTA, LITN + spare, COMPA, LZN, 0, 0, 0, fake_once, &l1); + CHECK(r == LITN && hz_stats.tail == t0 && hz_stats.differ == e0, "check: literals with a content size: %llu past the end, " + "%llu different", (unsigned long long)(hz_stats.tail - t0), (unsigned long long)(hz_stats.differ - e0)); + put(COMPA, LN, LNN); + Fake1 l2 = {OUTA, LITN + spare, COMPA, LNN, NULL, 0, 0, 0, 0}; + t0 = hz_stats.tail, e0 = hz_stats.differ; + r = hz_oneshot_check(OUTA, LITN + spare, COMPA, LNN, 0, 0, 0, fake_once, &l2); + CHECK(r == LITN && hz_stats.tail == t0 + 1 && hz_stats.differ == e0, "check: literals without a content size, room to spare: " + "%llu counted as past the end (want 1: what the exact room is for), %llu different", + (unsigned long long)(hz_stats.tail - t0), (unsigned long long)(hz_stats.differ - e0)); + + /* who gives the core up (HOS_HOST_ZSTD_AWAY): a whole buffer of 3 MB, then the same frame fed 64 KiB at a time */ + static const struct { int mode, whole, feeds; } want[] = {{1, 1, 0}, {0, 0, 0}, {2, 1, 1}}; + for (int k = 0; k < 3; k++) { + hz_away_mode = want[k].mode; + put(COMPA, A, AN); + aways = backs = 0; + r = once(OUTA, RAWN, COMPA, AN, 0, 0, 0); + CHECK(r == RAWN && aways == want[k].whole && backs == aways, "away %d: a whole buffer gave the core up %d times (%d back)", + want[k].mode, aways, backs); + aways = backs = 0; + state(SA, OUTA, (uint32_t)RAWN, WSA); + for (size_t off = 0; off < AN; off += 65536) r = feed(SA, COMPA + off, (int32_t)(AN - off < 65536 ? AN - off : 65536), 0); + CHECK(r == 0 && phase(SA) == 4 && (aways != 0) == want[k].feeds && backs == aways, + "away %d: a stream's feeds of 64 KiB gave the core up %d times (%d back; %#llx in phase %d)", want[k].mode, aways, + backs, (unsigned long long)r, phase(SA)); + } + hz_away_mode = 1; + /* (mode 1's line is the content's size where the frame says it, not the caller's room: 100 KB of content keeps the + core in a room of 1 MB as in one of its own size) */ + put(COMPA, S, SN); + aways = 0; + r = once(OUTA, 1 * MB, COMPA, SN, 0, 0, 0); + CHECK(r == SMALLN && aways == (SN >= HZ_AWAY_WHOLE), "away 1: 100 KB of content in a room of 1 MB gave the core up (%d)", aways); + aways = 0; + r = once(OUTA, SMALLN, COMPA, SN, 0, 0, 0); + CHECK(r == SMALLN && aways == (SN >= HZ_AWAY_WHOLE), "away 1: an input and a room under 128 KiB gave the core up (%d)", aways); + } + CHECK(logs > 0, "nothing was logged"); if (fails) printf("host zstd: FAILED (%d of %d checks)\n", fails, checks); else printf("host zstd: all %d checks passed\n", checks); @@ -691,11 +770,22 @@ def zstd_sources(): def check_source(): glue = (ROOT / "host/hos_zstd.c").read_text(encoding="utf-8") + header = (ROOT / "host/host_zstd.h").read_text(encoding="utf-8") build = (ROOT / "nx2/build.py").read_text(encoding="utf-8") docs = (ROOT / "docs/development.md").read_text(encoding="utf-8") for text, needle, why in ( (glue, 'getenv("HOS_HOST_ZSTD")', "host/hos_zstd.c does not read HOS_HOST_ZSTD"), - (glue, "v = e && *e ? atoi(e) : 0;\n v = v == 1 || v == 2 ? v : 0;", "the decoder is not off unless HOS_HOST_ZSTD is 1 or 2"), + # ((Triforce, 11-21, the performance audit's IO-01) on when not set; 2 the check; anything else off + (glue, "v = e && *e ? atoi(e) : 1;", "HOS_HOST_ZSTD not set does not mean the host's decoder"), + (glue, "v = v == 1 || v == 2 ? v : 0;", "the decoder is not off for a HOS_HOST_ZSTD that is neither 1 nor 2"), + (glue, 'const char *a = getenv("HOS_HOST_ZSTD_AWAY");', "host/hos_zstd.c does not read HOS_HOST_ZSTD_AWAY"), + (glue, "const int away = a && *a ? atoi(a) : 1;", "HOS_HOST_ZSTD_AWAY not set does not mean 1"), + (glue, "hz_away_mode = away == 0 || away == 2 ? away : 1;", "a HOS_HOST_ZSTD_AWAY that is neither 0 nor 2 does not mean 1"), + (header, "const uint64_t room = hz_room(cap, src, size, &exact);", "a whole buffer is not decoded into the content's own room"), + (header, "ZSTD_outBuffer out = {HZ_PTR(st->dst0), (size_t)st->content, (size_t)st->pos};", + "a stream is not decoded into the content's own room"), + (header, "const int away = hz_away_feed(in->size) ? HZ_AWAY() : 0;", "a stream's feed does not go by HOS_HOST_ZSTD_AWAY"), + (header, "const int away = hz_away_whole(size, cap) ? HZ_AWAY() : 0;", "a whole buffer does not go by HOS_HOST_ZSTD_AWAY"), (glue, 'strncmp(hos_main_build_id, "e0f3ccfa", 8)', "the game build the addresses are for is not checked"), (glue, "const int m = in_game ? 0 : mode();\n if (!m) return 0;", "a call does not answer 0 with the switch off, or inside the game's own function"), (glue, "#define GAME_MULTIFRAME 0x8010b380ULL", "the one-call function's address changed"), @@ -704,7 +794,7 @@ def check_source(): (glue, "#define HZ_OK(a, n) (hos_guest_range((a), (n)) != NULL)", "guest ranges are not checked against the game's memory"), (build, 'for f in HOST_ZSTD_SOURCES:\n jobs.append((cc, os.path.join(hzstd, *f.split("/")), os.path.join(obj_dir, "hostzstd_"', "nx2/build.py does not compile zstd's decoder for the runtime"), - (docs, "`HOS_HOST_ZSTD=1` or `2`", "docs/development.md does not name HOS_HOST_ZSTD")): + (docs, "`HOS_HOST_ZSTD=0` or `2`, `HOS_HOST_ZSTD_AWAY=0/1/2`", "docs/development.md does not name HOS_HOST_ZSTD and its core switch")): if needle not in text: raise SystemExit(why) for f in zstd_sources(): @@ -754,6 +844,13 @@ def frames(tmp): "tdicted.zst": zstandard.ZstdCompressor(level=3, dict_data=trained, write_dict_id=True).compress(traw), "traw.bin": traw, } + # ((Triforce, 11-21, IO-01) many literals and few matches: 16 symbols at random, which zstd packs as Huffman literals + # block after block. What libzstd does with spare room after such a block is what the content's own room is for) + lit = bytes(rng.choices(b"0123456789abcdef", k=700000)) + files["lit.bin"] = lit + files["lit.zst"] = plain.compress(lit) + files["litnosize.zst"] = zstandard.ZstdCompressor(level=3, write_content_size=False).compress(lit) + assert zstandard.get_frame_parameters(files["lit.zst"]).content_size == len(lit) assert zstandard.get_frame_parameters(files["tdicted.zst"]).dict_id == trained.dict_id() != 0 assert zstandard.get_frame_parameters(files["a.zst"]).content_size == len(raw) assert zstandard.get_frame_parameters(files["small.zst"]).has_checksum diff --git a/tests/test_import_budget.py b/tests/test_import_budget.py new file mode 100644 index 0000000..e1bd814 --- /dev/null +++ b/tests/test_import_budget.py @@ -0,0 +1,148 @@ +"""What the idle imports' valve measures the system heap against (gpu/vk/import_budget.h; no game, no GPU). + +(Triforce, 11-22, the performance audit's MEM-22, after Source A's.) The valve that gives idle imports back ran only +under a 12 GB system heap budget, or with RX_VK_IMPORT_HIGH set. With Mesa's RADV and a budget of 12 GB or more it now +runs against 4,500 MB; every other case is the rule as it was. + +Compiles a small driver against the header and checks, budgets in MB: + RADV 12,287 -> on, the whole budget (as before); 12,288 and 15,434 -> on, measured against 4,500; + with RX_VK_IMPORT_HIGH or _LOW set -> the whole budget, on only as before (HIGH set, or a small budget); + RX_VK_IMPORT_CAP_MB=0 -> no cap, off at 15,434 as before; + others NVIDIA, Intel, AMD's own drivers at 15,434 -> off, no cap; at 7,503 -> on, the whole budget; + exactly the rule as it was (on = HIGH set or under 12 GB), for every budget and every setting but the cap; + the cap RX_VK_IMPORT_CAP_MB=3000 -> on, 3,000 on any driver and budget; a cap over the budget measures the budget; + a pass against the cap, 1 GB at most; without a cap, all the way down to the target as before; nothing under it. + +And reads the source: imports_give_back asks the rule once with the driver, measures by_heap and the target against +import_budget::measured, bounds a capped pass with pass_want, keeps LOW at 75 and the idle filter of 600 submissions; +and the switches table documents RX_VK_IMPORT_HIGH, _LOW and _CAP_MB. + + python -I -B tests/test_import_budget.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +DRIVER = r''' +#include "import_budget.h" +#include +#include + +using namespace import_budget; +static int checks, fails; +static void want(const char* what, bool got, bool wanted) { + checks++; + if (got != wanted) fails++, printf("FAIL %s: %d, want %d\n", what, (int)got, (int)wanted); +} +static bool is(const Rule& r, uint64_t budget, bool on, uint64_t measure_mb) { + return r.on == on && measured(r, budget) == measure_mb * kMB; +} + +int main() { + const uint64_t under = 12287 * kMB, at = 12288 * kMB, big = 15434 * kMB, small = 7503 * kMB; + // RADV + want("RADV, 12,287 MB: the whole budget, on", is(rule(under, false, false, -1, true), under, true, 12287), true); + want("RADV, 12,288 MB: against 4,500", is(rule(at, false, false, -1, true), at, true, 4500), true); + want("RADV, 15,434 MB: against 4,500", is(rule(big, false, false, -1, true), big, true, 4500), true); + want("RADV, HIGH set: the whole budget, on", is(rule(big, true, false, -1, true), big, true, 15434), true); + want("RADV, LOW set alone: the whole budget, off as before", is(rule(big, false, true, -1, true), big, false, 15434), true); + want("RADV, the cap 0: no cap, off as before", is(rule(big, false, false, 0, true), big, false, 15434), true); + want("RADV, the cap 0 and HIGH set: on, the whole budget", is(rule(big, true, false, 0, true), big, true, 15434), true); + // every other driver: the rule as it was, whatever the budget and the percent switches + for (const uint64_t mb : {2048ull, 7503ull, 12287ull, 12288ull, 15434ull, 32768ull, 65536ull}) + for (int high = 0; high < 2; high++) + for (int low = 0; low < 2; low++) + for (const int64_t cap : {(int64_t)-1, (int64_t)0}) { + const uint64_t b = mb * kMB; + const Rule r = rule(b, high, low, cap, false); + want("another driver: on as before", r.on, high || b < kBigBudget); + want("another driver: no cap", r.cap == 0 && measured(r, b) == b, true); + want("another driver: a pass goes down to the target as before", pass_want(r, b, b, b / 100 * 75) == b - b / 100 * 75, true); + // and RADV is the same wherever its cap does not apply + if (cap == 0 || high || low || b < kBigBudget) { + const Rule a = rule(b, high, low, cap, true); + want("RADV outside its cap: as another driver", a.on == r.on && a.cap == r.cap, true); + } + } + want("another driver, 15,434 MB: off", is(rule(big, false, false, -1, false), big, false, 15434), true); + want("another driver, 7,503 MB: on, the whole budget", is(rule(small, false, false, -1, false), small, true, 7503), true); + // the switch's cap, any driver + for (int radv = 0; radv < 2; radv++) { + want("the cap 3000 at a big budget", is(rule(big, false, false, 3000, radv), big, true, 3000), true); + want("the cap 3000 at a small budget", is(rule(small, false, false, 3000, radv), small, true, 3000), true); + want("a cap over the budget measures the budget", is(rule(small, false, false, 9000, radv), small, true, 7503), true); + want("the cap over HIGH and LOW", is(rule(big, true, true, 3000, radv), big, true, 3000), true); + } + // a pass + const Rule capped = rule(big, false, false, -1, true), plain = rule(small, false, false, -1, false); + const uint64_t target = 4500 * kMB / 100 * 75; + want("capped: 1 GB at most", pass_want(capped, big, 9000 * kMB, target) == kPassMost, true); + want("capped: less than 1 GB as it is", pass_want(capped, big, target + 300 * kMB, target) == 300 * kMB, true); + want("capped: nothing under the target", pass_want(capped, big, target - kMB, target) == 0, true); + want("no cap: down to the target", pass_want(plain, small, 7400 * kMB, 5000 * kMB) == 2400 * kMB, true); + const Rule over = rule(small, false, false, 9000, false); // (a cap over the budget is no cap) + want("a cap over the budget: down to the target", pass_want(over, small, 7400 * kMB, 5000 * kMB) == 2400 * kMB, true); + want("the cap is 4,500 MB, the line 12 GB, a pass 1 GB", kCapMb == 4500 && kBigBudget == (12ull << 30) && kPassMost == (1ull << 30), true); + if (fails) printf("import budget: FAILED (%d of %d checks)\n", fails, checks); + else printf("import budget: all %d checks passed\n", checks); + return fails != 0; +} +''' + + +def check_source(): + src = (ROOT / "gpu/vk/vk_memory.cpp").read_text(encoding="utf-8") + if '#include "import_budget.h"' not in src: + raise SystemExit("vk_memory.cpp does not include import_budget.h") + at = src.index("void imports_give_back() {") + body = src[at:src.index("\n}\n", at)] + order = [body.find(n) for n in ( + "if (!system_heap_usage(&used, &budget)) return;", + "static import_budget::Rule live;", + "live = import_budget::rule(budget, env_high != nullptr, getenv(\"RX_VK_IMPORT_LOW\") != nullptr, env_cap ? atoll(env_cap) : -1,", + "dev.driver_id == VK_DRIVER_ID_MESA_RADV);", + "const uint64_t measure = import_budget::measured(live, budget);", + "const bool by_heap = small && used > measure / 100 * (uint64_t)high;", + "const uint64_t target = measure / 100 * (uint64_t)low;", + "const uint64_t want = by_heap ? import_budget::pass_want(live, budget, used, target) :", # (MEM-141: then the hold's episode) + "if (freed >= want) break;")] + if min(order) < 0 or order != sorted(order): + raise SystemExit("imports_give_back does not ask import_budget.h once with the driver and measure the heap, the target and " + "a pass by it") + for gone in ("used > budget / 100 * (uint64_t)high", "budget / 100 * (uint64_t)low", "env_high || budget < (12ull << 30)"): + if gone in body: + raise SystemExit("imports_give_back still measures against the whole budget itself: " + gone) + if 'getenv("RX_VK_IMPORT_CAP_MB")' not in body: + raise SystemExit("imports_give_back does not read RX_VK_IMPORT_CAP_MB") + if 'std::clamp(atoi(getenv("RX_VK_IMPORT_LOW")), 10, 95) : 75;' not in body or "constexpr uint64_t kImportIdle = 600;" not in src \ + or "const uint64_t idle_for = by_heap ? kImportIdle : kImportIdleRam;" not in body: + raise SystemExit("imports_give_back: LOW is no longer 75 by default, or the idle filter changed") + docs = (ROOT / "docs/development.md").read_text(encoding="utf-8") + for row in ("| `RX_VK_IMPORT_HIGH=` or `0`, `RX_VK_IMPORT_LOW=` |", "| `RX_VK_IMPORT_CAP_MB=` or `0` |", + "`gpu/vk/import_budget.h`"): + if row not in docs: + raise SystemExit("docs/development.md: the switches table lacks " + row) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="g++") + args = parser.parse_args() + check_source() + with tempfile.TemporaryDirectory(prefix="import-budget-") as tmp: + cpp, exe = Path(tmp) / "import_budget_test.cpp", Path(tmp) / "import-budget.exe" + cpp.write_text(DRIVER, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-Wall", "-Wextra", "-Werror", "-I" + str(ROOT / "gpu/vk"), str(cpp), "-o", str(exe)], + check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + print(r.stdout.rstrip()) + if r.returncode: + raise SystemExit("FAIL: the import budget's checks failed") + + +if __name__ == "__main__": + main() diff --git a/tests/test_import_chunk_default.py b/tests/test_import_chunk_default.py new file mode 100644 index 0000000..2c56b02 --- /dev/null +++ b/tests/test_import_chunk_default.py @@ -0,0 +1,143 @@ +"""An import's chunk is 4 MB where the mirrors are whole on an NVIDIA card, 16 MB everywhere else (vk_memory.cpp +small_chunks_wanted and import_chunk; no game, no GPU). + +(Triforce, 11-22, the performance audit's MEM-16, after Source A's.) An import covers whole 16 MB chunks of the game's +address space, and a whole mirror copies all of it into video memory, the padding too. Source A made the chunk 4 MB +wherever mirrors are on: about a quarter less mirror memory for about half again as many imports. Triforce's mirrors +in pieces bind memory only where bindings reach, so the gain is left where the mirrors are whole, and it is taken only +on NVIDIA's cards: an import stalled the replay 20 to 75 ms on an AMD card, so AMD and Intel keep 16 MB. + +This cuts the two functions out of vk_memory.cpp and runs them over stand-ins for mirror_on, mirror_pieces_on and the +device, a process a case (the chunk is read once): + - NVIDIA, mirrors on and whole: 4 MB; with the mirrors in pieces, or without mirrors: 16 MB; + - AMD and Intel with whole mirrors: 16 MB, and the mirrors' switches are not even asked there; + - RX_VK_IMPORT_CHUNK_MB=8, =16, =0 win on any card, and nothing is asked of the mirrors then; + - the chunk is logged once. +And reads the source: mirror_pieces_on is declared above the rule, the card's maker is asked first, the switch is in +the switches table, and tests/test_import_windows.py's fixture has the stand-in and runs a 4 MB chunk. + + python -I -B tests/test_import_chunk_default.py [--cxx clang++] +""" +import argparse +import os +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +FIXTURE = r''' +#include +#include +#include +#include +#include +#include + +static bool mirrors, pieces; +static int asked_mirrors, asked_pieces, logs; +static char last_log[512]; +static struct { struct { uint32_t vendorID = 0; } props; } dev; +static bool mirror_on() { asked_mirrors++; return mirrors; } +static bool mirror_pieces_on(); +static void logf(const char* fmt, ...) { + va_list ap; + va_start(ap, fmt); + std::vsnprintf(last_log, sizeof last_log, fmt, ap); + va_end(ap); + logs++; +} + +/* RULE */ +/* CHUNK */ + +static bool mirror_pieces_on() { asked_pieces++; return pieces; } + +int main(int argc, char** argv) { + if (argc < 4) return 2; + dev.props.vendorID = (uint32_t)std::strtoul(argv[1], nullptr, 16); + mirrors = argv[2][0] == '1', pieces = argv[3][0] == '1'; + const uint64_t a = import_chunk(), b = import_chunk(), c = import_chunk(); + if (a != b || b != c) return 3; + std::printf("%llu MB, %d line(s), mirrors asked %d, pieces asked %d\n%s\n", (unsigned long long)(a >> 20), logs, asked_mirrors, + asked_pieces, last_log); + return 0; +} +''' + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + mem = (ROOT / "gpu/vk/vk_memory.cpp").read_text(encoding="utf-8") + check("bool small_chunks_wanted() {" in mem, "vk_memory.cpp has no small_chunks_wanted()") + rule = block(mem, "bool small_chunks_wanted() {") + chunk = block(mem, "uint64_t import_chunk() {") + check("return dev.props.vendorID == 0x10de && mirror_on() && !mirror_pieces_on();" in rule, + "small_chunks_wanted is not: an NVIDIA card (asked first), mirrors on, and whole") + at = [mem.index(s) for s in ("bool mirror_pieces_on(); // (below", "bool small_chunks_wanted() {", "uint64_t import_chunk() {", + "Pool* pool_lookup(uint64_t guest, uint64_t size) {", "bool mirror_pieces_on() {\n static const bool on = [] {")] + check(at == sorted(at), "vk_memory.cpp: mirror_pieces_on is not declared above the rule, or the rule is not above import_chunk") + check('const char* e = getenv("RX_VK_IMPORT_CHUNK_MB");' in chunk and "const bool small = !e && small_chunks_wanted();" in chunk and + "const uint64_t mb = e ? (uint64_t)std::max(0, atoi(e)) : small ? 4 : 16;" in chunk and "static const uint64_t chunk = [] {" in chunk, + "import_chunk does not take RX_VK_IMPORT_CHUNK_MB first, then 4 MB where small chunks are wanted, else 16, once") + docs = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("| `RX_VK_IMPORT_CHUNK_MB=` |" in docs, "docs/development.md: RX_VK_IMPORT_CHUNK_MB is not in the switches table") + windows = (ROOT / "tests" / "test_import_windows.py").read_text(encoding="utf-8") + check("bool small_chunks_wanted()" in windows and "for chunk in ('16','4','3','0'):" in windows, + "tests/test_import_windows.py: the fixture lacks the rule's stand-in, or does not run a 4 MB chunk") + + env0 = {k: v for k, v in os.environ.items() if k != "RX_VK_IMPORT_CHUNK_MB"} + with tempfile.TemporaryDirectory(prefix="import-chunk-") as tmp: + cpp, exe = Path(tmp) / "chunk.cpp", Path(tmp) / "chunk.exe" + cpp.write_text(FIXTURE.replace("/* RULE */", rule).replace("/* CHUNK */", chunk), encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", str(cpp), "-o", str(exe)], check=True) + + def run(vendor, mirrors, pieces, env=None): + e = dict(env0) + if env is not None: + e["RX_VK_IMPORT_CHUNK_MB"] = env + r = subprocess.run([str(exe), vendor, "1" if mirrors else "0", "1" if pieces else "0"], capture_output=True, text=True, env=e) + check(r.returncode == 0, "the cut-out functions did not run (%s)" % r.returncode) + first, line = r.stdout.splitlines()[:2] + return first, line + + nv, amd, intel = "10de", "1002", "8086" + for vendor, mirrors, pieces, env, mb, asked, what in ( + (nv, True, False, None, 4, "mirrors asked 1, pieces asked 1", "an NVIDIA card with whole mirrors"), + (nv, True, True, None, 16, "mirrors asked 1, pieces asked 1", "an NVIDIA card with the mirrors in pieces"), + (nv, False, False, None, 16, "mirrors asked 1, pieces asked 0", "an NVIDIA card without mirrors"), + (nv, False, True, None, 16, "mirrors asked 1, pieces asked 0", "an NVIDIA card without mirrors"), + (amd, True, False, None, 16, "mirrors asked 0, pieces asked 0", "an AMD card with whole mirrors"), + (intel, True, False, None, 16, "mirrors asked 0, pieces asked 0", "an Intel card with whole mirrors"), + ("0", True, False, None, 16, "mirrors asked 0, pieces asked 0", "a card of no known maker"), + (nv, True, False, "8", 8, "mirrors asked 0, pieces asked 0", "RX_VK_IMPORT_CHUNK_MB=8 on an NVIDIA card with whole mirrors"), + (nv, True, False, "16", 16, "mirrors asked 0, pieces asked 0", "RX_VK_IMPORT_CHUNK_MB=16 on an NVIDIA card with whole mirrors"), + (nv, True, False, "0", 0, "mirrors asked 0, pieces asked 0", "RX_VK_IMPORT_CHUNK_MB=0"), + (nv, True, True, "-5", 0, "mirrors asked 0, pieces asked 0", "RX_VK_IMPORT_CHUNK_MB=-5"), + (amd, True, False, "4", 4, "mirrors asked 0, pieces asked 0", "RX_VK_IMPORT_CHUNK_MB=4 on an AMD card")): + first, line = run(vendor, mirrors, pieces, env) + check(first == "%d MB, 1 line(s), %s" % (mb, asked), "%s: %s, want %d MB said once and %s" % (what, first, mb, asked)) + check(line.startswith("vk: imports in chunks of %d MB" % mb), "%s: the log does not say the chunk: %s" % (what, line)) + print("PASS: 4 MB import chunks where the mirrors are whole on an NVIDIA card, 16 MB elsewhere; the switch wins") + + +if __name__ == "__main__": + main() diff --git a/tests/test_import_leaving.py b/tests/test_import_leaving.py new file mode 100644 index 0000000..8cf360c --- /dev/null +++ b/tests/test_import_leaving.py @@ -0,0 +1,168 @@ +"""The game's freed pages under an import given back and not freed yet (gpu/vk/vk_memory.cpp memory_guest_imported, +imports_leaving_overlap; no game, no GPU). + +The host asks the Vulkan back end, before it discards or decommits the game's freed pages (host/hos_mem.c discard_pages and +FreeMemoryPages, host/hos_svc.c UnmapPhysicalMemory), whether an import holds them: if one does they are zeroed and stay. The +answer counted the imports in the table, the retired pools still imported and the ranges under way on the background thread, +but not the imports already given back whose memory is freed only once the GPU is past them: an evicted pool +(imports_give_back) and a retired pool's import (retired_imports_give_back, dead_imports). The driver keeps their pages locked +until then, and a free in that window discarded pages under the lock: device lost (after Casual Gamer's #dev-fixes post: +v0.4.0 on a 16 GB laptop, 2 tours of 5 without the fix, 0 of 53 with it). The census's give-back (guest_release_unimported) +counted them already. + +Cuts memory_guest_imported and imports_leaving_overlap out of vk_memory.cpp and runs them against stand-ins: such imports +must answer "imported" over their pages, edges included, and not next to them; an evicted pool whose memory is gone is no +import's; the log line comes for the first 8 and every 64th; RX_VK_IMPORT_LEAVING=0 answers as before. With the new check +taken out, the fixture must fail. + + python -I -B tests/test_import_leaving.py [--cxx g++] +""" +import argparse +import os +import re +import subprocess +import tempfile +from pathlib import Path + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] + + +def block(text, marker): + if text.count(marker) != 1: + raise SystemExit("FAIL: not found once in vk_memory.cpp: " + marker) + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +FIXTURE = r""" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +typedef void* VkBuffer; +typedef void* VkDeviceMemory; +struct Pool { + uint64_t lo = 0, hi = 0; + VkDeviceMemory memory = nullptr; +}; +std::mutex pools_mu, bg_mu; +std::map imported; +std::deque::node_type> retired_pools, evicted_pools; +std::vector> bg_held; +@@DEAD_IMPORT@@ +std::deque dead_imports; +int logged = 0; +void logf(const char* f, ...) { logged += std::strstr(f, "still being given back") != nullptr; } +@@LEAVING@@ +@@IMPORTED@@ +int fails = 0; +#define CHECK(c, what) do { if (!(c)) { std::printf("FAIL: %s\n", what); fails++; } } while (0) +constexpr uint64_t MB = 1ull << 20; +const auto mem = (VkDeviceMemory)(uintptr_t)1; +std::map::node_type node(uint64_t lo, uint64_t hi, bool memory) { + std::map m; + m[lo] = Pool{lo, hi, memory ? mem : nullptr}; + return m.extract(lo); +} + +int main(int argc, char**) { + const bool off = argc > 1; // (RX_VK_IMPORT_LEAVING=0: as before, nothing of these counts) + const uint64_t a = 0x7F00000000ull, b = a + 16 * MB; + for (int kind = 0; kind < 2; kind++) { + evicted_pools.clear(), dead_imports.clear(); + if (kind == 0) evicted_pools.push_back(node(a, b, true)); + else { + DeadImport d{}; + d.memory = mem, d.bytes = b - a, d.lo = a, d.hi = b; + dead_imports.push_back(std::move(d)); + } + const char* name = kind ? "a retired pool's import given back, not freed yet" : "an evicted pool, not freed yet"; + char what[160]; + std::snprintf(what, sizeof what, "%s: its pages", name); + CHECK(memory_guest_imported(a + 4 * MB, a + 5 * MB) == !off, what); + std::snprintf(what, sizeof what, "%s: a range reaching its first page", name); + CHECK(memory_guest_imported(a - MB, a + 4096) == !off, what); + std::snprintf(what, sizeof what, "%s: a range reaching its last page", name); + CHECK(memory_guest_imported(b - 4096, b + MB) == !off, what); + std::snprintf(what, sizeof what, "%s: the pages just below and above it", name); + CHECK(!memory_guest_imported(a - 2 * MB, a) && !memory_guest_imported(b, b + MB), what); + } + evicted_pools.clear(), dead_imports.clear(); + evicted_pools.push_back(node(a, b, false)); + CHECK(!memory_guest_imported(a, b), "an evicted pool whose memory is freed: not imported"); + evicted_pools.clear(); + evicted_pools.push_back(node(a, b, true)); + logged = 0; + for (int i = 0; i < 200; i++) memory_guest_imported(a, a + MB); + if (!off) CHECK(logged == 8 + 3 - 6, "the line for the first 8 and every 64th (6 were told above)"); + else CHECK(logged == 0, "RX_VK_IMPORT_LEAVING=0: a line"); + if (fails) return 1; + std::printf(off ? "imports leaving: RX_VK_IMPORT_LEAVING=0 answers as before\n" + : "imports leaving: the pages of an import given back and not freed yet are zeroed, not discarded\n"); + return 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="g++") + args = parser.parse_args() + memory = (ROOT / "gpu/vk/vk_memory.cpp").read_text(encoding="utf-8") + dead = re.findall(r"^struct DeadImport \{[^\n]*\};$", memory, re.M) + if len(dead) != 1: + raise SystemExit("FAIL: vk_memory.cpp's struct DeadImport not found once") + imported = block(memory, "int memory_guest_imported(uint64_t lo, uint64_t hi) {") + leaving = block(memory, "bool imports_leaving_overlap(uint64_t lo, uint64_t hi) {") + release = block(memory, "int guest_release_unimported(uint64_t lo, uint64_t hi, int (*release)(uint64_t, uint64_t))") + if "evicted_pools" not in release or "dead_imports" not in release: + raise SystemExit("FAIL: guest_release_unimported no longer counts the imports given back and not freed yet") + flags = subprocess.CREATE_NO_WINDOW | subprocess.BELOW_NORMAL_PRIORITY_CLASS if os.name == "nt" else 0 + + def build(tmp, name, fn): + src, exe = Path(tmp) / (name + ".cpp"), Path(tmp) / (name + ".exe") + src.write_text(FIXTURE.replace("@@DEAD_IMPORT@@", dead[0]).replace("@@LEAVING@@", leaving).replace("@@IMPORTED@@", fn), + encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-Wno-unused-parameter", "-Wno-unused-variable", + str(src), "-o", str(exe)], check=True) + return exe + + with tempfile.TemporaryDirectory(prefix="import-leaving-") as tmp: + exe = build(tmp, "leaving", imported) + env = dict(os.environ) + env.pop("RX_VK_IMPORT_LEAVING", None) + ran = 0 + for argv, value in (([str(exe)], None), ([str(exe), "off"], "0")): + if value is not None: + env["RX_VK_IMPORT_LEAVING"] = value + r = subprocess.run(argv, env=env, capture_output=True, text=True, timeout=120, creationflags=flags) + print(r.stdout, end="") + ran = ran or r.returncode + if ran: + raise SystemExit("FAIL: the host's question about pages under an import still being given back") + cut = "leaving && imports_leaving_overlap(lo, hi)" + if imported.count(cut) != 1: + raise SystemExit("FAIL: memory_guest_imported no longer asks imports_leaving_overlap once") + env.pop("RX_VK_IMPORT_LEAVING", None) + m = subprocess.run([str(build(tmp, "mutant", imported.replace(cut, "false")))], env=env, capture_output=True, + creationflags=flags) + if m.returncode == 0: + raise SystemExit("FAIL: the fixture passes without the check of the imports given back") + print("PASS: without the check the fixture fails (the device lost's way)") + + +if __name__ == "__main__": + main() diff --git a/tests/test_import_windows.py b/tests/test_import_windows.py index a73ff25..14faaae 100644 --- a/tests/test_import_windows.py +++ b/tests/test_import_windows.py @@ -60,6 +60,7 @@ thread_local bool import_cooling = false; thread_local int import_result = 0; bool import_on_hold(uint64_t, uint64_t) { import_cooling = false; return false; } void import_remember_failure(uint64_t, uint64_t) {} +bool small_chunks_wanted() { return false; } // ((Triforce, 11-22, MEM-16) the chunk's default: tests/test_import_chunk_default.py) '''+guard+''' p.lo=lo;p.hi=hi;imports++;return true; } @@ -133,7 +134,7 @@ int main(int argc,char **argv) { with tempfile.TemporaryDirectory(prefix='import-windows-',dir=work) as temp: directory=Path(temp).resolve();directory.relative_to(work.resolve());cpp=directory/'fixture.cpp';exe=directory/'fixture.exe';cpp.write_text(fixture,encoding='utf-8') subprocess.run([a.cxx,'-std=c++20','-O2','-D_CRT_SECURE_NO_WARNINGS',str(cpp),'-o',str(exe)],check=True) - for chunk in ('16','3','0'): + for chunk in ('16','4','3','0'): subprocess.run([str(exe)],env=dict(os.environ,RX_VK_IMPORT_CHUNK_MB=chunk,RX_VK_PREFETCH_CHUNKS='0'),check=True) for mode in ('1','0','2'): subprocess.run([str(exe),mode],env=dict(os.environ,RX_VK_IMPORT_CHUNK_MB='16',RX_VK_PREFETCH_CHUNKS='0' if mode=='0' else '1'),check=True) diff --git a/tests/test_index_rebase.py b/tests/test_index_rebase.py new file mode 100644 index 0000000..3754321 --- /dev/null +++ b/tests/test_index_rebase.py @@ -0,0 +1,129 @@ +"""An indexed draw's index buffer is bound at offset 0, with the draw's offset as its first index (gpu/vk/ +index_rebase.h, vk_draw.cpp draw, bind_index and index_bind_same; no game, no GPU). + +(Triforce, 11-21, the performance audit's DESC-12, after Source A's.) The game's draws share one buffer at different +offsets, so the skip of an index buffer already on the command buffer rarely held: nearly every indexed draw called +vkCmdBindIndexBuffer. + +This compiles index_rebase.h with index_bind_same cut from the source and replays a frame's binds through them: 16-bit +draws at 0x100 and 0x2c0 of one buffer bind once, with first indices 0x80 and 0x160; an offset inside an index (3, +16-bit) and one past what a first index holds (0x200000000, 32-bit) fall back to their own offset; another buffer, or +another index type, binds again; with the switch off every offset binds, as before. + +And reads the source: the direct indexed draw asks the rule, binds at 0 when it holds and passes the first index to +vkCmdDrawIndexed; the indirect indexed draw still binds at its own offset; the switch is read by the rule and is in the +switches table; the binds made are counted. + + python -I -B tests/test_index_rebase.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +PROGRAM = r""" +#include +#include +#include "index_rebase.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +INDEX_BIND_SAME +// the draw's own lines, as vk_draw.cpp has them: what is bound, and the first index the draw is given +struct Bound { uint64_t buffer = 0, offset = 0; uint32_t type = 0; int on = 0; } last; +static int binds; +static uint32_t draw(bool rebase_on, uint64_t buffer, uint64_t offset, uint32_t unit) { + const uint32_t type = unit == 1 ? 2 : unit == 2 ? 0 : 1; + uint32_t first = 0; + const bool rebased = rebase_on && index_rebase::rebase(offset, unit, &first); + const uint64_t at = rebased ? 0 : offset; + if (!index_bind_same(buffer, at, type, last.buffer, last.offset, last.type, last.on)) binds++, last = {buffer, at, type, 1}; + return first; +} +int main() { + CHECK(index_rebase::on(nullptr) && index_rebase::on("") && index_rebase::on("1") && !index_rebase::on("0")); + uint32_t f = 99; + CHECK(index_rebase::rebase(0x100, 2, &f) && f == 0x80); + CHECK(index_rebase::rebase(0x2c0, 2, &f) && f == 0x160); + CHECK(index_rebase::rebase(0, 4, &f) && f == 0); + CHECK(index_rebase::rebase(7, 1, &f) && f == 7); + f = 99; + CHECK(!index_rebase::rebase(3, 2, &f) && f == 99); // (inside an index) + CHECK(!index_rebase::rebase(0x200000000ull, 4, &f) && f == 99); // (past what a first index holds) + CHECK(index_rebase::rebase(0x7fffffffull * 4, 4, &f) && f == 0x7fffffffu); + CHECK(!index_rebase::rebase(0x80000000ull * 4, 4, &f)); + CHECK(!index_rebase::rebase(16, 0, &f)); + // a frame's draws from one buffer + CHECK(draw(true, 0xB0, 0x100, 2) == 0x80 && binds == 1); + CHECK(draw(true, 0xB0, 0x2c0, 2) == 0x160 && binds == 1); // (the same buffer at another offset: no bind) + CHECK(draw(true, 0xB0, 0x100, 2) == 0x80 && binds == 1); + CHECK(draw(true, 0xB0, 3, 2) == 0 && binds == 2 && last.offset == 3); // (falls back: at its own offset) + CHECK(draw(true, 0xB0, 0x40, 2) == 0x20 && binds == 3 && last.offset == 0); + CHECK(draw(true, 0xB0, 0x40, 4) == 0x10 && binds == 4); // (another index type: bound again) + CHECK(draw(true, 0xC0, 0x40, 4) == 0x10 && binds == 5); // (another buffer) + CHECK(draw(true, 0xC0, 0x200000000ull, 4) == 0 && binds == 6 && last.offset == 0x200000000ull); + // the switch off: every offset binds, first index 0 + last = {}, binds = 0; + CHECK(draw(false, 0xB0, 0x100, 2) == 0 && binds == 1); + CHECK(draw(false, 0xB0, 0x2c0, 2) == 0 && binds == 2); + CHECK(draw(false, 0xB0, 0x2c0, 2) == 0 && binds == 2); + return fails ? 1 : 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + src = (VK / "vk_draw.cpp").read_text(encoding="utf-8") + check('#include "index_rebase.h"' in src, "vk_draw.cpp does not use the rule") + draw = block(src, "void draw(") + direct = draw[draw.index("} else if (d.indexed) {"):] + direct = direct[:direct.index("} else {")] + check('static const bool rebase_on = index_rebase::on(getenv("RX_VK_INDEX_REBASE"));' in direct, "the switch is not the rule's") + check("const bool rebased = rebase_on && index_rebase::rebase((uint64_t)ib.offset, unit, &first_index);" in direct and + "bind_index(cmd, ib.buffer, rebased ? 0 : ib.offset, itype);" in direct, + "the direct indexed draw does not bind at 0 where the rule holds, and at its own offset where it does not") + check("vkCmdDrawIndexed(cmd, (uint32_t)d.count, (uint32_t)std::max(1, d.instances), first_index, d.base_vertex," in direct, + "the draw is not given its first index") + check(direct.index("uint32_t first_index = 0;") < direct.index("index_rebase::rebase("), "the first index is not 0 unless the rule sets it") + indirect = draw[draw.index("if (d.indirect) {"):draw.index("} else if (d.indexed) {")] + check("bind_index(cmd, ib.buffer, ib.offset, itype);" in indirect and "index_rebase" not in indirect, + "an indirect indexed draw, whose first index is in the GPU's records, no longer binds at its own offset") + bind = block(src, "void bind_index(") + check(bind.index("index_bind_same(") < bind.index("FINE_N(FC_IBIND_MADE, 1);") < bind.index("vkCmdBindIndexBuffer("), + "the binds made are not counted after the skip") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`RX_VK_INDEX_REBASE=0`" in dev, "docs/development.md: RX_VK_INDEX_REBASE=0 is not in the switches table") + + with tempfile.TemporaryDirectory(prefix="index-rebase-") as tmp: + c, exe = Path(tmp) / "rebase.cpp", Path(tmp) / "rebase.exe" + c.write_text(PROGRAM.replace("INDEX_BIND_SAME", block(src, "bool index_bind_same(")), encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(VK), str(c), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + check(r.returncode == 0, "the rule as made:\n" + r.stdout) + print("PASS: draws at different offsets of one index buffer bind it once, at 0, each with its first index") + + +if __name__ == "__main__": + main() diff --git a/tests/test_input_background.py b/tests/test_input_background.py index d3e4373..0527d5d 100644 --- a/tests/test_input_background.py +++ b/tests/test_input_background.py @@ -107,6 +107,9 @@ static void host_drag_read(int32_t stick[4]) { (void)stick; } static void player_used(void) {} void hos_amiibo_poll(void) {} int hos_oot_mods_down(void) { return 0; } /* ((Triforce) host/oot_menu.c: no game list's last button has the cursor here) */ +static void device_used(int device) { (void)device; } /* ((Triforce) the device used last, for the button prompts: not looked at */ +static void mods_input(uint64_t *buttons, int32_t stick[4]) { (void)buttons, (void)stick; } /* (and no Mods entry's input) */ +int hos_bench_input_only(void) { return 0; } /* ((Triforce) host/hos_bench.c: benchmark mode off */ """ actual = "\n".join((cheat_state, cheat_spies, host_spies, block(hid, "static int key("), block(hid, "static int kmap_down("), block(hid, "static int game_in_front("), hid[start:end], block(hid, "static void stick_max("), diff --git a/tests/test_invalidate_quick.py b/tests/test_invalidate_quick.py new file mode 100644 index 0000000..6de2779 --- /dev/null +++ b/tests/test_invalidate_quick.py @@ -0,0 +1,374 @@ +"""An invalidate with nothing to wait for is not made, and its thread keeps its virtual core: RX_VK_INVALIDATE_QUICK=1 +(gpu/vk/page_marks.h, vk_replay.cpp invalidate_quick, host/rx_backend.c rx_nvn_invalidate; no game, no GPU). + +(Triforce, 11-22, the performance audit's THREAD-25, after Source A's.) Off unless set. The wait at stake is the one a +readback of GPU results needs: the thread that reads is the waiter, the write-back's registration is what it would wait +for, and "nothing to wait for" may only be said when no registration for the range was ever made. + +- PageMarks compiled and run: single pages, ranges that end inside a page, the edges of a word, of a chunk and of 512 GB + (past which nothing is kept and everything answers as marked), and 20,000 random marks and asks against a plain set + of pages; +- the two tables beside their marks, cut out of the sources (vk_memory.cpp's WbPages, wb_pages_note, + memory_writeback_value and memory_writeback_marked; vk_texture.cpp's vw_note, texture_gpu_written_in and + texture_gpu_written_marked) over random notes: wherever a table says "ever" or "written in", the marks say marked, + and a range no note touched is not marked; +- on threads: one thread notes write-backs while four ask. A note that has returned is seen by every ask that begins + after it (nothing registered is missed), and whenever the table itself, asked under its lock, says "ever", the marks + asked after it say marked (the mark is made first); +- the rule (inv_quick::nothing_to_wait_for) over every case, and the switch: off unless RX_VK_INVALIDATE_QUICK is a + number other than 0, and off with a switch that makes invalidate() log or wait at every call; +- the sources: the mark is made before the table's entry and under its lock, wb_pages has one writer, a texture is + listed in vw only by vw_note; invalidate_quick reads the ticket before the sections left and asks the three things + the rule names; invalidate() still returns without waiting exactly where the rule says it does (its ticket wait first, + then "ever written back", then "no texture written there, no texture storage there": a new wait added to it must be + added to the rule, and this check says so); the host asks before it gives the core up, only a Vulkan back end, found + by name; the switch is in the switches table. +tests/test_invalidate_vcore.py runs the host's wrapper with the question answered yes (no release, no take, no call) and no. + + python -I -B tests/test_invalidate_quick.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" + + +def fail(message): + raise SystemExit("FAIL: " + message) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +FIXTURE = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "page_marks.h" +static std::atomic fails{0}; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +namespace mkwvk { +// ---- vk_memory.cpp's table of write-backs, as it is ---- +constexpr uint64_t kPageShift = 12, kChunkPages = 65536, kPages = 2048 * kChunkPages; +static std::mutex wb_pages_mu; +/* WBPAGES */ wb_pages; +PageMarks wb_marks; +struct WbRun { uint64_t lo, hi, value; }; +static std::vector wb_runs_recent; +static uint64_t wb_runs_lost = 0; +static uint64_t vk_timeline_completed() { return ~0ull; } +/* WBNOTE */ +/* WBVALUE */ +/* WBMARKED */ +// ---- vk_texture.cpp's list of textures the GPU wrote, as it is ---- +struct Texture { uint64_t gpu = 0, guest_size = 0; bool vw_listed = false; }; +static bool flushable(const Texture& t) { return t.guest_size != 0; } +constexpr uint64_t kFlushMax = 16ull << 20; +static std::mutex vw_mu; +static std::map vw; +PageMarks vw_marks; +/* VWNOTE */ +/* VWIN */ +/* VWMARKED */ +} // namespace mkwvk +using namespace mkwvk; + +static uint64_t rnd_state = 0x9E3779B97F4A7C15ull; +static uint64_t rnd() { + rnd_state ^= rnd_state << 13, rnd_state ^= rnd_state >> 7, rnd_state ^= rnd_state << 17; + return rnd_state; +} +static const uint64_t GB = 1ull << 30, SPAN = PageMarks::kSpan; + +static void marks_alone() { + static PageMarks m; + CHECK(PageMarks::kSpan == (1ull << 39) && PageMarks::kPages == (1ull << 27) && PageMarks::kChunks * PageMarks::kChunkPages == PageMarks::kPages); + CHECK(!m.any(0, SPAN) && !m.any(0x1000, 1) && !m.any(0, 0)); + m.mark(0x5000, 0x5001); // one byte: its page + CHECK(m.any(0x5000, 1) && m.any(0x5fff, 1) && m.any(0x4fff, 2) && m.any(0x5fff, 2) && m.any(0x4000, 0x3000)); + CHECK(!m.any(0x4000, 0x1000) && !m.any(0x6000, 0x1000) && !m.any(0x4fff, 1) && !m.any(0x6000, 1)); + CHECK(!m.any(0x5000, 0)); // (nothing to invalidate: nothing marked) + m.mark(0x7000, 0x7000), m.mark(0x9000, 0x8000); // empty and backwards: nothing + CHECK(!m.any(0x7000, 0x3000)); + m.mark(0x10000 - 1, 0x10000 + 1); // two bytes across a page edge: both pages + CHECK(m.any(0xf000, 1) && m.any(0x10000, 1) && !m.any(0xe000, 0x1000) && !m.any(0x11000, 0x1000)); + // the edge of a word (64 pages) and of a chunk (1 GB) + const uint64_t w = 64 * 0x1000 * 5; + m.mark(w - 0x1000, w + 0x1000); + CHECK(m.any(w - 0x1000, 1) && m.any(w, 1) && !m.any(w - 0x2000, 0x1000) && !m.any(w + 0x1000, 0x1000)); + m.mark(3 * GB - 0x2000, 3 * GB + 0x2000); // across a chunk's edge + CHECK(m.any(3 * GB - 1, 1) && m.any(3 * GB, 1) && m.any(3 * GB - 0x2000, 1) && m.any(3 * GB + 0x1fff, 1)); + CHECK(!m.any(3 * GB - 0x3000, 0x1000) && !m.any(3 * GB + 0x2000, 0x1000)); + CHECK(m.chunks[2].load() && m.chunks[3].load() && !m.chunks[1].load() && !m.chunks[4].load()); // (made where marked only) + m.mark(7 * GB, 9 * GB); // two whole chunks + CHECK(m.any(7 * GB, 1) && m.any(9 * GB - 1, 1) && m.any(8 * GB + 12345, 77) && !m.any(9 * GB, 0x1000) && !m.any(7 * GB - 0x1000, 0x1000)); + CHECK(m.any(6 * GB, 2 * GB) && !m.any(5 * GB, GB) && m.any(0, SPAN)); // (asks that skip chunks never made) + // 512 GB: nothing kept at or past it, and every ask there answers as marked + CHECK(!m.any(SPAN - 0x1000, 0x1000)); + m.mark(SPAN - 0x800, SPAN + 0x100000); // (the part below is kept) + CHECK(m.any(SPAN - 0x1000, 0x1000) && m.any(SPAN - 1, 1)); + m.mark(SPAN, SPAN + 0x1000), m.mark(SPAN * 4, SPAN * 4 + 1); // (nothing kept, no fault) + CHECK(m.any(SPAN, 1) && m.any(SPAN - 1, 2) && m.any(SPAN * 3, 0x1000) && m.any(~0ull, 1) && m.any(0x1000, ~0ull)); + CHECK(!m.any(SPAN - 0x3000, 0x2000)); + + // random marks and asks against a set of pages (addresses in a few chunks, so pages meet) + static PageMarks r; + std::set pages; + auto where = [] { return (rnd() % 6) * 97 * GB / 16 + rnd() % (8ull << 20); }; + for (int i = 0; i < 20000; i++) { + const uint64_t lo = where(), size = rnd() % 3 ? rnd() % 0x3000 + 1 : rnd() % 0x80000 + 1; + if (i % 2) { + r.mark(lo, lo + size); + for (uint64_t p = lo >> 12; p <= (lo + size - 1) >> 12; p++) pages.insert(p); + } else { + const auto it = pages.lower_bound(lo >> 12); + const bool want = it != pages.end() && *it <= (lo + size - 1) >> 12; + if (r.any(lo, size) != want) { CHECK(!"a random ask disagrees with the pages marked"); break; } + } + } +} + +static void tables() { + // write-backs: the marks say marked wherever the table says ever, and not where no note reached + std::set noted; + auto where = [] { return 4 * GB + (rnd() % 3) * 5 * GB + rnd() % (4ull << 20); }; + for (int i = 0; i < 6000; i++) { + const uint64_t lo = where(), size = rnd() % 0x5000 + 1; + if (i % 3 == 0) { + wb_pages_note(lo, lo + size, 1 + rnd() % 1000); + for (uint64_t p = lo >> 12; p <= (lo + size - 1) >> 12; p++) noted.insert(p); + } else { + bool ever = false; + memory_writeback_value(lo, size, &ever); + const auto it = noted.lower_bound(lo >> 12); + const bool want = it != noted.end() && *it <= (lo + size - 1) >> 12; + if (ever != want || memory_writeback_marked(lo, size) != want) { CHECK(!"the write-backs' table and its marks disagree"); break; } + } + } + wb_pages_note(20 * GB, 20 * GB + 0x1000, 0); // (a note with no value is none: no mark either) + CHECK(!memory_writeback_marked(20 * GB, 0x1000)); + bool ever = true; + CHECK(memory_writeback_value(0, 64, &ever) == 0 && !ever && !memory_writeback_marked(0, 64)); // (an unmapped address: page 0) + // GPU-written textures: marked wherever a listed one lies; a texture listed twice, and one that is not flushable + static Texture t[400]; + std::set listed; + for (int i = 0; i < 400; i++) { + t[i].gpu = 2 * GB + (rnd() % 2) * 9 * GB + (rnd() % (2ull << 20) & ~0xfull), t[i].guest_size = i % 9 == 0 ? 0 : rnd() % 0x9000 + 16; + vw_note(t[i]), vw_note(t[i]); + if (t[i].guest_size) + for (uint64_t p = t[i].gpu >> 12; p <= (t[i].gpu + t[i].guest_size - 1) >> 12; p++) listed.insert(p); + CHECK(t[i].vw_listed == (t[i].guest_size != 0)); + } + for (int i = 0; i < 6000; i++) { + const uint64_t lo = 2 * GB + (rnd() % 2) * 9 * GB + rnd() % (2ull << 20) - 0x2000, size = rnd() % 0x4000 + 1; + const auto it = listed.lower_bound(lo >> 12); + const bool pages = it != listed.end() && *it <= (lo + size - 1) >> 12; + const bool in = texture_gpu_written_in(lo, size), marked = texture_gpu_written_marked(lo, size); + if ((in && !marked) || marked != pages) { CHECK(!"the GPU-written textures' list and its marks disagree"); break; } + } + CHECK(!texture_gpu_written_marked(30 * GB, GB)); + Texture far; // (a virtual pool past 512 GB: not kept, so always "marked") + far.gpu = SPAN + 5 * GB, far.guest_size = 0x4000; + vw_note(far); + CHECK(texture_gpu_written_in(far.gpu, 16) && texture_gpu_written_marked(far.gpu, 16)); +} + +// One thread notes write-backs; four ask. noted: how many notes have returned +static void threads() { + const int N = 4000; + const uint64_t base = 40 * GB; + auto range = [&](int i) { return base + (uint64_t)i * 0x3000; }; // (a page each, a free page on both sides) + std::atomic noted{0}; + std::atomic asks{0}; + std::thread writer([&] { + for (int i = 0; i < N; i++) { + wb_pages_note(range(i), range(i) + 0x1000, (uint64_t)i + 1); + noted.store(i + 1, std::memory_order_seq_cst); + } + }); + std::vector readers; + for (int r = 0; r < 4; r++) + readers.emplace_back([&, r] { + uint64_t s = 1234567 + (uint64_t)r * 7919; + while (noted.load() < N && !fails.load()) { + s = s * 6364136223846793005ull + 1442695040888963407ull; + const int k = noted.load(std::memory_order_seq_cst); + if (k) { // a note that has returned is seen + const int j = (int)((s >> 33) % (uint64_t)k); + if (!memory_writeback_marked(range(j), 0x1000)) { CHECK(!"a write-back noted before the ask was not seen"); break; } + } + const int any = (int)((s >> 20) % (uint64_t)N); // the table first, under its lock, then the marks + bool ever = false; + memory_writeback_value(range(any), 0x1000, &ever); + if (ever && !memory_writeback_marked(range(any), 0x1000)) { CHECK(!"the table had a value its marks did not have"); break; } + if (memory_writeback_marked(range(any) + 0x1000, 0x1000)) { CHECK(!"a page no note reached is marked"); break; } + asks++; + } + }); + writer.join(); + for (auto& t : readers) t.join(); + for (int i = 0; i < N; i++) + if (!memory_writeback_marked(range(i), 0x1000)) { CHECK(!"a note is missing at the end"); break; } + std::printf("threads: %d notes beside %ld asks\n", N, asks.load()); +} + +static void rule() { + using namespace inv_quick; + Look l; + l.gpu = 0x80000000, l.size = 64; + CHECK(nothing_to_wait_for(l)); // (an occlusion slot nothing ever happened to) + for (int which = 0; which < 4; which++) { // (each reason alone makes the call) + Look x = l; + (which == 0 ? x.ticket_pending : which == 1 ? x.tex_storage : which == 2 ? x.vw_marked : x.wb_marked) = true; + CHECK(!nothing_to_wait_for(x)); + } + Look z = l; + z.size = 0, z.ticket_pending = z.tex_storage = z.vw_marked = z.wb_marked = true; + CHECK(nothing_to_wait_for(z)); // (nothing to invalidate: invalidate() waits for nothing) + z.gpu = SPAN * 2; + CHECK(nothing_to_wait_for(z)); + Look far = l; + far.gpu = SPAN; + CHECK(!nothing_to_wait_for(far)); // (past 512 GB: not known, the call is made) + far.gpu = SPAN - 32; + CHECK(!nothing_to_wait_for(far)); // (a range that reaches past it) + far.gpu = SPAN - 64; + CHECK(nothing_to_wait_for(far)); + far.gpu = 0x1000, far.size = ~0ull; + CHECK(!nothing_to_wait_for(far)); + // the switch: off unless asked for, and off with a switch that logs or waits at every call + CHECK(!on(nullptr, nullptr, nullptr, nullptr) && !on("0", nullptr, nullptr, nullptr) && !on("", nullptr, nullptr, nullptr)); + CHECK(!on("off", nullptr, nullptr, nullptr)); + CHECK(on("1", nullptr, nullptr, nullptr) && on("2", nullptr, nullptr, nullptr)); + CHECK(!on("1", "1", nullptr, nullptr) && !on("1", nullptr, "300", nullptr) && !on("1", nullptr, nullptr, "1") && !on("1", "", nullptr, nullptr)); +} + +int main() { + marks_alone(); + tables(); + threads(); + rule(); + if (!fails) std::printf("PASS: the marks, the tables beside them, a note seen by every later ask, and the rule\n"); + return fails ? 1 : 0; +} +''' + + +def ordered(text, needles, what): + at = -1 + for needle in needles: + try: + nxt = text.index(needle, at + 1) + except ValueError: + fail("%s: %r is not there, or not after what comes before it" % (what, needle)) + at = nxt + + +def read_sources(): + memory = (VK / "vk_memory.cpp").read_text(encoding="utf-8") + texture = (VK / "vk_texture.cpp").read_text(encoding="utf-8") + replay = (VK / "vk_replay.cpp").read_text(encoding="utf-8") + backend = (ROOT / "host/rx_backend.c").read_text(encoding="utf-8") + dev = (ROOT / "docs/development.md").read_text(encoding="utf-8") + note = block(memory, "void wb_pages_note(uint64_t guest_lo, uint64_t guest_hi, uint64_t value) {") + ordered(note, ["std::lock_guard lk(wb_pages_mu);", "wb_marks.mark(guest_lo, guest_hi);", "wb_pages.set(p, value);"], + "vk_memory.cpp wb_pages_note (the mark under the table's lock and before its value)") + if memory.count("wb_pages.set(") != 1: + fail("vk_memory.cpp: wb_pages has more than one writer: each must make its mark first") + vw_note = block(texture, "void vw_note(Texture& t) {") + ordered(vw_note, ["std::lock_guard lk(vw_mu);", "vw_marks.mark(t.gpu, t.gpu + t.guest_size);", "uint64_t& end = vw[t.gpu];", "t.vw_listed = true;"], + "vk_texture.cpp vw_note (the mark under the list's lock and before its entry)") + if texture.count("vw_listed = true") != 1: + fail("vk_texture.cpp: a texture is listed in vw somewhere other than vw_note: that place must make the mark first") + forget = block(texture, "void vw_forget(const Texture& t) {") + if "if (it->second != &t && it->second->vw_listed) {" not in forget or texture.count("vw[t.gpu]") != 2: + fail("vk_texture.cpp: vw gains an entry for a texture vw_note did not list (vw_forget puts back listed ones only)") + quick = block(replay, "int invalidate_quick(uint64_t gpu, uint64_t size) {") + ordered(quick, ['inv_quick::on(getenv("RX_VK_INVALIDATE_QUICK"), getenv("RX_VK_LOG_INVALIDATE"),', 'getenv("RX_VK_LOG_READBACK"), getenv("RX_VK_INVALIDATE_WAIT"));', + "if (!asked) return 0;", 'if (!lab_ab::on("noinvq")) {', + "const uint64_t ticket = fence_poll_fast_on() ? order_ticket.load(std::memory_order_seq_cst) : 0;", + "l.ticket_pending = sync_order_mu.left.load(std::memory_order_acquire) < ticket;", + "l.tex_storage = tex_pages_any(gpu, size);", "l.vw_marked = texture_gpu_written_marked(gpu, size);", + "l.wb_marked = memory_writeback_marked(host->gpu_guest(gpu), size);", "nothing = inv_quick::nothing_to_wait_for(l);", + "return nothing ? 1 : 0;"], "vk_replay.cpp invalidate_quick") + for forbidden in ("std::lock_guard", "std::unique_lock", "wait_queue_empty", "wait_value", "wait_recorded", "sync_settle", "order_ticket_wait("): + if forbidden in quick: + fail("vk_replay.cpp invalidate_quick takes a lock or waits (%s): its caller holds a virtual core" % forbidden) + # (invalidate() itself: the path the rule says is taken without a wait, in the order the rule assumes) + inv = block(replay, "void invalidate(uint64_t gpu, uint64_t size) {") + ordered(inv, ["if (size && fence_poll_fast_on()) order_ticket_wait();", + 'static const bool log_inv = getenv("RX_VK_LOG_INVALIDATE") != nullptr;', + 'getenv("RX_VK_LOG_READBACK") ? atoll(getenv("RX_VK_LOG_READBACK")) : -1;', + "if (!off && (v0 = memory_writeback_value(host->gpu_guest(gpu), size, &ever), ever)) {", # (SYNC-19 keeps the value) + 'static const bool always = getenv("RX_VK_INVALIDATE_WAIT") != nullptr;', "if (!size) return;", + "if (!texture_gpu_written_in(gpu, size)) {", "if (!replay_behind()) return;", + "if (!always && !tex_pages_any(gpu, size)) return;"], + "vk_replay.cpp invalidate() no longer returns without waiting where invalidate_quick says it does: bring the rule up to it") + head = inv[:inv.index("if (!off && (v0 = memory_writeback_value(")] + for call in ("wait_queue_empty()", "sync_settle()", "wait_value(", "wait_recorded("): + if call in head: + fail("vk_replay.cpp invalidate() waits (%s) before it asks whether the range was ever written back: the rule does not know it" % call) + ordered(replay, ["int mkwvk_invalidate_quick(uint64_t gpu, uint64_t size) {", "return mkwvk::invalidate_quick(gpu, size);"], "the export") + wrapper = block(backend, "void rx_nvn_invalidate(uint64_t gpu, uint64_t size) {") + ordered(wrapper, ["if (vk && invalidate_quick_fn && invalidate_quick_fn(gpu, size)) return;", "const int away = hos_vcore_away();", + "vk ? vk->nvn_invalidate(gpu, size) : wgr_nvn_invalidate(gpu, size);", "if (away) hos_vcore_back();"], + "host/rx_backend.c rx_nvn_invalidate (the question before the core is given up)") + if 'invalidate_quick_fn = (int (*)(uint64_t, uint64_t))(void *)GetProcAddress(vk_module, "mkwvk_invalidate_quick");' not in backend: + fail("host/rx_backend.c: mkwvk_invalidate_quick is not looked up by name when the Vulkan back end starts") + if "| `RX_VK_INVALIDATE_QUICK=1` |" not in dev: + fail("docs/development.md: RX_VK_INVALIDATE_QUICK is not in the switches table") + return memory, texture + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + try: + memory, texture = read_sources() + source = FIXTURE + for mark, text, marker in (("/* WBPAGES */", memory, "struct WbPages {"), + ("/* WBNOTE */", memory, "void wb_pages_note(uint64_t guest_lo, uint64_t guest_hi, uint64_t value) {"), + ("/* WBVALUE */", memory, "uint64_t memory_writeback_value(uint64_t guest, uint64_t size, bool* ever) {"), + ("/* WBMARKED */", memory, "bool memory_writeback_marked(uint64_t guest, uint64_t size) {"), + ("/* VWNOTE */", texture, "void vw_note(Texture& t) {"), + ("/* VWIN */", texture, "bool texture_gpu_written_in(uint64_t lo, uint64_t size) {"), + ("/* VWMARKED */", texture, "bool texture_gpu_written_marked(uint64_t lo, uint64_t size) {")): + source = source.replace(mark, block(text, marker)) + except (ValueError, OSError) as e: + fail("a text this test reads is not there (%s)" % e) + with tempfile.TemporaryDirectory(prefix="invalidate-quick-") as tmp: + src, exe = Path(tmp) / "quick.cpp", Path(tmp) / "quick.exe" + src.write_text(source, encoding="utf-8") + built = subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-Wno-unused-function", "-Wno-unused-variable", + "-pthread", "-I" + str(VK), str(src), "-o", str(exe)], capture_output=True, text=True) + if built.returncode: + print(built.stdout + built.stderr) + fail("the fixture did not build") + done = subprocess.run([str(exe)], capture_output=True, text=True, timeout=300) + print(done.stdout.strip()) + if done.returncode or "PASS" not in done.stdout: + fail("the marks, the tables or the rule") + print("PASS: an invalidate is left out only where nothing was ever registered for its range, and the question is off unless asked for") + + +if __name__ == "__main__": + main() diff --git a/tests/test_invalidate_vcore.py b/tests/test_invalidate_vcore.py index 9ba4b3e..9ce5510 100644 --- a/tests/test_invalidate_vcore.py +++ b/tests/test_invalidate_vcore.py @@ -14,7 +14,9 @@ RUN_ALL_ARGS = ["--cxx", "g++"] ROOT = Path(__file__).resolve().parents[1] CASES = ("vulkan", "fallback", "immediate", "fallback-immediate", "zero", "already-away", - "non-guest", "disabled", "producer") + "non-guest", "disabled", "producer", + # ((Triforce, THREAD-25) the back end asked first: nothing to wait for (no call, the core kept), something, no Vulkan back end) + "quick", "quick-declined", "quick-fallback") def block(source, marker): diff --git a/tests/test_jobs_wait_for.py b/tests/test_jobs_wait_for.py index 2beaf2c..04c237f 100644 --- a/tests/test_jobs_wait_for.py +++ b/tests/test_jobs_wait_for.py @@ -71,7 +71,7 @@ def main(): check_source() with tempfile.TemporaryDirectory(prefix="jobs-wait-for-") as tmp: tmp = Path(tmp) - for name in ("vk_jobs.cpp", "vk_jobs.h"): + for name in ("vk_jobs.cpp", "vk_jobs.h", "cpu_count.h"): # ((Triforce, 11-22, OS-49) vk_jobs.cpp counts by cpu_count.h shutil.copy(ROOT / "gpu/vk" / name, tmp / name) (tmp / "vk_internal.h").write_text(STUB_INTERNAL, encoding="utf-8") (tmp / "vk_draw.h").write_text(STUB_DRAW, encoding="utf-8") diff --git a/tests/test_jobs_wait_runs_queued.py b/tests/test_jobs_wait_runs_queued.py index c9b9c6b..fb824eb 100644 --- a/tests/test_jobs_wait_runs_queued.py +++ b/tests/test_jobs_wait_runs_queued.py @@ -57,7 +57,7 @@ def main(): check_source() with tempfile.TemporaryDirectory(prefix="jobs-wait-") as tmp: tmp = Path(tmp) - for name in ("vk_jobs.cpp", "vk_jobs.h"): + for name in ("vk_jobs.cpp", "vk_jobs.h", "cpu_count.h"): # ((Triforce, 11-22, OS-49) vk_jobs.cpp counts by cpu_count.h shutil.copy(ROOT / "gpu/vk" / name, tmp / name) (tmp / "vk_internal.h").write_text(STUB_INTERNAL, encoding="utf-8") (tmp / "vk_draw.h").write_text(STUB_DRAW, encoding="utf-8") diff --git a/tests/test_lab_ab_each.py b/tests/test_lab_ab_each.py new file mode 100644 index 0000000..03fec28 --- /dev/null +++ b/tests/test_lab_ab_each.py @@ -0,0 +1,186 @@ +"""The lab's in-run A/B with a wait at every draw or dispatch (gpu/vk/lab_ab.h e_draw, e_dispatch, d_present_after; the +performance audit's REPLAY-51, after Source B's; no game, no GPU). + +lab_ab.h compiled and run: RX_VK_LAB_AB=e_draw:200,e_dispatch:2000,d_present_after:1000 is read as three knobs; in the none +phases and the other knobs' phases each() returns at once and counts nothing; in its own phase it waits at least the knob's +nanoseconds at every call, on any thread, and counts each wait; at the phase's end one line gives its waits and ms a second, +and the counts start again; d_present_after waits once a frame in its phase as the d_ knobs do. Then the sources: draw() and +dispatch() wait before their profile clocks; the present's after-wait follows drain(0); the replay logs a phase's line and +warns once when RX_VK_PARALLEL is not pinned. + + python -I -B tests/test_lab_ab_each.py [--cxx clang++] +""" +import argparse +import os +import subprocess +import tempfile +from pathlib import Path + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def in_order(text, parts, where): + at = 0 + for part in parts: + found = text.find(part, at) + if found < 0: + raise SystemExit("%s: %r is missing, or not after the part before it (update the test with the code)" % (where, part)) + at = found + len(part) + + +TEST = r''' +#include +#include +#include +#include +#include +#include "lab_ab.h" +#define CHECK(x) do { if (!(x)) { std::fprintf(stderr, "FAIL:%d %s\n", __LINE__, #x); std::exit(1); } } while (0) +using namespace mkwvk; +static int64_t timed(int event, int n) { + const auto t0 = std::chrono::steady_clock::now(); + for (int i = 0; i < n; i++) lab_ab::each(event); + return std::chrono::duration_cast(std::chrono::steady_clock::now() - t0).count(); +} +static void to_phase(int want) { // (presents until the phase's knob is `want`) + for (int i = 0; i < 100000; i++) { + const int c = lab_ab::phase_tick(); + if ((c < 0 ? -c - 1 : c) == want) return; + } + CHECK(false); +} +int main() { + lab_ab::init(); + CHECK(lab_ab::n_knobs == 3 && !std::strcmp(lab_ab::knobs[0].name, "e_draw") && lab_ab::knobs[0].value == 200); + CHECK(!std::strcmp(lab_ab::knobs[1].name, "e_dispatch") && lab_ab::knobs[1].value == 2000); + CHECK(!std::strcmp(lab_ab::knobs[2].name, "d_present_after") && lab_ab::knobs[2].value == 1000 && lab_ab::any_each()); + // (no phase yet: no wait, nothing counted) + CHECK(timed(lab_ab::E_DRAW, 1000) < 5000000 && lab_ab::each_spins[lab_ab::E_DRAW].load() == 0); + to_phase(0); + CHECK(lab_ab::each_ns[lab_ab::E_DRAW].load() == 0 && lab_ab::each_ns[lab_ab::E_DISPATCH].load() == 0); + timed(lab_ab::E_DRAW, 1000); + CHECK(lab_ab::each_spins[lab_ab::E_DRAW].load() == 0); + // (e_draw's phase: at least 200 ns a draw, each counted, on two threads; dispatches untouched) + to_phase(1); + CHECK(lab_ab::each_ns[lab_ab::E_DRAW].load() == 200 && lab_ab::each_ns[lab_ab::E_DISPATCH].load() == 0); + const int64_t spent = timed(lab_ab::E_DRAW, 2000); + std::thread other([] { for (int i = 0; i < 500; i++) lab_ab::each(lab_ab::E_DRAW); }); + other.join(); + CHECK(spent >= 2000 * 200); + CHECK(lab_ab::each_spins[lab_ab::E_DRAW].load() == 2500 && lab_ab::each_spun_ns[lab_ab::E_DRAW].load() >= 2500ull * 200); + timed(lab_ab::E_DISPATCH, 100); + CHECK(lab_ab::each_spins[lab_ab::E_DISPATCH].load() == 0); + // (its end: the line, and the counts start again) + to_phase(0); + CHECK(std::strstr(lab_ab::each_line, "phase e_draw:200 over") && std::strstr(lab_ab::each_line, "(2500 waits in")); + CHECK(lab_ab::each_spins[lab_ab::E_DRAW].load() == 0 && lab_ab::each_ns[lab_ab::E_DRAW].load() == 0); + // (e_dispatch's phase: 2000 ns a dispatch) + to_phase(2); + CHECK(lab_ab::each_ns[lab_ab::E_DISPATCH].load() == 2000 && lab_ab::each_ns[lab_ab::E_DRAW].load() == 0); + CHECK(timed(lab_ab::E_DISPATCH, 300) >= 300 * 2000 && lab_ab::each_spins[lab_ab::E_DISPATCH].load() == 300); + to_phase(0); + CHECK(std::strstr(lab_ab::each_line, "phase e_dispatch:2000 over") && std::strstr(lab_ab::each_line, "(300 waits in")); + // (d_present_after's phase: no event waits, and no line at its end; its delay once a frame, at least 1 ms) + to_phase(3); + CHECK(lab_ab::each_line[0] == 0); + CHECK(lab_ab::each_ns[lab_ab::E_DISPATCH].load() == 0 && lab_ab::each_ns[lab_ab::E_DRAW].load() == 0); + auto t0 = std::chrono::steady_clock::now(); + lab_ab::delay("present_after"); + const int64_t first = std::chrono::duration_cast(std::chrono::steady_clock::now() - t0).count(); + t0 = std::chrono::steady_clock::now(); + lab_ab::delay("present_after"); + const int64_t second = std::chrono::duration_cast(std::chrono::steady_clock::now() - t0).count(); + CHECK(first >= 1000 && second < 500); + lab_ab::delay("present"); // (d_present is another knob: nothing here) + std::puts("PASS lab_ab.h: e_draw, e_dispatch and d_present_after read; waits only in their phases, at least their time, " + "counted on any thread; the phase's line; d_present_after once a frame"); + return 0; +} +''' + +MUTATIONS = [ + ("no wait without its knob", " if (!ns) return;\n const auto t0", " const auto t0"), + ("each wait counted", " each_spins[event].fetch_add(1, std::memory_order_relaxed);\n", ""), + ("the waits set by the phase", " each_tick(k); //", " (void)0; //"), + ("only the phase's own event", "k > 0 && !strcmp(knobs[k - 1].name, each_names[e])", "k > 0"), + ("the counts start again", "const uint64_t spins = each_spins[e].exchange(0, std::memory_order_relaxed);", + "const uint64_t spins = each_spins[e].load(std::memory_order_relaxed);"), +] + + +def check_source(draw, replay, dev): + d = block(draw, "void draw(VkCommandBuffer cmd, NvnState& s, const DrawArgs& d) {") + in_order(d, ["Pass& ps = pass();", "lab_ab::each(lab_ab::E_DRAW);", "PROF_HOT(PROF_DRAW);"], "vk_draw.cpp draw(): the wait before the profile") + disp = block(draw, "void dispatch(VkCommandBuffer cmd, NvnState& s, uint32_t x, uint32_t y, uint32_t z, uint64_t indirect) {") + if not disp.split("\n", 2)[1].strip().startswith("lab_ab::each(lab_ab::E_DISPATCH);"): + raise SystemExit("vk_draw.cpp dispatch(): the lab's wait is not its first statement") + in_order(disp, ["lab_ab::each(lab_ab::E_DISPATCH);", "PROF(PROF_DISPATCH);", "DispatchClock dclock;"], "vk_draw.cpp dispatch()") + pres = block(replay, "void present(const nvn_tex* t, const int crop[4]) {") + in_order(pres, ['lab_ab::delay("present");', "drain(0);", 'lab_ab::delay("present_after");', "do_present(t, crop);"], + "vk_replay.cpp present(): d_present before the drain, d_present_after just after it") + tick = block(replay, "void status_tick() {") + in_order(tick, ["const int lab_col = lab_ab::phase_tick();", 'if (lab_ab::each_line[0]) logf("%s", lab_ab::each_line)', + 'if (lab_ab::any_each() && !getenv("RX_VK_PARALLEL"))'], "vk_replay.cpp status_tick: the phase's line and the warning") + if "e_draw" not in dev or "d_present_after" not in dev: + raise SystemExit("docs/development.md: the e_ knobs and d_present_after are not in RX_VK_LAB_AB's row") + + +def build(cxx, d, name, include): + cpp, exe = d / (name + ".cpp"), d / (name + ".exe") + cpp.write_text(TEST, encoding="utf-8") + r = subprocess.run([cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-Wno-unused-function", "-pthread", "-I" + str(include), + str(cpp), "-o", str(exe)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + return (exe, "") if r.returncode == 0 else (None, r.stdout) + + +def run(exe): + env = dict(os.environ, RX_VK_LAB_AB="e_draw:200,e_dispatch:2000,d_present_after:1000", RX_VK_LAB_AB_FRAMES="10", + RX_VK_LAB_AB_SKIP="0") + return subprocess.run([str(exe)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=120, env=env) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--cxx", default=os.environ.get("CXX", "g++")) + a = ap.parse_args() + check_source((VK / "vk_draw.cpp").read_text(encoding="utf-8"), (VK / "vk_replay.cpp").read_text(encoding="utf-8"), + (ROOT / "docs" / "development.md").read_text(encoding="utf-8")) + print("PASS sources: draw() and dispatch() wait before their clocks, d_present_after after drain(0), the phase's line and " + "the warning, the row") + header = (VK / "lab_ab.h").read_text(encoding="utf-8") + with tempfile.TemporaryDirectory(prefix="lab-ab-each-") as tmp: + d = Path(tmp) + exe, out = build(a.cxx, d, "each", VK) + if not exe: + raise SystemExit("the build failed:\n" + out) + r = run(exe) + if r.returncode: + raise SystemExit("the test failed:\n" + r.stdout) + print(r.stdout.strip()) + for i, (what, old, new) in enumerate(MUTATIONS): + if header.count(old) != 1: + raise SystemExit("mutation %r: its text is not once in lab_ab.h (update the test with the header)" % what) + inc = d / ("mutant%d" % i) + inc.mkdir() + (inc / "lab_ab.h").write_text(header.replace(old, new), encoding="utf-8") + exe, out = build(a.cxx, d, "mutant%d" % i, inc) + if exe and run(exe).returncode == 0: + raise SystemExit("mutation %r: the test still passed without it" % what) + print("PASS mutations: %d rules of the e_ knobs, each taken out, fail the test" % len(MUTATIONS)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_late_inline.py b/tests/test_late_inline.py new file mode 100644 index 0000000..4812ba7 --- /dev/null +++ b/tests/test_late_inline.py @@ -0,0 +1,130 @@ +"""The late pages' partial rule alone, and one read of the GPU's progress for it (gpu/vk/late_inline.h, vk_memory.cpp +capture_make and take_capture; no game, no GPU). + +(Triforce, 11-22, the performance audit's MEM-80, after Source B's.) RX_VK_LATE_NEW (knob latenew) turns four no-wait rules +for late pages on; RX_VK_LATE_INLINE=1 (knob lateinline), OFF unless set, turns on the partial rule alone (a page written +back in part has its other parts synced, no wait), and every other late page is waited for as by default. The partial rule +measures the write-backs still on the GPU against the progress capture_make read before its copies (kept in the Capture), +not a second read at the take. + +Compiles the header: the switch's text and knob; over every input, with RX_VK_LATE_INLINE off the rules are the ones +RX_VK_LATE_NEW gave (all four by their own switches, or none), with it on and RX_VK_LATE_NEW off the partial rule alone. + +And reads the source: the Capture keeps late_done, cleared with late_value when a capture is reused and set where +capture_make reads the progress; take_capture's gate is the rules' any, the four rules each asked by their flag, a page of +a pool with no mirror waited for when the skip rule is off, the partial rule's progress the capture's with no +vk_timeline_completed() in that block, the switch read once, the switches table. + + python -I -B tests/test_late_inline.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +PROGRAM = r""" +#include "late_inline.h" +#include +using namespace late_inline; +static int fails = 0; +#define CHECK(c) do { if (!(c)) { std::printf("FAIL line %d: %s\n", __LINE__, #c); fails++; } } while (0) + +int main() { + CHECK(!on(nullptr, false) && on(nullptr, true) && !on("0", true) && on("1", false) && !on("x", true)); + for (int bits = 0; bits < 32; bits++) { + const bool nw = bits & 1, in = bits & 2, same = bits & 4, part = bits & 8, defer = bits & 16; + const Rules r = rules(nw, in, same, part, defer); + if (!in) { // (as RX_VK_LATE_NEW gave them: late_all = !late_new, each rule by its own switch) + CHECK(r.any == nw && r.skip == nw && r.same == (nw && same) && r.part == (nw && part) && r.defer == (nw && defer)); + } else if (!nw) { // (the partial rule alone) + CHECK(r.any && !r.skip && !r.same && !r.defer && r.part == part); + } else { // (both: RX_VK_LATE_NEW's rules) + CHECK(r.any && r.skip && r.same == same && r.part == part && r.defer == defer); + } + } + std::printf(fails ? "%d failed\n" : "ok\n", fails); + return fails ? 1 : 0; +} +""" + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def in_order(text, steps, what): + at = -1 + for s in steps: + k = text.find(s, at + 1) + if k < 0: + raise SystemExit("FAIL: %s: %r is missing or out of its order" % (what, s)) + at = k + + +def check_source(): + mem = (VK / "vk_memory.cpp").read_text(encoding="utf-8") + if '#include "late_inline.h"' not in mem: + raise SystemExit("FAIL: vk_memory.cpp does not include late_inline.h") + cap = block(mem, "struct Capture {") + if "uint64_t late_done = 0;" not in cap: + raise SystemExit("FAIL: the Capture keeps no late_done") + make = block(mem, "std::unique_ptr capture_make(bool behind) {") + in_order(make, ["c->late.clear(), c->late_value = 0, c->late_done = 0;", "const uint64_t done = vk_timeline_completed();", + "c->late_done = done;", "if (pv > done) {", "c->late.push_back((uint32_t)i);", "c->late_value = v;"], + "vk_memory.cpp capture_make") + take = mem[mem.index("if (c->late_value && c->map) { // (the pages a submission still on the GPU writes back: see capture_make)"):] + take = take[:take.index("c->late.assign(need.begin(), need.end());")] + in_order(take, ['static const char* late_inline_env = getenv("RX_VK_LATE_INLINE");', + 'const late_inline::Rules rules = late_inline::rules(late_new_env || lab_ab::on("latenew"),', + 'late_inline::on(late_inline_env, lab_ab::on("lateinline")), late_same_on(),', "late_part_on(), late_defer_on());", + "const bool late_all = !rules.any;", "if (!late_all) {", + "if (p && p->mstate != Pool::MIRROR_ON && p->mstate != Pool::MIRROR_WANTED) {", + "if (rules.skip) late_skipped_now.push_back(i);", "else need.push_back(i);", "continue;", + "if (rules.same && std::binary_search(c->slots.begin(), c->slots.end(), i)) {", + "if (rules.part && std::binary_search(c->slots.begin(), c->slots.end(), i)) late_part_cand.push_back(i);", + "else need.push_back(i);", "if (!late_part_cand.empty()) {", "const uint64_t done = c->late_done;", + "wb_inflight_in(c->pages[i], done, g, LatePartial::kRanges, &n);", "&& rules.defer &&"], + "vk_memory.cpp take_capture's late pages") + part = take[take.index("if (!late_part_cand.empty()) {"):] + part = part[:part.index("st_late_part += ")] + if "vk_timeline_completed()" in part: + raise SystemExit("FAIL: the partial rule reads the GPU's progress again at the take") + for old in ("if (late_same_on() && std::binary_search", "if (late_part_on() && std::binary_search", "&& late_defer_on() &&"): + if old in take: + raise SystemExit("FAIL: take_capture asks a rule's switch directly, not its flag: %s" % old) + dev_md = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + if not any(line.startswith("| `RX_VK_LATE_INLINE=1` |") and "**off unless set**" in line and "`lateinline`" in line + for line in dev_md.splitlines()): + raise SystemExit("FAIL: docs/development.md has no row for RX_VK_LATE_INLINE with its default") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + check_source() + with tempfile.TemporaryDirectory(prefix="late-inline-") as tmp: + src = Path(tmp) / "t.cpp" + exe = Path(tmp) / "t.exe" + src.write_text(PROGRAM, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(VK), str(src), "-o", str(exe)], + check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True, timeout=60) + print(r.stdout.strip()) + if r.returncode: + raise SystemExit("FAIL: late_inline.h") + print("ok: test_late_inline") + + +if __name__ == "__main__": + main() diff --git a/tests/test_lazy_wb_host.py b/tests/test_lazy_wb_host.py index 6fb9bc2..18344e9 100644 --- a/tests/test_lazy_wb_host.py +++ b/tests/test_lazy_wb_host.py @@ -34,6 +34,8 @@ GUEST = re.compile(r"\b(GP|A64_MEM|HZ_PTR|hos_guest_range|G8P|G16P|G32P|G64P)\b" # (file, function, call): why its buffer or pages are never guest memory. Reviewed one by one (2026-11-15) ALLOWED = { + ("achievements.c", "read_text", "fread"): "into a buffer it mallocs", + ("achievements.c", "earned_save", "fwrite"): "from a static buffer of its own, the earned achievements' text", ("bug_capture.c", "read_part", "fread"): "into a buffer it mallocs", ("nn_fs.c", "hos_fs_rom_file", "ReadFile"): "a RomFS file for the menu sounds (host/menu_sound.c), into a buffer it mallocs", ("bug_capture.c", "write_file", "fwrite"): "the report's own files: the options file and a log read into host memory", @@ -80,6 +82,9 @@ ALLOWED_GUEST = { ("mkw_glue.c", "commit_on_touch", "VirtualAlloc"): "the host's own fault handler committing a page that is only reserved " "(it asks VirtualQuery first): every page the lazy write-back protects " "is committed, imported whole, and its handler comes first", + ("mkw_glue.c", "commit_wait", "VirtualAlloc"): "(Triforce, 11-21, MEM-09) the commit its two callers were refused at the commit " + "limit, tried again: hos_commit's is inside its hos_guest_io_begin / " + "hos_guest_io_end, and commit_on_touch's pages are only reserved, as above", ("hos_mem.c", "census_release", "VirtualFree"): "the census's give-back: its caller census_offer wraps the renderer's call " "that runs it in hos_guest_io_begin / hos_guest_io_end; inside, the renderer " "holds its import locks, under which the lazy write-back's begin is not taken", diff --git a/tests/test_lift_key_env.py b/tests/test_lift_key_env.py new file mode 100644 index 0000000..ee9cd73 --- /dev/null +++ b/tests/test_lift_key_env.py @@ -0,0 +1,254 @@ +"""The lift key takes only the environment that shapes the lifted C (nx2/make_game.py lift_key). + +(Triforce, 11-22, the performance audit's LIFT-27, after Source A's.) The key decides whether a build translates the +game again. It hashed every A64_, OOT_ and RX_ variable and the folders' paths, so a build from another folder, a +release build after a development one, or one more variable in the environment translated the same C again. It now +takes the lifter's own switches (A64_*) and the target (RX_TARGET); HOOKSHOT_KEEP_LIFT=0 is the key as it was. + +What must hold for that to be safe, and is held here: + - every environment variable the lift's files read (nx2/lift.py's and nx2/split_hot.py's imports, found as + tests/test_lift_key_files.py finds them) is read by its literal name, and is either in the key (A64_*, RX_TARGET) or + one of those listed below with why it cannot change the C: folders whose files the key hashes by their contents, job + counts, where the compiler is. A variable read any other way, or a new one, fails this test until it is listed; + - lift_key itself, on a made-up tree: a folder's path, RX_RELEASE, the profile's path, a job count and a stray RX_ + variable leave the key alone; an A64_ switch, in the environment or given, and RX_TARGET change it; + - with HOOKSHOT_KEEP_LIFT=0 the key is the old formula's, computed here beside it, and each of those changes it again; + - nx2/released_translation_stamp.json, which build.py alone reads, is out of the key, and back in with the switch. + + python -I -B tests/test_lift_key_env.py +""" +import ast +import hashlib +import os +from pathlib import Path +import sys +import tempfile +import types +import unittest +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from test_lift_key_files import lift_imports, production # (the lift's files, and lift_key cut out of make_game.py) + +# (what the lift's files read beside the key's own, and why none of it shapes the C) +NOT_IN_KEY = { + "OOT_WORK": "the work folder: the files read there (hot_funcs.txt, image/chunks.txt) are hashed by their contents", + "OOT_DUMP": "the game's folder: the lift is given the exefs, whose files are hashed by their contents", + "OOT_EXEFS": "the game's code folder: its files are hashed by their contents", + "OOT_ROMFS": "the game's data folder: kit.py names it, the lift reads nothing there", + "HOOKSHOT_LIFT_JOBS": "how many processes write the C: the C is the same whatever the count (tests/test_spawn_emit.py)", + "HOOKSHOT_LIFTED_C_REGEX": "how nx2/lifted_c.py finds the definitions, by jumps or by the patterns: the same matches " + "either way (the performance audit's IO-40, tests/test_lifted_c_scan.py)", + "RX_JOBS": "how many compilers run at once (the kit's toolchain.default_jobs)", + "EMSDK": "where a compiler is (the kit's toolchain)", + "HOOKSHOT_MINGW": "where a compiler is (the kit's toolchain)", +} + + +def in_key(name): + return name.startswith("A64_") or name == "RX_TARGET" + + +def environment_reads(path): + """Each use of the environment in a Python file: (line, the variable's name, or None when it is not read by a + literal name)""" + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + parents = {child: node for node in ast.walk(tree) for child in ast.iter_child_nodes(node)} + reads = [] + + def literal(node): + return node.value if isinstance(node, ast.Constant) and isinstance(node.value, str) else None + + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom) and node.module == "os": # (under another name it would be read unseen) + reads += [(node.lineno, None) for a in node.names if a.name in ("environ", "getenv") and a.asname] + is_environ = isinstance(node, ast.Attribute) and node.attr == "environ" or isinstance(node, ast.Name) and node.id == "environ" + is_getenv = isinstance(node, ast.Attribute) and node.attr == "getenv" or isinstance(node, ast.Name) and node.id == "getenv" + if not (is_environ or is_getenv): + continue + up = parents.get(node) + name = None + if is_getenv and isinstance(up, ast.Call) and up.func is node and up.args: + name = literal(up.args[0]) + elif is_environ and isinstance(up, ast.Subscript) and up.value is node: + name = literal(up.slice) + elif is_environ and isinstance(up, ast.Attribute) and up.attr in ("get", "pop", "setdefault"): + call = parents.get(up) + if isinstance(call, ast.Call) and call.func is up and call.args: + name = literal(call.args[0]) + elif is_environ and isinstance(up, ast.Compare) and node in up.comparators and len(up.ops) == 1 and \ + isinstance(up.ops[0], (ast.In, ast.NotIn)): + name = literal(up.left) + reads.append((node.lineno, name)) + return reads + + +def old_key(exefs, work, env, here, root, implemented): + """The lift key as nx2/make_game.py computed it before the performance audit's LIFT-27 (chunks.txt included)""" + h = hashlib.sha1(sys.version.encode(), usedforsecurity=False) + h.update(" ".join(sorted(implemented)).encode()) + not_key = ("make_game.py", "build.py", "nup.py", "version.py", "shader_census.py", "prompt_assets.py", "prompt_draw.py", + "prompt_faces.py", "prompt_requirements.txt", "prompt_keys.py", "code_mods.py") + + def kept(rel): + name = rel.rsplit("/", 1)[-1] + return not (rel in not_key or rel.startswith("prompt_faces/") or (name.startswith("test_") and name.endswith(".py"))) + + files = [(exefs, os.path.join(exefs, f)) for f in sorted(os.listdir(exefs))] + for top in (here, os.path.join(root, "third_party", "a64-kit")): + for d, dirs, fs in os.walk(top): + dirs[:] = sorted(x for x in dirs if x != "__pycache__") + files += [(root, os.path.join(d, f)) for f in sorted(fs) if f.endswith((".py", ".c", ".h", ".cpp", ".hpp", ".txt", ".json")) + and (top != here or kept(os.path.relpath(os.path.join(d, f), here).replace(os.sep, "/")))] + files.append((work, os.path.join(work, "hot_funcs.txt"))) + files.append((work, os.path.join(work, "image", "chunks.txt"))) + for base, f in files: + if os.path.isfile(f): + h.update(os.path.relpath(f, base).replace(os.sep, "/").encode() + bytes(1)) + with open(f, "rb") as src: + h.update(hashlib.sha1(src.read(), usedforsecurity=False).digest()) + lift_env = dict(env, **{k: v for k, v in os.environ.items() if k.startswith(("A64_", "OOT_", "RX_"))}) + for name in ("RX_VK_PGO", "RX_VK_PGO_FILE"): + lift_env.pop(name, None) + h.update(repr(sorted(lift_env.items())).encode()) + return h.hexdigest() + + +class LiftKeyEnvironment(unittest.TestCase): + def test_the_lift_reads_no_variable_that_is_not_listed(self): + files = sorted(p for p in lift_imports() if p.suffix == ".py") + self.assertTrue(any(p.name == "a64_lift.py" for p in files) and any(p.name == "kit.py" for p in files)) + seen = set() + for path in files: + for line, name in environment_reads(path): + where = "%s:%d" % (path.relative_to(ROOT).as_posix(), line) + self.assertIsNotNone(name, "%s uses the environment other than by a variable's literal name: the lift key " + "cannot tell what it reads" % where) + self.assertTrue(in_key(name) or name in NOT_IN_KEY, + "%s reads %s, which the lift key does not hash: put it in the key (lift_key) if it can change " + "the lifted C, or list it in this test with why it cannot" % (where, name)) + seen.add(name) + # (the scan did find the lift's reads, of both kinds, and the list holds nothing the lift no longer reads) + self.assertTrue({"A64_HOT_FUNCS", "A64_NO_VFPX", "RX_TARGET", "OOT_WORK", "HOOKSHOT_LIFT_JOBS"} <= seen, sorted(seen)) + self.assertEqual(sorted(n for n in NOT_IN_KEY if n not in seen), []) + + def test_the_scan_sees_each_way_of_reading(self): + with tempfile.TemporaryDirectory(prefix="lift-key-env-") as tmp: + path = Path(tmp) / "reads.py" + path.write_text("import os\nfrom os import environ, getenv\n" + "a = os.environ.get('A64_ONE')\nb = os.environ['A64_TWO']\nc = 'A64_THREE' in os.environ\n" + "d = os.getenv('A64_FOUR')\ne = getenv('A64_FIVE', '')\nf = environ.get('A64_SIX')\n" + "g = os.environ.get(name)\nh = dict(os.environ)\nfor k in os.environ:\n pass\n" + "from os import environ as hidden\n", encoding="utf-8") + names = [name for _, name in environment_reads(path)] + self.assertEqual(sorted(n for n in names if n), ["A64_FIVE", "A64_FOUR", "A64_ONE", "A64_SIX", "A64_THREE", "A64_TWO"]) + self.assertEqual(names.count(None), 4) # (a name not written out, the whole environment twice, and another name for it) + + def test_the_key_follows_what_shapes_the_c_only(self): + scope = production() + self.addCleanup(lambda: scope.update(HERE=str(ROOT / "nx2"), ROOT=str(ROOT))) + with tempfile.TemporaryDirectory(prefix="lift-key-env-") as tmp: + root = Path(tmp) + here = root / "nx2" + (root / "third_party" / "a64-kit").mkdir(parents=True) + here.mkdir() + exefs, work, other_work = root / "exefs", root / "work", root / "other-work" + exefs.mkdir() + for w in (work, other_work): + (w / "image").mkdir(parents=True) + (w / "hot_funcs.txt").write_text("f_80000000\n") + (w / "image" / "chunks.txt").write_text("80001000\n") + (exefs / "main").write_bytes(b"game code") + (here / "lift.py").write_text("# the lift\n", encoding="utf-8") + (here / "build.py").write_text("# the build\n", encoding="utf-8") + stamp_json = here / "released_translation_stamp.json" + stamp_json.write_text('{"stamp": "1"}\n', encoding="utf-8") + (root / "third_party" / "a64-kit" / "a64_lift.py").write_text("# kit\n", encoding="utf-8") + scope.update(HERE=str(here), ROOT=str(root)) + implemented = ["nn::os::A", "nn::os::B"] + standalone = types.ModuleType("standalone") + standalone.implemented = lambda: implemented + given = {"OOT_WORK": str(work), "OOT_EXEFS": str(exefs), "OOT_ROMFS": str(root / "romfs"), "OOT_DUMP": str(root), + "RX_RELEASE": "1", "A64_HOT_CHUNK": "1", "RX_PGO": str(work / "pieces.prof")} + clean = {k: v for k, v in os.environ.items() + if not k.startswith(("A64_", "OOT_", "RX_")) and k not in ("HOOKSHOT_KEEP_LIFT", "HOOKSHOT_LIFT_JOBS")} + + def key(env=given, where=work, **environ): + with mock.patch.dict(os.environ, dict(clean, **environ), clear=True): + return scope["lift_key"](str(exefs), str(where), env) + + def before(env=given, where=work, **environ): + with mock.patch.dict(os.environ, dict(clean, **environ), clear=True): + return old_key(str(exefs), str(where), env, str(here), str(root), implemented) + + with mock.patch.dict(sys.modules, {"kit": types.ModuleType("kit"), "standalone": standalone}): + first = key() + self.assertEqual(first, key()) + # ---- what does not shape the C leaves the key alone + same = { + "another work folder with the same files": key(dict(given, OOT_WORK=str(other_work), RX_PGO="x"), other_work), + "the game's folders elsewhere": key(dict(given, OOT_ROMFS="r", OOT_DUMP="d", OOT_EXEFS="e")), + "not a release build": key({k: v for k, v in given.items() if k != "RX_RELEASE"}), + "a job count": key(RX_JOBS="3", HOOKSHOT_LIFT_JOBS="2"), + "a stray RX_ variable": key(RX_SOMETHING_ELSE="1", RX_MARCH="x86-64-v2"), + "a folder variable in the environment": key(OOT_WORK="elsewhere", OOT_ROMFS="there"), + "the switch itself, on": key(HOOKSHOT_KEEP_LIFT="1"), + } + for what, k in same.items(): + self.assertEqual(k, first, what) + # ---- what does changes it + different = { + "a lifter switch in the environment": key(A64_NO_VFPX="1"), + "a lifter switch given": key(dict(given, A64_HOT_CHUNK="")), + "the hot list's variable": key(dict(given, A64_HOT_FUNCS=str(work / "hot_funcs.txt"))), + "the target": key(RX_TARGET="windows"), + } + for what, k in different.items(): + self.assertNotEqual(k, first, what) + self.assertEqual(len(set(different.values())), len(different)) + # ---- the release's translation stamp: out of the key + stamp_json.write_text('{"stamp": "2"}\n', encoding="utf-8") + self.assertEqual(key(), first) + stamp_json.write_text('{"stamp": "1"}\n', encoding="utf-8") + (here / "lift.py").write_text("# the lift, changed\n", encoding="utf-8") + self.assertNotEqual(key(), first) # (a file of the translator still changes it) + (here / "lift.py").write_text("# the lift\n", encoding="utf-8") + + # ---- HOOKSHOT_KEEP_LIFT=0: the key as it was, to the digit, and everything above changes it again + old = key(HOOKSHOT_KEEP_LIFT="0") + self.assertEqual(old, before()) + self.assertNotEqual(old, first) # (the key changes once with this change: one lift again, the same C) + for what, env, where, environ in ( + ("another work folder", dict(given, OOT_WORK=str(other_work)), other_work, {}), + ("not a release build", {k: v for k, v in given.items() if k != "RX_RELEASE"}, work, {}), + ("a stray RX_ variable", given, work, {"RX_SOMETHING_ELSE": "1"}), + ("a lifter switch", given, work, {"A64_NO_VFPX": "1"})): + was = key(env, where, HOOKSHOT_KEEP_LIFT="0", **environ) + self.assertEqual(was, before(env, where, **environ), what) + self.assertNotEqual(was, old, what) + stamp_json.write_text('{"stamp": "2"}\n', encoding="utf-8") + changed = key(HOOKSHOT_KEEP_LIFT="0") + self.assertEqual(changed, before()) + self.assertNotEqual(changed, old) + # (the profile's path was never in it, before or now: LIFT-10) + self.assertEqual(key(HOOKSHOT_KEEP_LIFT="0", RX_VK_PGO="p"), changed) + + def test_source_and_docs(self): + source = (ROOT / "nx2" / "make_game.py").read_text(encoding="utf-8") + body = source[source.index("def lift_key(exefs, work, env, chunks=True):"):source.index("def key_after_lift(")] + for needle in ('key_as_before = os.environ.get("HOOKSHOT_KEEP_LIFT", "1") == "0"', + 'lift_env = {k: v for k, v in lift_env.items() if k.startswith("A64_") or k == "RX_TARGET"}', + 'or (key_as_before and d == HERE and f == "released_translation_stamp.json")'): + self.assertTrue(needle in body, needle) + self.assertTrue('"released_translation_stamp.json"' in source[source.index("LIFT_KEY_NOT = ("):source.index("def lift_key_file(")]) + build = (ROOT / "nx2" / "build.py").read_text(encoding="utf-8") + self.assertTrue("released_translation_stamp.json" in build) # (its one reader) + docs = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + rows = [line for line in docs.splitlines() if line.startswith("| `HOOKSHOT_KEEP_LIFT=0` |")] + self.assertEqual(len(rows), 1) + self.assertTrue("LIFT-27" in rows[0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_lift_key_files.py b/tests/test_lift_key_files.py index a2c2d8a..f4b63a7 100644 --- a/tests/test_lift_key_files.py +++ b/tests/test_lift_key_files.py @@ -137,7 +137,9 @@ class LiftKeyFiles(unittest.TestCase): (exefs / "main").write_bytes(b"game code") names = ["lift.py", "chunks.py", "make_game.py", "build.py", "nup.py", "version.py", "shader_census.py", "prompt_assets.py", "prompt_draw.py", "prompt_faces.py", "prompt_requirements.txt", "prompt_keys.py", - "test_nup_paths.py", "prompt_faces/readme.txt", "code_mods.py"] + "test_nup_paths.py", "prompt_faces/readme.txt", "code_mods.py", + # ((11-22, the performance audit's LIFT-27) the release's shader translation stamp: build.py's alone) + "released_translation_stamp.json"] for n in names: (here / n).write_text("# %s\n" % n, encoding="utf-8") (root / "third_party" / "a64-kit" / "a64_lift.py").write_text("# kit\n", encoding="utf-8") diff --git a/tests/test_lifted_c_scan.py b/tests/test_lifted_c_scan.py new file mode 100644 index 0000000..79c5811 --- /dev/null +++ b/tests/test_lifted_c_scan.py @@ -0,0 +1,218 @@ +"""The scans for the lifted functions' definitions give the patterns' own matches (nx2/lifted_c.py definitions, +noted_starts). + +(Triforce, 11-22, the performance audit's IO-40, after Source A's.) nx2/build.py's memory estimate, nx2/chunks.py's +costs and cuts and nx2/split_hot.py's pieces found the definitions with finditer of a pattern that begins with ^, tried +at every line of 1.5 GB of C. definitions and noted_starts jump from one "\nvoid f_" to the next and try the pattern only +there. What they return decides where the lifted files are cut, so it must be the patterns' answer to the byte. +HOOKSHOT_LIFTED_C_REGEX=1 is the patterns' own finditer, as before. + +Game-free, on made-up C: + - on fixtures with \n and with \r\n line ends (notes above definitions, two notes in a row, a note that does not end + its line, a definition at offset 0, a definition after a note at offset 0, near-miss signatures, text with nothing + in it, text without a final line end) and on 300 fixtures made from a seed: definitions' matches are HEAD's (the + spans and the names), for str and for bytes, and noted_starts is HEAD_NOTED's starts; + - the three callers on a made-up lift, both ways: chunks.text_cost and chunks.split_one write the same parts with the + same costs, split_hot.split writes the same pieces, build.py's mem_need gives the same number; + - the switch is read from the environment and turns both into the patterns' finditer; the four callers ask the scans. + + python -I -B tests/test_lifted_c_scan.py +""" +import ast +import os +from pathlib import Path +import random +import re +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +NX2 = ROOT / "nx2" +sys.path.insert(0, str(NX2)) +import lifted_c # noqa: E402 + +DEF = "void f_%08x(Ctx *__restrict A64_NOESC c) {" +PIECES = [ + DEF % 0x80001000 + "\n c->x[0] = 1;\n}\n", + "/* f_80002000: 3 blocks */\n" + DEF % 0x80002000 + "\n}\n", + "/* one */\n/* two */\n" + DEF % 0x80003000 + "\n /* inside */\n}\n", + "/* not closed\n" + DEF % 0x80004000 + "\n}\n", + "/* closed */ and more\n" + DEF % 0x80005000 + "\n}\n", + "/* a */ b */\n" + DEF % 0x80006000 + "\n}\n", + "/**/\n" + DEF % 0x80007000 + "\n}\n", # (no space inside: not a note) + " " + DEF % 0x80008000 + "\n}\n", # (not at the line's start) + "static " + DEF % 0x80009000 + "\n}\n", + "void f_8000a000(Ctx *__restrict A64_NOESC c);\n", # (a declaration) + "void f_8000B000(Ctx *__restrict A64_NOESC c) {\n}\n", # (not lower-case hex) + "void f_(Ctx *__restrict A64_NOESC c) {\n}\n", + "void f_8000c000(Ctx *restrict c) {\n}\n", + "void g_8000d000(Ctx *__restrict A64_NOESC c) {\n}\n", + "\n\n", + "/* a note with nothing under it */\n\n", + "/* x */\n" + DEF % 0x8000e000 + " /* same line */\n}\n" + DEF % 0x8000f000 + "\n}\n", + "#include \"a64.h\"\nstatic int t;\n", + " f_80001000(c);\n void f_x;\n", +] + + +def made(seed, crlf, final_newline=True): + """A made-up file of the pieces above in a seeded order""" + r = random.Random(seed) + text = "".join(r.choice(PIECES) for _ in range(r.randrange(0, 40))) + if not final_newline: + text = text.rstrip("\n") + return text.replace("\n", "\r\n") if crlf else text + + +def spans(matches): + return [(m.start(), m.end(), m.group(1)) for m in matches] + + +class Scans(unittest.TestCase): + def setUp(self): + self.assertFalse(lifted_c.REGEX_SCAN) + + def same(self, text): + self.assertEqual(spans(lifted_c.definitions(text)), spans(lifted_c.HEAD.finditer(text))) + self.assertEqual(list(lifted_c.noted_starts(text)), [m.start() for m in lifted_c.HEAD_NOTED.finditer(text)]) + data = text.encode() + self.assertEqual(spans(lifted_c.definitions(data)), spans(lifted_c.HEAD_BYTES.finditer(data))) + self.assertEqual(spans(lifted_c.definitions(bytearray(data))), spans(lifted_c.HEAD_BYTES.finditer(data))) + return len(list(lifted_c.definitions(text))) + + def test_fixtures(self): + found = 0 + for piece in PIECES: + for text in (piece, piece + piece, "".join(PIECES), "".join(reversed(PIECES)), piece.rstrip("\n"), "x" + piece): + for crlf in (False, True): + found += self.same(text.replace("\n", "\r\n") if crlf else text) + self.assertGreater(found, 100) + for text in ("", "\n", "void f_", "\nvoid f_", DEF % 1, "\n" + DEF % 2, "/* n */\n" + DEF % 3, "\r\n" + DEF % 4, + "/* n */\r" + DEF % 5, "/* n */\r\n\r\n" + DEF % 6): + self.same(text) + # (the answers themselves, not only that the two agree) + text = "/* note */\n" + DEF % 0x10 + "\n}\n" + DEF % 0x20 + "\n}\n/* a */\n/* b */\r\n" + DEF % 0x30 + "\n}\n" + self.assertEqual([m.group(1) for m in lifted_c.definitions(text)], ["f_00000010", "f_00000020", "f_00000030"]) + self.assertEqual(list(lifted_c.noted_starts(text)), [0, text.index(DEF % 0x20), text.index("/* b */")]) + self.assertEqual([m.start() for m in lifted_c.definitions(text)], [text.index(DEF % n) for n in (0x10, 0x20, 0x30)]) + + def test_seeded_files(self): + found = 0 + for seed in range(300): + found += self.same(made(seed, crlf=seed % 3 == 0, final_newline=seed % 5 != 0)) + self.assertGreater(found, 1000) + + def test_switch(self): + text = "".join(PIECES) + with mock.patch.object(lifted_c, "REGEX_SCAN", True), \ + mock.patch.object(lifted_c, "_NOTE", None): # (the jumps' own pattern is not used at all then) + self.assertEqual(spans(lifted_c.definitions(text)), spans(lifted_c.HEAD.finditer(text))) + self.assertEqual(list(lifted_c.noted_starts(text)), [m.start() for m in lifted_c.HEAD_NOTED.finditer(text)]) + self.assertEqual(spans(lifted_c.definitions(text.encode())), spans(lifted_c.HEAD_BYTES.finditer(text.encode()))) + for value, want in ((None, "False"), ("0", "False"), ("1", "True")): + env = {k: v for k, v in os.environ.items() if k != "HOOKSHOT_LIFTED_C_REGEX"} + if value is not None: + env["HOOKSHOT_LIFTED_C_REGEX"] = value + said = subprocess.run([sys.executable, "-I", "-B", "-c", + "import sys; sys.path.insert(0, sys.argv[1]); import lifted_c; print(lifted_c.REGEX_SCAN)", + str(NX2)], env=env, capture_output=True, text=True) + self.assertEqual(said.stdout.strip(), want, said.stderr) + + +def lift_text(seed, functions): + """A made-up lifted file: a preamble, then functions of mixed sizes that call each other, some with a note above""" + r = random.Random(seed) + names = ["f_%08x" % (0x80000000 + 0x40 * i) for i in range(functions)] + out = ["#include \"a64.h\"\n/* a made-up lift */\n"] + for i, name in enumerate(names): + if r.random() < 0.6: + out.append("/* %s: %d blocks */\n" % (name, r.randrange(1, 9))) + body = "".join(" %s(c);\n" % r.choice(names) if r.random() < 0.3 else " c->x[%d] += %d;\n" % (r.randrange(31), j) + for j in range(r.choice((2, 5, 40, 400)))) + out.append("void %s(Ctx *__restrict A64_NOESC c) {\n%s}\n" % (name, body)) + return "".join(out) + + +class Callers(unittest.TestCase): + """What the scans decide, with the jumps and with the patterns' finditer: the same files and numbers""" + + def both(self, run): + results = [] + for regex in (False, True): + with tempfile.TemporaryDirectory(prefix="lifted-c-scan-") as tmp, mock.patch.object(lifted_c, "REGEX_SCAN", regex): + results.append(run(Path(tmp))) + self.assertEqual(results[0], results[1]) + return results[0] + + @staticmethod + def files(folder): + return {p.name: p.read_bytes() for p in sorted(folder.iterdir())} + + def test_chunks_costs_and_cuts(self): + import kit # noqa: F401 (a64-kit on the path, for chunks) + import chunks + text = lift_text(1, 220) + + def run(tmp): + path = tmp / "code_007.c" + path.write_bytes(text.encode()) + made_, costs, said = chunks.split_one(str(tmp), str(path), 40000) + return chunks.text_cost(text), sorted(os.path.basename(p) for p in made_), costs, said, self.files(tmp) + + cost, parts, costs, said, files = self.both(run) + self.assertGreater(len(parts), 2) # (the file was cut) + self.assertEqual(sum(data.count(b"\nvoid f_") for data in files.values()), 220) # (every function, in one part each) + + def test_split_hot_pieces(self): + import split_hot + text = lift_text(2, 150) + + def run(tmp): + (tmp / "code_hot.c").write_bytes(text.encode()) + with open(os.devnull, "w") as quiet, mock.patch.object(sys, "stdout", quiet): + split_hot.split(str(tmp), 4) + return self.files(tmp) + + files = self.both(run) + pieces = [n for n in files if n.startswith("code_hot_")] + self.assertEqual(len(pieces), 4) + self.assertEqual(sum(files[n].count(b"\nvoid f_") for n in pieces), 150) + + def test_build_memory_estimate(self): + source = (NX2 / "build.py").read_text(encoding="utf-8") + tree = ast.parse(source) + function = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "mem_need"] + self.assertEqual(len(function), 1) + scope = dict(os=os, lifted_c=lifted_c, GIANT_MB=1500, GIANT_BYTES=2000, COMPILER_MB=1500, giants_counted=lambda: True) + exec(compile(ast.Module(body=function, type_ignores=[]), "build.py", "exec"), scope) + text = lift_text(3, 120) + + def run(tmp): + needs = [] + for name in ("code_003.c", "code_hot_1.c"): + (tmp / name).write_bytes(text.encode()) + needs.append(scope["mem_need"](str(tmp / name))) + return needs + + needs = self.both(run) + self.assertTrue(all(n > 300 for n in needs), needs) + + def test_the_callers_ask_the_scans(self): + for name, needle in (("build.py", "starts = [m.start() - 1 for m in lifted_c.definitions(text)] + [len(text)]"), + ("chunks.py", "marks = list(lifted_c.noted_starts(text)) + [len(text)]"), + ("chunks.py", "marks = list(lifted_c.noted_starts(text)) #"), + ("split_hot.py", "marks = list(lifted_c.definitions(text))")): + text = (NX2 / name).read_text(encoding="utf-8") + self.assertTrue(needle in text, "%s: %s" % (name, needle)) + self.assertIsNone(re.search(r"^[^#\n]*HEAD(_BYTES|_NOTED)?\.finditer\(", text, re.M), name) # (outside a comment) + docs = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + rows = [line for line in docs.splitlines() if line.startswith("| `HOOKSHOT_LIFTED_C_REGEX=1` |")] + self.assertEqual(len(rows), 1) + self.assertTrue("IO-40" in rows[0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_linux_pressure.cpp b/tests/test_linux_pressure.cpp new file mode 100644 index 0000000..fd8fcaa --- /dev/null +++ b/tests/test_linux_pressure.cpp @@ -0,0 +1,140 @@ +// (Triforce, 11-22, the performance audit's MEM-141, after Source A's) gpu/vk/linux_pressure.h: the swap-aware memory +// pressure hold's rule. No game, no GPU, no Linux: tests/test_linux_pressure.py compiles and runs this. +#include "linux_pressure.h" + +#include + +static int checks, fails; +#define CHECK(x) do { checks++; if (!(x)) { fails++; printf("FAIL line %d: %s\n", __LINE__, #x); } } while (0) + +using namespace linux_pressure; + +static Sample at(long long avail_mb, long long swap_mb) { // (swap_mb < 0: the swap not known) + Sample s; + s.avail_known = avail_mb >= 0, s.avail = avail_mb >= 0 ? (uint64_t)avail_mb * kMB : 0; + s.swap_known = swap_mb >= 0, s.swap_free = swap_mb >= 0 ? (uint64_t)swap_mb * kMB : 0; + return s; +} + +int main() { + const uint64_t R = kSourceReserve; // (2560 MB) + CHECK(R == 2560 * kMB && kAvailLine == 1536 * kMB && kClear == 256 * kMB && kSampleEvery == 30); + + // --- the switch --- + CHECK(reserve_of(nullptr) == 0 && reserve_of("") == 0 && reserve_of("0") == 0 && reserve_of("x") == 0 && reserve_of("-5") == 0); + CHECK(reserve_of("2560") == 2560 * kMB && reserve_of("1024") == 1024 * kMB); + + // --- /proc/meminfo --- + const char* usual = + "MemTotal: 16252084 kB\nMemFree: 412300 kB\nMemAvailable: 8388608 kB\nBuffers: 10000 kB\n" + "Cached: 3000000 kB\nSwapCached: 123456 kB\nActive: 5000000 kB\nSwapTotal: 8388604 kB\n" + "SwapFree: 6291456 kB\nZswap: 200000 kB\n"; + Meminfo m = parse(usual); + CHECK(m.has_total && m.has_avail && m.has_swap_total && m.has_swap_free); + CHECK(m.total == 16252084ull << 10 && m.avail == 8388608ull << 10 && m.swap_total == 8388604ull << 10 && m.swap_free == 6291456ull << 10); + Sample s = sample_of(m); + CHECK(s.avail_known && s.swap_known && s.avail == 8192 * kMB && s.swap_free == 6144 * kMB); + CHECK(shortfall(s, R) == 0 && clear(s, R)); + + // no swap at all: SwapTotal 0. Not known, never short by the sum + m = parse("MemTotal: 16252084 kB\nMemAvailable: 2048000 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\n"); + s = sample_of(m); + CHECK(m.has_swap_total && m.has_swap_free && s.avail_known && !s.swap_known); + CHECK(shortfall(s, R) == 0 && clear(s, R)); // (2000 MB available: over the 1536 line and its 256) + // ... but MemAvailable's own line still counts there + s = sample_of(parse("MemTotal: 16252084 kB\nMemAvailable: 1048576 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\n")); + CHECK(!s.swap_known && shortfall(s, R) == 512 * kMB && !clear(s, R)); + // the swap lines missing, or SwapFree alone missing: not known + s = sample_of(parse("MemTotal: 16252084 kB\nMemAvailable: 2048000 kB\n")); + CHECK(s.avail_known && !s.swap_known && shortfall(s, R) == 0); + s = sample_of(parse("MemAvailable: 2048000 kB\nSwapTotal: 8388604 kB\n")); + CHECK(!s.swap_known && shortfall(s, R) == 0); + // SwapCached is not SwapFree; MemFree is not MemAvailable + m = parse("MemTotal: 16252084 kB\nMemFree: 100 kB\nSwapCached: 5 kB\n"); + CHECK(m.has_total && !m.has_avail && !m.has_swap_total && !m.has_swap_free); + // no MemAvailable (an old kernel): not known, never short, whatever the swap says + s = sample_of(parse("MemTotal: 16252084 kB\nMemFree: 100 kB\nSwapTotal: 8388604 kB\nSwapFree: 0 kB\n")); + CHECK(!s.avail_known && s.swap_known && shortfall(s, R) == 0 && clear(s, R)); + // nothing read at all, or not numbers + s = sample_of(parse("")); + CHECK(!s.avail_known && !s.swap_known && shortfall(s, R) == 0); + m = parse("MemAvailable: lots\nSwapFree:\n"); + CHECK(!m.has_avail && !m.has_swap_free); + // swap nearly full: MemAvailable looks fine, the sum does not + s = sample_of(parse("MemTotal: 16252084 kB\nMemAvailable: 2048000 kB\nSwapTotal: 8388604 kB\nSwapFree: 102400 kB\n")); + CHECK(s.avail == 2000 * kMB && s.swap_free == 100 * kMB && shortfall(s, R) == 460 * kMB && !clear(s, R)); + // the lab's played values, over what the file says + s = sample_of(parse(usual), 900, -1); + CHECK(s.avail == 900 * kMB && s.swap_free == 6144 * kMB && shortfall(s, R) == 636 * kMB); + s = sample_of(parse(usual), 2000, 100); + CHECK(shortfall(s, R) == 460 * kMB); + s = sample_of(parse("MemTotal: 1 kB\nMemAvailable: 2048000 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\n"), -1, 50); + CHECK(s.swap_known && shortfall(s, R) == 510 * kMB); // (a played swap counts as known) + + // --- how short --- + CHECK(shortfall(at(1535, -1), R) == 1 * kMB && shortfall(at(1536, -1), R) == 0); + CHECK(shortfall(at(1000, 100), R) == 1460 * kMB); // (the larger: 536 by MemAvailable, 1460 by the sum) + CHECK(shortfall(at(1000, 4000), R) == 536 * kMB); + CHECK(shortfall(at(2000, 560), R) == 0 && shortfall(at(2000, 559), R) == 1 * kMB); + CHECK(shortfall(at(500, 0), 0) == 0 && clear(at(500, 0), 0)); // (no reserve: no hold at all) + CHECK(!clear(at(1791, 9000), R) && clear(at(1792, 9000), R)); + CHECK(!clear(at(2000, 815), R) && clear(at(2000, 816), R) && clear(at(1792, -1), R)); + + // --- the hold --- + { + Hold h; + CHECK(h.step(at(8000, 6000), R) == Hold::Same && !h.held && h.episode_want() == 0 && h.cut == 0); + CHECK(h.step(at(2000, 100), R) == Hold::Entered && h.held && h.episodes == 1); + CHECK(h.now == 460 * kMB && h.deepest == 460 * kMB && h.episode_want() == 460 * kMB && h.cut == 460 * kMB); + CHECK(h.step(at(1800, 0), R) == Hold::Same && h.deepest == 760 * kMB && h.cut == 760 * kMB); + h.gave(300 * kMB); + CHECK(h.episode_want() == 460 * kMB); + // better, not clear yet (within 256 MB of the lines): still held, nothing lacking now + CHECK(h.step(at(2000, 700), R) == Hold::Same && h.held && h.now == 0 && h.deepest == 760 * kMB); + CHECK(h.cut == 504 * kMB); // (back up 256 MB a sample) + h.gave(500 * kMB); + CHECK(h.episode_want() == 0); // (the episode has given back what it may: 800 of 760) + // short again within the episode: the limit is the worst so far, less what went + CHECK(h.step(at(1700, 0), R) == Hold::Same && h.deepest == 860 * kMB && h.episode_want() == 60 * kMB && h.cut == 860 * kMB); + // clear of both lines by 256 MB: the hold ends; the cut keeps coming back + CHECK(h.step(at(2100, 716), R) == Hold::Left && !h.held && h.episode_want() == 0 && h.now == 0); + CHECK(h.cut == 604 * kMB); + CHECK(h.step(at(4000, 4000), R) == Hold::Same && h.cut == 348 * kMB); + CHECK(h.step(at(4000, 4000), R) == Hold::Same && h.cut == 92 * kMB); + CHECK(h.step(at(4000, 4000), R) == Hold::Same && h.cut == 0); + // a new episode starts from nothing + CHECK(h.step(at(1400, -1), R) == Hold::Entered && h.episodes == 2 && h.deepest == 136 * kMB && h.given == 0); + CHECK(h.episode_want() == 136 * kMB); + // MemAvailable no longer readable: not known is not short, the hold ends + CHECK(h.step(at(-1, 0), R) == Hold::Left && !h.held); + } + { // no reserve: never held, whatever the PC says + Hold h; + for (int i = 0; i < 5; i++) CHECK(h.step(at(100, 0), 0) == Hold::Same && !h.held && h.cut == 0 && h.episode_want() == 0); + CHECK(h.episodes == 0); + } + { // swapless PC: only MemAvailable's line, with its own 256 MB to clear + Hold h; + CHECK(h.step(at(1600, -1), R) == Hold::Same && !h.held); + CHECK(h.step(at(1500, -1), R) == Hold::Entered && h.now == 36 * kMB); + CHECK(h.step(at(1700, -1), R) == Hold::Same && h.held); + CHECK(h.step(at(1792, -1), R) == Hold::Left); + } + + // --- the valves --- + CHECK(spare_cap(false, 768 * kMB) == 768 * kMB && spare_cap(true, 768 * kMB) == 32 * kMB && spare_cap(true, 16 * kMB) == 16 * kMB); + CHECK(arena_idle_s(false, 30) == 30 && arena_idle_s(true, 30) == 5 && arena_idle_s(true, 2) == 2 && arena_idle_s(true, 0) == 0); + CHECK(trim_every(false, 1800) == 1800 && trim_every(true, 1800) == 600 && trim_every(true, 300) == 300); + CHECK(trim_mb(false, 192) == 192 && trim_mb(true, 192) == 64 && trim_mb(true, 32) == 32 && trim_mb(true, 0) == 0); + CHECK(cut_budget(2560 * kMB, 0) == 2560 * kMB && cut_budget(2560 * kMB, 460 * kMB) == 2100 * kMB); + CHECK(cut_budget(2560 * kMB, 2500 * kMB) == 512 * kMB && cut_budget(2560 * kMB, 9000 * kMB) == 512 * kMB); + CHECK(cut_budget(400 * kMB, 300 * kMB) == 400 * kMB && cut_budget(512 * kMB, 100 * kMB) == 512 * kMB); // (never raised, never under the floor) + for (uint64_t cut = 0; cut <= 4096 * kMB; cut += 64 * kMB) { + const uint64_t b = cut_budget(2560 * kMB, cut); + CHECK(b <= 2560 * kMB && b >= 512 * kMB); + } + + if (fails) printf("linux pressure: FAILED (%d of %d checks)\n", fails, checks); + else printf("linux pressure: all %d checks passed\n", checks); + return fails != 0; +} diff --git a/tests/test_linux_pressure.py b/tests/test_linux_pressure.py new file mode 100644 index 0000000..80f8692 --- /dev/null +++ b/tests/test_linux_pressure.py @@ -0,0 +1,242 @@ +"""A swap-aware memory pressure hold for Linux and the Steam Deck (gpu/vk/linux_pressure.h; no game, no GPU, no Linux). + +(Triforce, 11-22, the performance audit's MEM-141, after Source A's.) The RAM rules looked at MemAvailable alone, which a +PC with swap keeps up by pushing the game's pages out. With RX_VK_SWAP_RESERVE_MB set the PC is also short when +MemAvailable and SwapFree together are under that reserve, and while it is short a hold gives memory back sooner: +capture spares, idle staging arenas, malloc_trim, idle imports (at most the episode's worst shortfall), an integrated +GPU's texture budget. Off unless the switch is set; never on Windows. + +Compiles and runs tests/test_linux_pressure.cpp against the header: + /proc/meminfo's text: the usual one, no swap (not known, never short by the sum), no MemAvailable (never short), swap + nearly full (short while MemAvailable looks fine), SwapCached and MemFree not taken for the fields, the lab's played + values; + how short, the 256 MB to clear, the hold's entry and exit, the episode's limit for the imports and what was given, + a new episode, a swapless PC, no reserve (never held), the budget's cut down at once and back 256 MB a sample; + the valves' sizes held and not held. +Cuts the hold's own code (its state, swap_hold_step and the calls the valves make) out of gpu/vk/vk_memory.cpp and runs +it over a meminfo file of the test's and a log stand-in, each case a process of its own (the switch is read once): + the reserve not set: /proc/meminfo is never opened, nothing is held; + set: looked at every 30 presents only; the hold on at a short sample with its line, memory_swap_hold and + memory_swap_cut following, the imports' limit counted down by memory_swap_gave, off at a clear sample with the + line that says what each valve gave; RX_VK_TEST_SWAP_MB and RX_VK_TEST_AVAIL_MB played over the file. +And reads the source: memory_ram_watch steps the hold off Windows only; cap_buf_free, imports_give_back, the staging +arenas, malloc_trim and the integrated GPU's budget each ask the hold through the header's rule; the switches table. + + python -I -B tests/test_linux_pressure.py [--cxx clang++] +""" +import argparse +import os +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +FIXTURE = r'''#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "linux_pressure.h" + +static int checks, fails; +static std::string logged; +#define CHECK(x) do { checks++; if (!(x)) { fails++; printf("FAIL line %d: %s (logged: %s)\n", __LINE__, #x, logged.c_str()); } } while (0) +static void logf(const char* fmt, ...) { + char line[1024]; + va_list ap; + va_start(ap, fmt); + vsnprintf(line, sizeof line, fmt, ap); + va_end(ap); + logged += line, logged += "\n"; +} +static const char* t_meminfo; // (the file that plays /proc/meminfo) +static int t_opens; +static FILE* t_fopen(const char* path, const char* mode) { + if (strcmp(path, "/proc/meminfo")) printf("FAIL: opened %s\n", path), exit(1); + t_opens++; + return fopen(t_meminfo, mode); +} +#define fopen t_fopen +@CUT@ +#undef fopen +static void meminfo(long long avail_mb, long long swap_total_mb, long long swap_free_mb) { + FILE* f = fopen(t_meminfo, "w"); + if (!f) printf("FAIL: cannot write %s\n", t_meminfo), exit(1); + fprintf(f, "MemTotal: 16252084 kB\nMemFree: 100000 kB\nMemAvailable: %lld kB\nSwapCached: 1000 kB\n" + "SwapTotal: %lld kB\nSwapFree: %lld kB\n", avail_mb * 1024, swap_total_mb * 1024, swap_free_mb * 1024); + fclose(f); +} +static bool said(const char* what) { return logged.find(what) != std::string::npos; } +static const uint64_t MB = 1ull << 20; + +int main(int argc, char** argv) { + if (argc < 3) return 2; + const std::string mode = argv[1]; + t_meminfo = argv[2]; + if (mode == "off") { // RX_VK_SWAP_RESERVE_MB not set (or 0): nothing is read, nothing is ever held + meminfo(200, 8192, 0); + for (uint64_t p = 0; p <= 600; p++) swap_hold_step(p); + CHECK(t_opens == 0 && !memory_swap_hold() && memory_swap_cut() == 0 && memory_swap_episode_want() == 0 && logged.empty()); + } else if (mode == "on") { // RX_VK_SWAP_RESERVE_MB=2560 + meminfo(8000, 8192, 6000); + for (uint64_t p = 1; p < 30; p++) swap_hold_step(p); + CHECK(t_opens == 0); // (every 30 presents only) + swap_hold_step(30); + CHECK(t_opens == 1 && !memory_swap_hold() && logged.empty()); + meminfo(2000, 8192, 100); // swap nearly full: 460 MB short of the reserve + swap_hold_step(45); + CHECK(t_opens == 1 && !memory_swap_hold()); + swap_hold_step(60); + CHECK(t_opens == 2 && memory_swap_hold() && memory_swap_cut() == 460 * MB && memory_swap_episode_want() == 460 * MB); + CHECK(said("vk: swap hold: on (2000 MB available, 100 MB of swap free: 460 MB short of 1536 MB available or 2560 MB with the swap)")); + memory_swap_gave(200 * MB); // (imports given back: counted against the episode's limit) + memory_swap_valve(1, 128 * MB); // (staging arenas) + memory_swap_valve(2, 64 * MB); // (capture spares) + memory_swap_valve(7, 1 * MB); // (no such valve: nothing) + CHECK(memory_swap_episode_want() == 260 * MB); + meminfo(1800, 8192, 0); // worse: 760 MB short + swap_hold_step(90); + CHECK(memory_swap_hold() && memory_swap_cut() == 760 * MB && memory_swap_episode_want() == 560 * MB); + memory_swap_gave(600 * MB); + CHECK(memory_swap_episode_want() == 0); + meminfo(2000, 8192, 700); // over the lines, not by 256 MB: still held + swap_hold_step(120); + CHECK(memory_swap_hold() && memory_swap_cut() == 504 * MB && !said("swap hold: off")); + meminfo(4000, 8192, 4000); // clear + swap_hold_step(150); + CHECK(!memory_swap_hold() && memory_swap_cut() == 248 * MB && memory_swap_episode_want() == 0); + CHECK(said("vk: swap hold: off after ") && said("(4000 MB available, 4000 MB of swap free; 760 MB short at its worst); given back: " + "imports 800 MB, staging arenas 128 MB, capture spares not kept 64 MB")); + swap_hold_step(180); + CHECK(memory_swap_cut() == 0); + // a swapless PC: only MemAvailable's own line + logged.clear(); + meminfo(1400, 0, 0); + swap_hold_step(210); + CHECK(memory_swap_hold() && memory_swap_cut() == 136 * MB && said("(1400 MB available, swap not known: 136 MB short")); + CHECK(memory_swap_episode_want() == 136 * MB); // (a new episode: nothing given yet) + meminfo(2000, 0, 0); + swap_hold_step(240); + CHECK(!memory_swap_hold() && said("imports 0 MB, staging arenas 0 MB, capture spares not kept 0 MB")); + } else if (mode == "played") { // RX_VK_SWAP_RESERVE_MB=2560 RX_VK_TEST_SWAP_MB=100 RX_VK_TEST_AVAIL_MB=2000 + meminfo(9000, 0, 0); // (the file says plenty, and no swap: the played values are over it) + swap_hold_step(30); + CHECK(memory_swap_hold() && memory_swap_cut() == 460 * MB && said("(2000 MB available, 100 MB of swap free: 460 MB short")); + } else return 2; + if (fails) printf("linux pressure, the hold (%s): FAILED (%d of %d checks)\n", mode.c_str(), fails, checks); + else printf("linux pressure, the hold (%s): all %d checks passed\n", mode.c_str(), checks); + return fails != 0; +} +''' + + +def block(text, marker, what): + start = text.find(marker) + if start < 0: + raise SystemExit("FAIL: %s has no '%s'" % (what, marker)) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def in_order(body, needles, what): + order = [body.find(n) for n in needles] + if min(order) < 0 or order != sorted(order): + raise SystemExit("FAIL: %s (%r)" % (what, order)) + + +def check_source(): + mem = (ROOT / "gpu/vk/vk_memory.cpp").read_text(encoding="utf-8") + tex = (ROOT / "gpu/vk/vk_texture.cpp").read_text(encoding="utf-8") + api = (ROOT / "gpu/vk/vk_internal.h").read_text(encoding="utf-8") + for name, text in (("vk_memory.cpp", mem), ("vk_texture.cpp", tex)): + if '#include "linux_pressure.h"' not in text: + raise SystemExit("FAIL: %s does not include linux_pressure.h" % name) + for decl in ("bool memory_swap_hold();", "uint64_t memory_swap_cut();", "void memory_swap_valve(int valve, uint64_t bytes);", + "uint64_t memory_swap_episode_want();", "void memory_swap_gave(uint64_t bytes);"): + if decl not in api: + raise SystemExit("FAIL: vk_internal.h does not declare " + decl) + # the hold's own code: from its state to the last of the calls the valves make + first, last = "namespace {\nstd::mutex swap_hold_mu;\n", "void memory_swap_gave(uint64_t bytes) {" + if mem.count(first) != 1 or mem.count(last) != 1: + raise SystemExit("FAIL: vk_memory.cpp: the swap hold's code was not found") + hold = mem[mem.index(first):mem.index(last)] + block(mem, last, "vk_memory.cpp") + step = block(hold, "void swap_hold_step(uint64_t presents) {", "the hold") + in_order(step, ['static const uint64_t reserve = linux_pressure::reserve_of(getenv("RX_VK_SWAP_RESERVE_MB"));', + "if (!reserve || presents % linux_pressure::kSampleEvery) return;", + 'getenv("RX_VK_TEST_AVAIL_MB")', 'getenv("RX_VK_TEST_SWAP_MB")', 'fopen("/proc/meminfo", "r")', + "linux_pressure::sample_of(linux_pressure::parse(text), test_avail, test_swap);", + "std::lock_guard lk(swap_hold_mu);", "swap_hold.step(s, reserve);"], + "swap_hold_step does not read its switch once, look every 30 presents, read SwapFree with MemAvailable from " + "/proc/meminfo (the played values over it) and step the hold under its lock") + watch = block(mem, "void memory_ram_watch(uint64_t presents) {", "vk_memory.cpp") + in_order(watch, ["#ifndef _WIN32\n swap_hold_step(presents);", "#endif\n if (presents % 60) return;"], + "memory_ram_watch does not step the hold off Windows only, before its own look every 60 presents") + if mem.count("swap_hold_step(presents);") != 1: + raise SystemExit("FAIL: the hold is stepped from more than one place (Windows keeps its own rules)") + # the valves + free = block(mem, "void cap_buf_free(VkBuffer buffer, VmaAllocation alloc, void* map, uint64_t size, bool from_worker = false) {", + "vk_memory.cpp") + in_order(free, ["const bool swap_hold_on = memory_swap_hold();", + # ((Triforce, 11-22) merged over MEM-33: its rule takes the hold's cap and the smaller of that and 768 MB) + "linux_pressure::spare_cap(swap_hold_on, cap_spare_max()), keep_old)) {", + "if (swap_hold_on) memory_swap_valve(2, size);"], + "cap_buf_free does not cap the capture spares by the hold") + give = block(mem, "void imports_give_back() {", "vk_memory.cpp") + in_order(give, ["const bool by_hold = ram_rule && !by_heap && memory_swap_hold();", + "if (!by_heap && !ram_short && !by_hold) return;", + ": by_hold ? memory_swap_episode_want() : (1ull << 30);", "if (by_hold && !want) return;", + "constexpr uint64_t kImportIdleRam = 3600;", "if (by_hold) memory_swap_gave(freed);"], + "imports_give_back does not take the hold as a RAM trigger with the episode's limit, the idle rule kept") + in_order(tex, ["const bool swap_hold_on = memory_swap_hold();", "const bool swap_hold_began = swap_hold_on && !swap_hold_was;", + "const int64_t idle_now = linux_pressure::arena_idle_s(swap_hold_on, idle_s);", + "const bool stale = idle_now && (swap_hold_began || std::chrono::steady_clock::now() - it->used_at > std::chrono::seconds(idle_now));", + "if (swap_hold_on) memory_swap_valve(1, it->size);", + "if (frame_no % linux_pressure::trim_every(swap_hold_on, 1800) == 0 && !busy.exchange(true))", + "const uint64_t trim_mb = linux_pressure::trim_mb(swap_hold_on, trim_set);", + "if (const uint64_t cut = memory_swap_cut(); cut && !vram_env && dev.props.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU)", + "budget = linux_pressure::cut_budget(budget, cut);"], + "vk_texture.cpp: the staging arenas, malloc_trim and the integrated GPU's budget do not ask the hold") + docs = (ROOT / "docs/development.md").read_text(encoding="utf-8") + if "| `RX_VK_SWAP_RESERVE_MB=`, `RX_VK_TEST_SWAP_MB=` |" not in docs or "`gpu/vk/linux_pressure.h`" not in docs: + raise SystemExit("FAIL: docs/development.md: the switches table has no RX_VK_SWAP_RESERVE_MB row") + return hold + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="g++") + args = parser.parse_args() + hold = check_source() + with tempfile.TemporaryDirectory(prefix="linux-pressure-") as tmp: + tmp = Path(tmp) + flags = [args.cxx, "-std=c++20", "-Wall", "-Wextra", "-Werror", "-pthread", "-I" + str(ROOT / "gpu/vk")] + exe = tmp / "linux-pressure.exe" + subprocess.run(flags + [str(ROOT / "tests/test_linux_pressure.cpp"), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + print(r.stdout.rstrip()) + if r.returncode: + raise SystemExit("FAIL: the pressure rule's checks failed") + src, exe = tmp / "swap_hold_test.cpp", tmp / "swap-hold.exe" + src.write_text(FIXTURE.replace("@CUT@", hold), encoding="utf-8") + subprocess.run(flags + ["-Wno-unused-function", str(src), "-o", str(exe)], check=True) + clean = {k: v for k, v in os.environ.items() if k not in ("RX_VK_SWAP_RESERVE_MB", "RX_VK_TEST_SWAP_MB", "RX_VK_TEST_AVAIL_MB")} + for mode, env in (("off", {}), ("off", {"RX_VK_SWAP_RESERVE_MB": "0"}), ("on", {"RX_VK_SWAP_RESERVE_MB": "2560"}), + ("played", {"RX_VK_SWAP_RESERVE_MB": "2560", "RX_VK_TEST_SWAP_MB": "100", "RX_VK_TEST_AVAIL_MB": "2000"})): + r = subprocess.run([str(exe), mode, str(tmp / "meminfo.txt")], capture_output=True, text=True, env=dict(clean, **env)) + print(r.stdout.rstrip()) + if r.returncode: + raise SystemExit("FAIL: the hold's checks failed (%s, %r)" % (mode, env)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_lock_census.py b/tests/test_lock_census.py new file mode 100644 index 0000000..379d946 --- /dev/null +++ b/tests/test_lock_census.py @@ -0,0 +1,381 @@ +"""The lock census: who holds the lock a thread waits for (gpu/vk/lock_census.h, RX_VK_LOCK_CENSUS=1; no game, no GPU). + +(Triforce, 11-22, the performance audit's THREAD-68, after Source B's.) A busy lock notes its waiter, the thread that held +it and the time waited; a thread of the census's own logs the pairs that waited longest. Off unless set, and off, every +lock is taken exactly as before. + +DrainingMutex (vk_recorder.h: replay_mu) and OrderMutex (vk_replay.cpp: sync_order_mu) are cut out of the sources and +compiled with lock_census.h; the fixture runs as a process a case: + +- on: a free lock reads no clock and counts nothing, and the lock knows its holder while it is held and not after; a + thread that comes for a held lock is counted against the holder for exactly the time it waited (the fixture's own + clock: the holder lets go 50 ms after the waiter's first clock read) under that lock's name, for both locks and for a + CensusMutex under std::lock_guard and std::unique_lock; the report is the 12 longest pairs, longest first, in the log's + form, and starts again after it; the reporting thread starts at the first counted wait and logs; what the locks do + besides is unchanged (the order lock's sections counted, the drain before the replay lock is let go); a storm of four + threads through both locks in the product's order keeps what they guard exact; +- off (the switch not set, and 0): the same held lock and the same storm count nothing, read no clock, start no thread, + and no lock ever notes a holder; +- the lab's phases: with the switch on and the phase off, nothing is counted. + +The sources are read for the rest: the six locks go through the census, the holder is cleared just before each unlock +(after the replay lock's drain and the order lock's section count), and the switch is in the switches table. + + python -I -B tests/test_lock_census.py [--cxx clang++] +""" +import argparse +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" + + +def fail(message): + raise SystemExit("FAIL: " + message) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +FIXTURE = r''' +#include +#include +#include "lock_census.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +namespace mkwvk { +static std::mutex log_mu; +static std::vector logged; +void logf(const char* fmt, ...) { + char line[512]; + va_list ap; + va_start(ap, fmt); + vsnprintf(line, sizeof line, fmt, ap); + va_end(ap); + std::lock_guard lk(log_mu); + logged.emplace_back(line); +} +namespace rec { +// (what DrainingMutex uses of the recorder, standing in) +struct Recorder {}; +static Recorder g_rec; +static thread_local Recorder* t_prod = nullptr; +enum { DEFER = 1, THREAD = 2 }; +static int mode() { return THREAD; } +static bool (*g_direct_hook)() = nullptr; // (the lab's norec knob: not set here) +enum Bound { B_REPLAY_UNLOCK }; +static std::atomic boundaries{0}; +static std::atomic held_at_boundary{true}; +struct DrainingMutex; +static DrainingMutex* watched = nullptr; +static void boundary(Bound); +/* DRAINING */; +// (the drain runs while the lock is still held and still says who holds it) +static void boundary(Bound) { + boundaries++; + if (watched && lock_census::counting() && !watched->owner.load()) held_at_boundary = false; +} +} // namespace rec +static std::atomic order_waiters{0}; +static std::mutex order_wait_mu; +static std::condition_variable order_wait_cv; +/* ORDER */; +/* ORDER_UNLOCK */ +} // namespace mkwvk +using namespace mkwvk; + +// the fixture's clock: it moves only when a case moves it, and counts its reads +static std::atomic clock_now{1000000000000LL}; +static std::atomic clock_reads{0}; +static int64_t test_clock() { + clock_reads++; + return clock_now.load(); +} +// (a thread's name as the renderer would read its description: lock_census.h has no system call of its own) +static thread_local const char* my_name = ""; +static void name_thread(const char* name) { my_name = name; } +static void read_name(char* out, size_t cap) { snprintf(out, cap, "%s", my_name); } +static std::atomic report_starts{0}; +static size_t pairs() { + lock_census::Book& b = lock_census::book(); + std::lock_guard lk(b.mu); + return b.pairs.size(); +} +static bool reporter() { + lock_census::Book& b = lock_census::book(); + std::lock_guard lk(b.mu); + return b.reporter; +} +// A holder has the lock; a waiter comes for it; the holder lets go 50 ms of the fixture's clock after the waiter's first clock +// read (which it makes only when it finds the lock busy and the census counts). Without the census the waiter reads no clock, so +// the holder lets go once the waiter is on its way. Returns the clock reads the waiter made +template +static int held_and_waited(M& m, bool counted) { + std::atomic stage{0}; + const int reads0 = clock_reads.load(); + std::thread holder([&] { + name_thread("holder"); + m.lock(); + stage = 1; + while (stage.load() < 2) std::this_thread::yield(); + if (counted) { + while (clock_reads.load() == reads0) std::this_thread::yield(); // (the waiter found it busy and read the clock) + clock_now += 50000000; + } else { + std::this_thread::sleep_for(std::chrono::milliseconds(30)); // (long enough for the waiter to be at the lock) + } + m.unlock(); + }); + while (stage.load() < 1) std::this_thread::yield(); + std::thread waiter([&] { + name_thread("waiter"); + stage = 2; + m.lock(); + m.unlock(); + }); + holder.join(), waiter.join(); + return clock_reads.load() - reads0; +} +static void expect_line(const char* lock) { + // (10 s after the census's count began: 50 ms waited in 10 s is 5.0 ms/s, a tenth of a wait a second) + int64_t since = 0; + { + lock_census::Book& b = lock_census::book(); + std::lock_guard lk(b.mu); + since = b.since_ns; + } + const std::vector lines = lock_census::report(since + 10000000000LL); + CHECK(lines.size() == 1); + if (lines.size() != 1) return; + const std::string want = std::string("vk: lock census: waiter <- holder (") + lock + ") 5.0 ms/s (0/s, 50000 us each)"; + if (lines[0] != want) { std::printf("FAIL: the line is\n %s\nnot\n %s\n", lines[0].c_str(), want.c_str()); fails++; } + CHECK(pairs() == 0 && lock_census::report(since + 20000000000LL).empty()); // (the counts start again) +} +// four threads through the order lock and then the replay lock, as the product takes them: what they guard stays exact +template +static void storm(O& order, R& replay) { + long guarded = 0; + std::vector threads; + for (int t = 0; t < 4; t++) + threads.emplace_back([&] { + for (int i = 0; i < 20000; i++) { + std::lock_guard a(order); + std::lock_guard b(replay); + guarded = guarded + 1; + } + }); + for (auto& t : threads) t.join(); + CHECK(guarded == 4 * 20000); +} + +int main(int argc, char** argv) { + const std::string c = argc > 1 ? argv[1] : ""; + lock_census::now_ns = test_clock; + lock_census::g_thread_name = read_name; + lock_census::g_report_start = [] { report_starts++; }; + lock_census::period_ms = 3600000; // (the reporting thread stays out of the scripted checks until it is asked for) + static rec::DrainingMutex replay_mu; + static OrderMutex sync_order_mu; + static lock_census::CensusMutex held_mu{"held_mu"}; + rec::watched = &replay_mu; + if (c == "on") { + CHECK(lock_census::enabled() && lock_census::counting()); + // free locks: no clock, nothing counted, the holder known only while held + for (int i = 0; i < 1000; i++) { + replay_mu.lock(); + CHECK(replay_mu.owner.load() == lock_census::me()); + replay_mu.unlock(); + sync_order_mu.lock(); + CHECK(sync_order_mu.owner.load() == lock_census::me() && sync_order_mu.entered.load() - sync_order_mu.left.load() == 1); + sync_order_mu.unlock(); + { std::lock_guard lk(held_mu); CHECK(held_mu.owner.load() == lock_census::me()); } + CHECK(replay_mu.try_lock() && replay_mu.owner.load() == lock_census::me()); + replay_mu.unlock(); + CHECK(sync_order_mu.try_lock() && sync_order_mu.owner.load() == lock_census::me()); + sync_order_mu.unlock(); + std::unique_lock ul(held_mu, std::try_to_lock); + CHECK(ul.owns_lock()); + } + CHECK(!replay_mu.owner.load() && !sync_order_mu.owner.load() && !held_mu.owner.load()); + CHECK(clock_reads.load() == 0 && pairs() == 0 && !reporter()); + CHECK(sync_order_mu.entered.load() == 2000 && sync_order_mu.left.load() == 2000); // (the sections counted as before) + CHECK(rec::boundaries.load() == 2000 && rec::held_at_boundary.load()); // (the drain at every unlock, the lock still held) + CHECK(rec::t_prod == nullptr); + // a held lock: the wait counted against the holder, for the time waited, under the lock's name + CHECK(held_and_waited(replay_mu, true) == 2); // (two clock reads: before the wait and after it) + CHECK(reporter()); // (the first counted wait starts the reporting thread) + for (int i = 0; i < 500 && !report_starts.load(); i++) std::this_thread::sleep_for(std::chrono::milliseconds(10)); + CHECK(report_starts.load() == 1); + expect_line("replay_mu"); + CHECK(held_and_waited(sync_order_mu, true) == 2); + expect_line("sync_order_mu"); + CHECK(held_and_waited(held_mu, true) == 2); + expect_line("held_mu"); + // a busy try_lock is not a wait + { + std::lock_guard lk(held_mu); + std::thread([&] { CHECK(!held_mu.try_lock()); }).join(); + } + CHECK(pairs() == 0); + // the report: the 12 longest of 20 pairs, longest first + { + static lock_census::Slot holders[20]; + lock_census::Book& b = lock_census::book(); + for (int i = 0; i < 20; i++) { + char name[16]; + snprintf(name, sizeof name, "h%02d", i); + { std::lock_guard lk(b.mu); holders[i].name = lock_census::intern(b, name); } + lock_census::waited(lock_census::me(), &holders[i], "pools_mu", (i + 1) * 1000000LL, clock_now.load()); + lock_census::waited(lock_census::me(), &holders[i], "pools_mu", (i + 1) * 1000000LL, clock_now.load()); + } + lock_census::waited(lock_census::me(), nullptr, "cap_mu", 500, clock_now.load()); // (a holder not known) + CHECK(pairs() == 21); + const std::vector lines = lock_census::report(clock_now.load() + 100000000000LL); + CHECK(lines.size() == 12); + for (size_t i = 0; i < lines.size(); i++) { + char want[64]; + snprintf(want, sizeof want, " <- h%02d (pools_mu) ", 19 - (int)i); + CHECK(lines[i].find(want) != std::string::npos); + } + CHECK(pairs() == 0); + } + // the reporting thread logs what is counted (the fixture's clock stands still: it reports by its own sleep) + clock_now += 200000000000LL; + lock_census::waited(lock_census::me(), nullptr, "sync_mu", 7000000, clock_now.load()); + clock_now += 1000000000; + lock_census::period_ms = 20; + bool told = false; + for (int i = 0; i < 500 && !told; i++) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + std::lock_guard lk(log_mu); + for (const std::string& line : logged) told = told || line.find("(sync_mu)") != std::string::npos; + } + CHECK(told); + storm(sync_order_mu, replay_mu); + CHECK(!replay_mu.owner.load() && !sync_order_mu.owner.load()); + CHECK(sync_order_mu.entered.load() == sync_order_mu.left.load()); + } else if (c == "off") { + CHECK(!lock_census::enabled() && !lock_census::counting()); + CHECK(held_and_waited(replay_mu, false) == 0); + CHECK(held_and_waited(sync_order_mu, false) == 0); + CHECK(held_and_waited(held_mu, false) == 0); + replay_mu.lock(); + CHECK(!replay_mu.owner.load()); // (off: a lock notes no holder) + replay_mu.unlock(); + storm(sync_order_mu, replay_mu); + CHECK(clock_reads.load() == 0 && pairs() == 0 && !reporter() && report_starts.load() == 0); + CHECK(!replay_mu.owner.load() && !sync_order_mu.owner.load() && !held_mu.owner.load()); + CHECK(sync_order_mu.entered.load() == sync_order_mu.left.load() && sync_order_mu.entered.load() > 80000); + CHECK(rec::boundaries.load() > 80000); + } else if (c == "phase") { // (the switch on, the lab's phase off) + CHECK(lock_census::enabled()); + lock_census::phase_on = false; + CHECK(!lock_census::counting()); + CHECK(held_and_waited(replay_mu, false) == 0); + storm(sync_order_mu, replay_mu); + CHECK(clock_reads.load() == 0 && pairs() == 0 && !reporter()); + lock_census::phase_on = true; + replay_mu.lock(); + CHECK(replay_mu.owner.load() == lock_census::me()); // (and counting again in the next phase) + replay_mu.unlock(); + } else { + std::printf("FAIL: unknown case\n"); + fails++; + } + if (!fails) std::printf("PASS %s\n", c.c_str()); + std::fflush(stdout); + std::_Exit(fails ? 1 : 0); // (the reporting thread sleeps on: no wait for it) +} +''' + + +def read_sources(recorder, replay, memory, dev): + if '#include "lock_census.h"' not in recorder or '#include "lock_census.h"' not in memory: + fail("vk_recorder.h or vk_memory.cpp does not include lock_census.h") + draining = block(recorder, "struct DrainingMutex {") + if 'lock_census::lock(m, owner, "replay_mu");' not in draining or "if (!lock_census::try_lock(m, owner)) return false;" not in draining: + fail("DrainingMutex does not take its mutex through the census") + if not draining.index("boundary(B_REPLAY_UNLOCK);") < draining.index("lock_census::unlocking(owner);") < draining.index("m.unlock();"): + fail("DrainingMutex: the holder is not cleared after the drain and just before the unlock") + order = block(replay, "struct OrderMutex {") + if 'lock_census::lock(mu, owner, "sync_order_mu");' not in order or "if (!lock_census::try_lock(mu, owner)) return false;" not in order: + fail("OrderMutex does not take its mutex through the census") + unlock = block(replay, "void OrderMutex::unlock() {") + if not unlock.index("left.fetch_add(1, std::memory_order_seq_cst);") < unlock.index("lock_census::unlocking(owner);") < unlock.index("mu.unlock();"): + fail("OrderMutex::unlock: the holder is not cleared after the section is counted and just before the unlock") + if 'lock_census::CensusMutex held_mu{"held_mu"};' not in replay: + fail("vk_replay.cpp: held_mu is not counted") + srw = block(memory, "struct SrwMutex {") + if 'lock_census::lock(raw, owner, "pools_mu");' not in srw or "bool try_lock() { return TryAcquireSRWLockExclusive(l) != 0; }" not in srw or \ + not srw.index("lock_census::unlocking(owner);") < srw.index("ReleaseSRWLockExclusive(&l);"): + fail("vk_memory.cpp SrwMutex: pools_mu is not tried first and counted, or its holder is not cleared before the release") + if 'static lock_census::CensusMutex sync_mu{"sync_mu"};' not in memory or "std::lock_guard sync_lk(sync_mu);" not in memory: + fail("vk_memory.cpp: sync_mu is not counted") + if 'lock_census::CensusMutex cap_mu{"cap_mu"};' not in memory: + fail("vk_memory.cpp: cap_mu is not counted") + if "lock_census::g_thread_name = [](char* out, size_t cap) {" not in replay or "GetThreadDescription(GetCurrentThread(), &w)" not in replay: + fail("vk_replay.cpp no longer gives the census the threads' descriptions: every waiter and holder would be a number") + if "#include " in (VK / "lock_census.h").read_text(encoding="utf-8"): + fail("lock_census.h includes windows.h: vk_recorder.h, which includes it, is built without") + if 'occ::knob_names(lab_ab::knobs[i].name, "lockcensus")' not in replay or 'lock_census::phase_on.store(lab_ab::on("lockcensus")' not in replay: + fail("vk_replay.cpp: the lab's knob no longer sets the census's phases") + header = (VK / "lock_census.h").read_text(encoding="utf-8") + lock = block(header, "inline void lock(M& m, std::atomic& owner, const char* name) {") + off = lock.index("if (!counting()) {") + if not off < lock.index("m.lock();") < lock.index("return;") < lock.index("if (m.try_lock()) {"): + fail("lock_census.h: with the census off a lock is not taken by its own lock() alone, before anything else") + if "| `RX_VK_LOCK_CENSUS=1` |" not in dev: + fail("docs/development.md: RX_VK_LOCK_CENSUS is not in the switches table") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + try: + recorder = (VK / "vk_recorder.h").read_text(encoding="utf-8") + replay = (VK / "vk_replay.cpp").read_text(encoding="utf-8") + memory = (VK / "vk_memory.cpp").read_text(encoding="utf-8") + read_sources(recorder, replay, memory, (ROOT / "docs/development.md").read_text(encoding="utf-8")) + source = FIXTURE.replace("/* DRAINING */", block(recorder, "struct DrainingMutex {")) \ + .replace("/* ORDER */", block(replay, "struct OrderMutex {")) \ + .replace("/* ORDER_UNLOCK */", block(replay, "void OrderMutex::unlock() {")) + except (ValueError, OSError) as e: + fail("a text this test reads is not there (%s)" % e) + env = {k: v for k, v in os.environ.items() if k != "RX_VK_LOCK_CENSUS"} + with tempfile.TemporaryDirectory(prefix="lock-census-") as tmp: + src, exe = Path(tmp) / "census.cpp", Path(tmp) / "census.exe" + src.write_text(source, encoding="utf-8") + built = subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-Wno-unused-function", "-Wno-unused-variable", + "-pthread", "-I" + str(VK), str(src), "-o", str(exe)], capture_output=True, text=True) + if built.returncode: + print(built.stdout + built.stderr) + fail("the fixture did not build") + for case, switch in (("on", "1"), ("on", "2"), ("off", None), ("off", "0"), ("phase", "1")): + case_env = dict(env) if switch is None else dict(env, RX_VK_LOCK_CENSUS=switch) + try: + done = subprocess.run([str(exe), case], env=case_env, capture_output=True, text=True, timeout=180) + except subprocess.TimeoutExpired: + fail("%s, RX_VK_LOCK_CENSUS %s: no end in 180 s (a lock never let go)" % (case, switch)) + if done.returncode or "PASS " + case not in done.stdout: + print(done.stdout[-3000:] + done.stderr[-1000:]) + fail("%s, RX_VK_LOCK_CENSUS %s" % (case, "not set" if switch is None else switch)) + print("PASS %s (RX_VK_LOCK_CENSUS %s)" % (case, "not set" if switch is None else switch)) + print("PASS: a wait for a held lock is counted against its holder for the time waited, a free lock reads no clock, and off, nothing is counted") + + +if __name__ == "__main__": + main() diff --git a/tests/test_log_batch.py b/tests/test_log_batch.py index 722d557..c0d9700 100644 --- a/tests/test_log_batch.py +++ b/tests/test_log_batch.py @@ -25,11 +25,12 @@ CASES = {"batch": "1000", "default": None, "off": "0", "burst": "2000", "flush": def log_block(source): - start_marker = "#if defined(_WIN32) && !defined(__EMSCRIPTEN__)\nstatic SRWLOCK log_lock" + # ((Triforce, 11-22, the performance audit's THREAD-56) the writer is Linux's too now: its guard leaves out only the web build + start_marker = "#if !defined(__EMSCRIPTEN__)\nstatic SRWLOCK log_lock" end_marker = "\n#else\nstatic int log_queue(" if source.count(start_marker) != 1 or source.count(end_marker) != 1: - raise SystemExit("FAIL: the Windows log queue block of host/hos_thread.c was not found") - start = source.index(start_marker) + len("#if defined(_WIN32) && !defined(__EMSCRIPTEN__)\n") + raise SystemExit("FAIL: the log queue block of host/hos_thread.c was not found") + start = source.index(start_marker) + len("#if !defined(__EMSCRIPTEN__)\n") return source[start:source.index(end_marker)] diff --git a/tests/test_log_off_lock.py b/tests/test_log_off_lock.py new file mode 100644 index 0000000..0bc357f --- /dev/null +++ b/tests/test_log_off_lock.py @@ -0,0 +1,206 @@ +"""The log's writer thread on Linux too, and the recordings' two log lines out of their lock (no game needed). + +(Triforce, 11-22, the performance audit's THREAD-56, after Source A's.) On Windows a writer thread makes every log +write; on Linux and the Steam Deck hos_log wrote stderr in the thread that logged, a game thread with its virtual core +held, and nvnCommandBufferEndRecording did so with the recordings' lock held exclusively: one blocked write stopped +every thread that records or calls a recording. Source A saw a 717 ms frame of it. + +Reads the source: + host/hos_thread.c: the writer's guard leaves out only the web build; HOS_LOG_SYNC still turns it off; on the + systems other than Windows the queue is also flushed at quick_exit, which is how the POSIX shim's ExitProcess + ends the process (atexit's functions do not run there); + every Win32 name the writer block uses is declared in host/posix/windows.h; + host/nvn_native.c: sweep_overwritten_copies returns its count and logs nothing; EndRecording calls hos_log nowhere + between taking copy_lock and releasing it. +And cuts EndRecording's tail (from the recordings' counter to the end of its two lines) out of host/nvn_native.c and +runs it 120,000 times over stand-ins for the lock, the sweep, the virtual core and the log: + no line is logged with the lock held; every line is logged with the virtual core given up, and the core is taken + back after; the sweep's line comes at most 12 times with the count the sweep returned; the 50,000-recording line + comes twice, with the numbers as they were under the lock; the lock is released exactly once a recording. + + python -I -B tests/test_log_off_lock.py [--cc clang] +""" +import argparse +import re +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +RUN_ALL_ARGS = ["--cc", "clang"] + + +def block(text, marker, what): + start = text.find(marker) + if start < 0: + raise SystemExit("FAIL: %s has no '%s'" % (what, marker)) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +FIXTURE = r'''#include +#include +#include +#include +#include +#include +typedef long long LONG64; +typedef int SRWLOCK; +static SRWLOCK copy_lock; +static int held, releases, away_now, aways, backs, fails; +static int swept_lines, made_lines, sweeps; +static size_t ncopies = 100, npins = 7; +static volatile LONG64 copy_bytes = 3 << 20, seg_bytes = 5 << 20, seg_count = 9; +#define SWEEP_EVERY 1024u +#define FAIL(...) do { fails++; if (fails < 10) { printf("FAIL: " __VA_ARGS__); printf("\n"); } } while (0) +static void AcquireSRWLockExclusive(SRWLOCK *l) { (void)l; if (held) FAIL("the lock taken twice"); held = 1; } +static void ReleaseSRWLockExclusive(SRWLOCK *l) { (void)l; if (!held) FAIL("the lock released while not held"); held = 0; releases++; } +static int vcore_held = 1; /* (the calling game thread holds a virtual core) */ +static int hos_vcore_away(void) { if (held) FAIL("the virtual core given up with the lock held"); aways++; if (!vcore_held) return 0; vcore_held = 0; away_now = 1; return 1; } +static void hos_vcore_back(void) { backs++; vcore_held = 1; away_now = 0; } +static size_t sweep_overwritten_copies(void) { /* (copy_lock held: every third sweep lets 5 copies go) */ + if (!held) FAIL("the sweep ran without the lock"); + sweeps++; + if (sweeps % 3) return 0; + ncopies -= 5; + return 5; +} +static size_t live_at_made[4]; +static void hos_log(const char *fmt, ...) { + char line[512]; + va_list ap; + va_start(ap, fmt); + vsnprintf(line, sizeof line, fmt, ap); + va_end(ap); + if (held) FAIL("logged with copy_lock held: %s", line); + if (!away_now || vcore_held) FAIL("logged with the virtual core held: %s", line); + if (strstr(line, "let go")) { + swept_lines++; + static const char pre[] = "nvn: 5 copies of recordings whose control memory was reused let go ("; + if (strncmp(line, pre, sizeof pre - 1)) FAIL("the sweep's line: %s", line); + } else if (strstr(line, "recordings made")) { + char want[256]; + if (made_lines < 4) { + snprintf(want, sizeof want, "nvn: %d recordings made, %zu live copies (3.0 MB), 7 texture copies; 9 command buffers' segments (5.0 MB)", + 50000 * (made_lines + 1), live_at_made[made_lines]); + if (strcmp(line, want)) FAIL("the recordings' line: %s (expected %s)", line, want); + } + made_lines++; + } else FAIL("another line: %s", line); +} +static void end_recording_tail(void) { + AcquireSRWLockExclusive(©_lock); + ncopies++; +@CUT@ + if (held) FAIL("the lock still held after the tail"); +} +int main(void) { + for (int i = 1; i <= 120000; i++) { + /* (what the 50,000-recording line must say: the live copies as they are under the lock at that recording) */ + if (i % 50000 == 0) live_at_made[i / 50000 - 1] = ncopies + 1 - ((i % 1024 == 0 && (sweeps + 1) % 3 == 0) ? 5 : 0); + end_recording_tail(); + if (!vcore_held) FAIL("the virtual core not taken back after recording %d", i); + } + if (releases != 120000) FAIL("the lock released %d times in 120000 recordings", releases); + if (sweeps != 120000 / 1024) FAIL("%d sweeps", sweeps); + if (swept_lines != 12) FAIL("the sweep's line came %d times (39 sweeps let copies go; 12 lines at most)", swept_lines); + if (made_lines != 2) FAIL("the 50,000-recording line came %d times", made_lines); + if (aways != backs || aways < 13 || aways > 14) FAIL("the core was given up %d times and taken back %d times for 14 lines", aways, backs); + if (fails) printf("log off lock: FAILED (%d)\n", fails); + else printf("log off lock: EndRecording's tail passed (120000 recordings, %d sweep lines, %d recordings' lines)\n", swept_lines, made_lines); + return fails != 0; +} +''' + +# (C's own names the writer block spells in capitals, and the block's own: not Win32's) +NOT_WIN32 = {"NULL", "LOG_BATCH_FULL", "HOS_LOG_BATCH_MS", "HOS_LOG_SYNC", "CRT"} + + +def check_writer(thread_c): + start_marker = "#if !defined(__EMSCRIPTEN__)\nstatic SRWLOCK log_lock = SRWLOCK_INIT, log_write_lock = SRWLOCK_INIT;" + end_marker = "\n#else\nstatic int log_queue(const char *line) { (void)line; return 0; }\nstatic void log_flush(void) {}\n#endif" + if thread_c.count(start_marker) != 1 or thread_c.count(end_marker) != 1: + raise SystemExit("FAIL: host/hos_thread.c: the log's writer is not built for every system but the web build " + "(its guard must be #if !defined(__EMSCRIPTEN__))") + writer = thread_c[thread_c.index(start_marker):thread_c.index(end_marker)] + if "_WIN32) && !defined(__EMSCRIPTEN__)\nstatic SRWLOCK log_lock" in thread_c: + raise SystemExit("FAIL: host/hos_thread.c: the log's writer is Windows only") + queue = block(writer, "static int log_queue(const char *line) {", "the writer") + order = [queue.find(n) for n in ('int on = !getenv("HOS_LOG_SYNC");', "CreateThread(NULL, 0, log_main, NULL, 0, NULL)", + "if (on) atexit(log_flush);", "#ifndef _WIN32", "if (on) at_quick_exit(log_flush);", + "#endif", "log_async = on;")] + if min(order) < 0 or order != sorted(order): + raise SystemExit("FAIL: host/hos_thread.c log_queue: HOS_LOG_SYNC, the writer's start, atexit and (off Windows) " + "at_quick_exit are not there in that order (%r)" % (order,)) + shim_c = (ROOT / "host/posix/win32_posix.c").read_text(encoding="utf-8") + if "quick_exit((int)code);" not in block(shim_c, "void ExitProcess(UINT code) {", "host/posix/win32_posix.c"): + raise SystemExit("FAIL: the POSIX shim's ExitProcess no longer ends with quick_exit: look again at how the log's " + "queue is flushed when the game ends on Linux") + # every Win32 name the writer uses is one the POSIX shim declares + shim_h = (ROOT / "host/posix/windows.h").read_text(encoding="utf-8") + code = re.sub(r"/\*.*?\*/", " ", writer, flags=re.S) + code = re.sub(r'"(\\.|[^"\\])*"', '""', code) + names = set(re.findall(r"\b(?:[A-Z][A-Za-z0-9]*[a-z][A-Za-z0-9]*|[A-Z][A-Z0-9_]{2,}|_strdup)\b", code)) - NOT_WIN32 + for must in ("SRWLOCK", "CONDITION_VARIABLE", "SleepConditionVariableSRW", "WakeConditionVariable", "CreateThread", + "GetCurrentThreadId", "GetTickCount64", "WriteFile", "GetStdHandle", "SetUnhandledExceptionFilter", + "TryAcquireSRWLockExclusive", "_strdup"): + if must not in names: + raise SystemExit("FAIL: the writer block was not read as expected (%s not found in it)" % must) + missing = sorted(n for n in names if not re.search(r"\b%s\b" % re.escape(n), shim_h)) + if missing: + raise SystemExit("FAIL: host/posix/windows.h does not declare what the log's writer uses on Linux: " + ", ".join(missing)) + return len(names) + + +def check_recordings(nvn_c): + sweep = block(nvn_c, "static size_t sweep_overwritten_copies(void) {", "host/nvn_native.c") + if "hos_log(" in sweep or "return swept;" not in sweep or "if (off || !rec_buckets) return 0;" not in sweep: + raise SystemExit("FAIL: sweep_overwritten_copies (copy_lock held) must return its count and log nothing") + if "static void sweep_overwritten_copies(" in nvn_c: + raise SystemExit("FAIL: sweep_overwritten_copies still returns nothing") + end = block(nvn_c, "NV(nvnCommandBufferEndRecording) {", "host/nvn_native.c") + if end.count("AcquireSRWLockExclusive(©_lock);") != 1 or end.count("ReleaseSRWLockExclusive(©_lock);") != 1: + raise SystemExit("FAIL: nvnCommandBufferEndRecording no longer takes and releases copy_lock once") + locked = end[end.index("AcquireSRWLockExclusive(©_lock);"):end.index("ReleaseSRWLockExclusive(©_lock);")] + if "hos_log(" in locked or "fprintf(" in locked or "fputs(" in locked: + raise SystemExit("FAIL: nvnCommandBufferEndRecording writes a log line with copy_lock held") + if "sweep_overwritten_copies()" not in locked: + raise SystemExit("FAIL: nvnCommandBufferEndRecording no longer sweeps under copy_lock") + first, last = " static uint64_t made;\n", " if (away) hos_vcore_back();\n }\n" + if end.count(first) != 1 or end.count(last) != 1 or end.index(first) > end.index(last): + raise SystemExit("FAIL: nvnCommandBufferEndRecording's tail (the counter, the sweep, the lines) was not found") + tail = end[end.index(first):end.index(last) + len(last)] + order = [tail.find(n) for n in ("ReleaseSRWLockExclusive(©_lock);", "const int away = hos_vcore_away();", + "let go (%zu live)", "recordings made, %zu live copies", "if (away) hos_vcore_back();")] + if min(order) < 0 or order != sorted(order): + raise SystemExit("FAIL: nvnCommandBufferEndRecording's two lines are not written after the lock is released, " + "with the virtual core given up (%r)" % (order,)) + return tail + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cc", default="clang") + args = parser.parse_args() + n = check_writer((ROOT / "host/hos_thread.c").read_text(encoding="utf-8")) + print("log off lock: the writer is every system's but the web build's; its %d Win32 names are in host/posix/windows.h" % n) + tail = check_recordings((ROOT / "host/nvn_native.c").read_text(encoding="utf-8")) + docs = (ROOT / "docs/development.md").read_text(encoding="utf-8") + if "| `HOS_LOG_SYNC=1`, `HOS_LOG_BATCH_MS=` |" not in docs: + raise SystemExit("FAIL: docs/development.md: the switches table has no HOS_LOG_SYNC row") + with tempfile.TemporaryDirectory(prefix="log-off-lock-") as tmp: + src, exe = Path(tmp) / "log_off_lock_test.c", Path(tmp) / "log-off-lock.exe" + src.write_text(FIXTURE.replace("@CUT@", tail), encoding="utf-8") + subprocess.run([args.cc, "-std=c11", "-Wall", "-Wextra", "-Werror", str(src), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + print((r.stdout + r.stderr).rstrip()) + if r.returncode: + raise SystemExit("FAIL: EndRecording's tail logs under its lock, or with the virtual core held") + + +if __name__ == "__main__": + main() diff --git a/tests/test_make_game_cache.py b/tests/test_make_game_cache.py index effd5b8..b3b1edd 100644 --- a/tests/test_make_game_cache.py +++ b/tests/test_make_game_cache.py @@ -37,6 +37,8 @@ class NupCacheTests(unittest.TestCase): def setUp(self): self.temporary = tempfile.TemporaryDirectory(prefix="hookshot-cache-test-") self.addCleanup(self.temporary.cleanup) + # (the performance audit's IO-47: main() locks its work folder; let go before the folder is deleted) + self.addCleanup(make_game.release_work) self.root = Path(self.temporary.name).resolve() self.out = self.root / "output" self.work = self.root / "work" @@ -327,6 +329,103 @@ class RomfsCopyTests(unittest.TestCase): make_game.refuse_dump_links(src) link.unlink() + # ---- (11-21, the performance audit's IO-41) the link check from the folders' listings, each folder made once ---- + + def test_the_link_check_asks_no_file_one_by_one(self): + with tempfile.TemporaryDirectory(prefix="hookshot-dump-listing-") as folder: + src = Path(folder) / "romfs" + (src / "Model" / "Deep" / "er").mkdir(parents=True) + for name in ("a.zs", "Model/b.zs", "Model/Deep/c.zs", "Model/Deep/er/d.zs"): + (src / name).write_bytes(b"game data") + with mock.patch.object(make_game.os, "lstat", side_effect=AssertionError("an os.lstat for one entry")), \ + mock.patch.object(make_game.os, "walk", side_effect=AssertionError("os.walk passes over what it cannot list")): + make_game.refuse_dump_links(src) + with mock.patch.dict(os.environ, {"HOOKSHOT_DUMP_LINKS_LSTAT": "1"}), \ + mock.patch.object(make_game.os, "lstat", side_effect=AssertionError("asked")): + with self.assertRaisesRegex(AssertionError, "asked"): # (the switch: as before) + make_game.refuse_dump_links(src) + + def test_a_link_three_folders_down_is_refused(self): + with tempfile.TemporaryDirectory(prefix="hookshot-dump-deep-") as folder: + src, outside = Path(folder) / "romfs", Path(folder) / "outside" + deep = src / "Model" / "Deep" / "er" + deep.mkdir(parents=True) + outside.mkdir() + (outside / "secret").write_bytes(b"not the game's") + (deep / "d.zs").write_bytes(b"game data") + make_game.refuse_dump_links(src) + made = 0 + try: + (deep / "link.zs").symlink_to(outside / "secret") + made += 1 + except OSError: + pass + else: + with contextlib.redirect_stdout(io.StringIO()) as said, self.assertRaises(SystemExit): + make_game.refuse_dump_links(src) + self.assertIn("contain a link", said.getvalue()) + (deep / "link.zs").unlink() + if os.name == "nt": # (a junction needs no privilege: a folder that is a reparse point) + import subprocess + junction = deep / "Junction" + if subprocess.run(["cmd", "/c", "mklink", "/J", str(junction), str(outside)], capture_output=True).returncode == 0: + made += 1 + with contextlib.redirect_stdout(io.StringIO()) as said, self.assertRaises(SystemExit): + make_game.refuse_dump_links(src) + self.assertIn("contain a link", said.getvalue()) + self.assertIn("Junction", said.getvalue()) + os.rmdir(junction) + if not made: + self.skipTest("neither symbolic links nor junctions here") + make_game.refuse_dump_links(src) # (with the links gone: plain again) + + def test_a_folder_that_cannot_be_listed_stops_the_build(self): + with tempfile.TemporaryDirectory(prefix="hookshot-dump-unlisted-") as folder: + src = Path(folder) / "romfs" + (src / "Model" / "Locked").mkdir(parents=True) + (src / "Model" / "a.zs").write_bytes(b"game data") + real = os.scandir + + def listing(path): + if os.path.basename(str(path)) == "Locked": + raise PermissionError(errno.EACCES, "authored: access denied") + return real(path) + with mock.patch.object(make_game.os, "scandir", side_effect=listing), \ + contextlib.redirect_stdout(io.StringIO()) as said, self.assertRaises(SystemExit): + make_game.refuse_dump_links(src) + self.assertIn("Could not read the game files at", said.getvalue()) + self.assertIn("Locked", said.getvalue()) + + def test_the_copy_makes_each_folder_once_and_asks_for_no_target(self): + with tempfile.TemporaryDirectory(prefix="hookshot-romfs-walk-") as folder: + src, dst = Path(folder) / "source", Path(folder) / "destination" + (src / "Model" / "Deep").mkdir(parents=True) + names = ["top.bin"] + ["Model/m%d.bin" % i for i in range(40)] + ["Model/Deep/d%d.bin" % i for i in range(40)] + for name in names: + (src / name).write_bytes(name.encode()) + real_makedirs, made = os.makedirs, [] + + def makedirs(path, *a, **k): + made.append(os.path.normcase(str(path))) + return real_makedirs(path, *a, **k) + real_exists = os.path.exists + + def exists(path): # (os.makedirs asks about folders itself: only a file's target counts) + if str(path).endswith(".bin"): + raise AssertionError("asked whether a target exists: " + str(path)) + return real_exists(path) + with mock.patch.object(make_game.os, "makedirs", side_effect=makedirs), \ + mock.patch.object(make_game.os.path, "exists", side_effect=exists), \ + contextlib.redirect_stdout(io.StringIO()): + make_game.link_or_copy(src, dst) + self.assertEqual(len(made), 3, "one makedirs a folder, not one a file: %r" % made) + self.assertEqual(len(set(made)), 3) + for name in names: # (the top folder's files and the nested ones: the same file, not a copy) + self.assertTrue(os.path.samefile(src / name, dst / name), name) + with contextlib.redirect_stdout(io.StringIO()): + make_game.link_or_copy(src, dst) # (again over what is there: every link says it exists, and is passed over) + self.assertEqual(sorted(p.name for p in dst.iterdir()), ["Model", "top.bin"]) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_mem_parts.py b/tests/test_mem_parts.py index 63e07b0..a51492a 100644 --- a/tests/test_mem_parts.py +++ b/tests/test_mem_parts.py @@ -217,6 +217,23 @@ int main() { CHECK(keep_all_pipelines(32 * GB, 19121 * MB, 7696 * MB) && !keep_all_pipelines(32 * GB, 19121 * MB, own_for_keep(7696 * MB, 4874 * MB, false))); CHECK(keep_all_pipelines(32 * GB, 19121 * MB, own_for_keep(7696 * MB, 4874 * MB, true))); // (AMD: as before) CHECK(keep_all_pipelines(64 * GB, 40 * GB, own_for_keep(6 * GB, 5 * GB, false))); // (a 64 GB PC keeps all still) + // (11-22: the retired imports kept for good given back all the same past the heap's high mark, or for a minute after an + // import was refused for want of room) + CHECK(!retired_pressed(6000 * MB, 8000 * MB, 85, 100000, 0) && retired_pressed(6801 * MB, 8000 * MB, 85, 100000, 0)); + CHECK(!retired_pressed(6800 * MB, 8000 * MB, 85, 100000, 0) && !retired_pressed(9 * GB, 0, 85, 100000, 0)); + CHECK(!retired_pressed(9 * GB, 8 * GB, 0, 100000, 0)); // (no mark: the heap says nothing) + CHECK(retired_pressed(0, 0, 85, 100000, 100000 - 59999) && !retired_pressed(0, 0, 85, 100000, 100000 - 60000)); + CHECK(retired_pressed(1 * GB, 8 * GB, 85, 100000, 99000) && !retired_pressed(1 * GB, 8 * GB, 85, 100000, 0)); + // (11-22: every pipeline kept on a discrete GPU in a PC of under 20 GB while the game is told its mapped pages are used; + // the switches read as the host reads them) + CHECK(query_used_on(nullptr, nullptr, nullptr) && query_used_on("1", "5", nullptr) && query_used_on("00", nullptr, nullptr)); + CHECK(!query_used_on("0", nullptr, nullptr) && !query_used_on(nullptr, "0", nullptr) && !query_used_on(nullptr, "", nullptr)); + CHECK(query_used_on(nullptr, "0", "1") && !query_used_on(nullptr, "0", "0") && !query_used_on("0", "0", "2")); + CHECK(keep_all_small_pc(true, 15731 * MB, true) && !keep_all_small_pc(false, 15731 * MB, true)); + CHECK(!keep_all_small_pc(true, 15731 * MB, false) && !keep_all_small_pc(true, 20 * GB, true) && !keep_all_small_pc(true, 0, true)); + // (11-22: the capture's spares held to 32 MB on a small card or a PC of under 20 GB; a 16 GB PC reports ~15.7 GB) + CHECK(spare_capped(true, 64 * GB) && spare_capped(false, 15731 * MB) && spare_capped(false, 20 * GB - 1)); + CHECK(!spare_capped(false, 20 * GB) && !spare_capped(false, 32 * GB) && !spare_capped(false, 0)); // (the RAM's rules) CHECK(short_now(1535 * MB) && !short_now(1536 * MB) && short_now(0)); CHECK(low_now(3 * GB - 1, 16 * GB) && !low_now(3 * GB, 16 * GB) && !low_now(3 * GB, 30 * GB)); @@ -266,6 +283,11 @@ def check_source(): raise SystemExit("the precompile's keep goes by the PC's total RAM, not the RAM the game can have") if "mem_parts::own_for_keep(ws, cache_read, amd)" not in keep or "pcache_read_ws.load()" not in keep: raise SystemExit("the precompile's keep counts the pipeline cache's read as room for the pipelines") + small = "if (!e && mem_parts::keep_all_small_pc(discrete, ms.ullTotalPhys, query_on)) {" + if small not in keep or "VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU" not in keep or \ + not keep.index("allowed < (6656ull << 20)") < keep.index(small) < keep.index("mem_parts::keep_all_pipelines("): + raise SystemExit("the precompile does not keep every pipeline on a discrete GPU in a PC of under 20 GB (after the card's " + "rule, before the RAM's, and not with RX_VK_PRECOMPILE_KEEP=auto)") init = between(draw, "void pcache_init() {", "\n}\n") if "pcache_read_ws = ws1 > ws0 ? ws1 - ws0 : 0;" not in init: raise SystemExit("pcache_init does not record what the cache's read added to the working set") @@ -275,6 +297,24 @@ def check_source(): bg = between(draw, "void predict_background() {", "\n}\n") if "while (!prepare_stopped() && memory_ram_low()) Sleep(500);" not in bg: raise SystemExit("the builder behind the game does not rest while the RAM is low") + give = between(mem, "void retired_imports_give_back() {", "\n}\n") + if "mem_parts::retired_pressed(used, budget, high, now, refused)" not in give or "give_back = pressed;" not in give or \ + "const bool pressed_back = keep && give_back && p.mstate == Pool::MIRROR_NONE" not in give or \ + "if (keep && !retired_keep_set()) {" not in give: + raise SystemExit("the retired imports kept for good are not given back when the system heap is full or an import was " + "refused (or RX_VK_KEEP_RETIRED=1 no longer keeps them always)") + refused = between(mem, "const VkResult allocated = vkAllocateMemory(dev.device, &ai, nullptr, &p.memory);", "return false;") + if "import_room_refused_ms.store(import_now_ms()" not in refused: + raise SystemExit("an import refused for want of room is not noted for the retired imports' give-back") + spare = between(mem, "uint64_t cap_spare_max() {", "\n}\n") + if "mem_parts::spare_capped(small_card, total)" not in spare or "ram_now(&avail, &total)" not in spare or \ + spare.index('getenv("RX_VK_CAP_SPARE_MB")') > spare.index("ram_now("): + raise SystemExit("the capture's spares are not held to 32 MB on a PC short of RAM, or RX_VK_CAP_SPARE_MB no longer " + "decides first") + imports = between(mem, "void imports_give_back() {", "\n}\n") + if 'static const bool ram_rule = getenv("RX_VK_IMPORT_RAM") && atoi(getenv("RX_VK_IMPORT_RAM")) != 0;' not in imports or \ + "if (ram_rule && calls % 256 == 0) {" not in imports: + raise SystemExit("the imports' RAM trigger is on by default (it gave back every unbound import every ~20 s on 16 GB PCs)") dev_md = (ROOT / "docs/development.md").read_text(encoding="utf-8") for switch in ("`RX_VK_MEM_PARTS=0`", "`RX_VK_PCACHE_RAM=1`", "`RX_VK_PRECOMPILE_KEEP=1` or `0`"): if switch not in dev_md: diff --git a/tests/test_mem_trim.py b/tests/test_mem_trim.py index 561a449..b321adb 100644 --- a/tests/test_mem_trim.py +++ b/tests/test_mem_trim.py @@ -39,7 +39,11 @@ def main(): need("r.streams[r.si].buffer" in nxt and "r.before += r.used" in nxt, "a stream move does not switch the buffer") begin = block(draw, "void draw_begin_recording(") need("r.si = 0" in begin and "r.used = 0, r.before = 0" in begin, "a recording begins in a stream that is not its first") - need(begin.index("r.streams.resize(1)") < begin.index("r.si = 0"), "the streams given back are used after") + # ((11-21, REPLAY-04) the slot's reset, the streams' give-back in it, is a function of its own, made before the stream + # in use is chosen) + reset = block(draw, "void slot_reset_locked(") + need("r.streams.resize(1)" in reset and begin.index("slot_reset_locked(r);") < begin.index("r.si = 0"), + "the streams given back are used after") tex = (ROOT / "gpu/vk/vk_texture.cpp").read_text(encoding="utf-8") astc = block(tex, "bool upload_astc_gpu(") diff --git a/tests/test_memory_priority.py b/tests/test_memory_priority.py new file mode 100644 index 0000000..51ce418 --- /dev/null +++ b/tests/test_memory_priority.py @@ -0,0 +1,205 @@ +"""VK_EXT_memory_priority behind RX_VK_MEMORY_PRIORITY: the switch's reading, the tiers, and that nothing of it is on +without the switch and the extension (gpu/vk/memory_priority.h, vk_main.cpp, vk_vma.cpp, vk_texture.cpp; no game, no GPU). + +(Triforce, 11-22, the performance audit's TEX-47, after Source A's.) When a card is short of video memory the system +moves some of the game's device memory to system memory. With the extension the game ranks it: the render targets 1.0, +the sampled textures 0.25, the rest 0.5. Off unless the switch is set. + +Compiles a driver against the header: the switch unset, empty, 0, off, no (off); 1 (1.0, 0.25, 0.5); `0.5,0.25` and +`1,0.5` (those two); `0,5`, nan, one other number, three numbers, text (off: what cannot be read changes nothing); the +tiers by kind, the lab's ballast at 1.0, everything at the driver's 0.5 while off or where the device lacks the +extension. + +And reads the source: + - every VmaPoolCreateInfo and VmaAllocationCreateInfo in vk_vma.cpp names a priority before it is used (with the + extension on, one that names none gets 0, below the textures), the pools their own tier, and no other file of the + renderer makes one; + - the allocator's bit is set only with dev.memory_priority, which is the switch, the extension and its feature; + - vk_main.cpp asks for the extension and chains its feature only with the switch; + - vk_texture.cpp takes the textures' pool for the rank only with dev.memory_priority and leaves the compaction's own + line as it was, and a sparse texture's chunks chain a priority only then; + - the switch is in the switches table. + + python -I -B tests/test_memory_priority.py [--cxx clang++] +""" +import argparse +import re +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +PROGRAM = r''' +#include +#include "memory_priority.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +using namespace memory_priority; +static bool off(const char* env) { + const Tiers t = parse(env); + return !t.on && t.targets == 0.5f && t.textures == 0.5f && t.other == 0.5f; +} +static bool is(const char* env, float targets, float textures) { + const Tiers t = parse(env); + return t.on && t.targets == targets && t.textures == textures && t.other == 0.5f; +} +int main() { + CHECK(kDefault == 0.5f); + // off: unset, empty, 0, off, no + CHECK(off(nullptr) && off("") && off(" ") && off("0") && off("off") && off("OFF") && off("no") && off(" 0 ")); + // 1: Source A's tiers + CHECK(is("1", 1.0f, 0.25f) && is(" 1 ", 1.0f, 0.25f)); + // two numbers: the render targets' and the textures' + CHECK(is("0.5,0.25", 0.5f, 0.25f) && is("1,0.5", 1.0f, 0.5f) && is("0,0", 0.0f, 0.0f) && is("1,1", 1.0f, 1.0f)); + CHECK(is("0.75,0.125", 0.75f, 0.125f)); + // what cannot be read is off, not a guess + CHECK(off("0,5")); // (5 is no priority; not read as 0.5) + CHECK(off("nan") && off("nan,0.5") && off("0.5,nan") && off("inf,0.5") && off("0.5,-inf")); + CHECK(off("1,0.25,0.5")); // (three numbers) + CHECK(off("0.5") && off("2") && off("1.0") && off("yes") && off("on") && off("high")); + CHECK(off("-0.1,0.5") && off("0.5,1.5") && off("0.5,") && off(",0.5") && off("0.5;0.25") && off("0.5 0.25") && off("0.5,0.25x")); + CHECK(off("0.50000000000000000000000000000,0.25")); // (longer than a switch is) + // the tiers by what the memory is for + const Tiers t = parse("1"); + CHECK(of(t, Kind::Target) == 1.0f && of(t, Kind::Texture) == 0.25f && of(t, Kind::Other) == 0.5f && of(t, Kind::Held) == 1.0f); + const Tiers low = parse("0.5,0.25"); + CHECK(of(low, Kind::Target) == 0.5f && of(low, Kind::Texture) == 0.25f && of(low, Kind::Held) == 1.0f); // (the ballast stays) + // off: the driver's own 0.5 for everything, the ballast too + const Tiers none; + CHECK(of(none, Kind::Target) == 0.5f && of(none, Kind::Texture) == 0.5f && of(none, Kind::Other) == 0.5f && of(none, Kind::Held) == 0.5f); + // the device decides with the switch: without the extension the switch gives nothing + CHECK(!active("1", false).on && of(active("1", false), Kind::Texture) == 0.5f && of(active("0.9,0.1", false), Kind::Target) == 0.5f); + CHECK(active("1", true).on && of(active("1", true), Kind::Texture) == 0.25f && !active(nullptr, true).on && !active("0", true).on); + return fails ? 1 : 0; +} +''' + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def check_create_infos(vma): + """Every create info of the allocator's names a priority before its address or a copy of it is taken""" + made = list(re.finditer(r"\b(VmaPoolCreateInfo|VmaAllocationCreateInfo) (\w+)(\{\};| = (\w+);)", vma)) + check(len(made) >= 16, "vk_vma.cpp: the create infos are no longer found (%d)" % len(made)) + for m in made: + kind, name, source = m.group(1), m.group(2), m.group(4) + where = "vk_vma.cpp line %d: %s %s" % (vma.count("\n", 0, m.start()) + 1, kind, name) + if source: # (a copy of one that has its priority) + before = vma[:m.start()] + check(before.rfind(source + ".priority = ") > max(before.rfind("VmaAllocationCreateInfo " + source + "{};"), + before.rfind("VmaAllocationCreateInfo " + source + " = ")) or + re.search(r"VmaAllocationCreateInfo %s = \w+;" % source, before) is not None, + where + " is a copy of an info that names no priority") + continue + rest = vma[m.end():] + used = re.search(r"&%s\b|= %s;" % (name, name), rest) + check(used is not None, where + " is never used") + check(("%s.priority = " % name) in rest[:used.start()], where + " names no priority before it is used: with the extension on it gets 0") + return len(made) + + +def check_source(): + vma = (VK / "vk_vma.cpp").read_text(encoding="utf-8") + n = check_create_infos(vma) + for path in sorted(VK.glob("*.cpp")) + sorted(VK.glob("*.h")): + if path.name != "vk_vma.cpp": + text = path.read_text(encoding="utf-8", errors="replace") + check("VmaPoolCreateInfo" not in text and "VmaAllocationCreateInfo" not in text, + path.name + " makes an allocator create info of its own: this check reads vk_vma.cpp alone") + for head, tier, what in (("VmaPool small_targets_pool() {", "pi.priority = prio(PrioKind::Target);", "the small render targets' pool"), + ("VmaPool target_images_pool() {", "pi.priority = prio(PrioKind::Target);", "the render targets' pool"), + ("VmaPool images_pool() {", "pi.priority = prio(PrioKind::Other);", "the other images' pool"), + ("VmaPool textures_pool() {", "pi.priority = prio(PrioKind::Texture);", "the textures' pool"), + ("bool vma_piece_alloc(", "pi.priority = prio(PrioKind::Other);", "the mirrors' pieces' pool"), + ("bool vma_init(PFN_vkGetInstanceProcAddr gipa) {", "ai.priority = prio(PrioKind::Held);", "the lab's ballast"), + ("bool create_image_inner_(const VkImageCreateInfo& ci, VkImage& image, VmaAllocation& allocation, bool dedicated, bool target, bool textures) {", + "ai.priority = prio(target ? PrioKind::Target : textures ? PrioKind::Texture : PrioKind::Other);", "an image")): + check(tier in block(vma, head), "vk_vma.cpp: %s does not name its tier" % what) + init = block(vma, "bool vma_init(PFN_vkGetInstanceProcAddr gipa) {") + check("(dev.memory_priority ? VMA_ALLOCATOR_CREATE_EXT_MEMORY_PRIORITY_BIT : 0)" in init and + vma.count("VMA_ALLOCATOR_CREATE_EXT_MEMORY_PRIORITY_BIT") == 2, # (the comment's and the flag's) + "vk_vma.cpp: the allocator's priority bit is not set with dev.memory_priority alone") + check('static const memory_priority::Tiers t = memory_priority::active(getenv("RX_VK_MEMORY_PRIORITY"), dev.memory_priority);' in vma and + "float prio(PrioKind k) { return memory_priority::of(memory_tiers(), k); }" in vma, + "vk_vma.cpp: the tiers are not the switch's where the device has the extension") + check("vk: memory priorities on" in init, "vk_vma.cpp: the tiers are not logged at the start") + + main = (VK / "vk_main.cpp").read_text(encoding="utf-8") + at = [main.index(s) for s in ( + 'const bool prio_asked = memory_priority::parse(getenv("RX_VK_MEMORY_PRIORITY")).on;', + "if (prio_asked) optional(VK_EXT_MEMORY_PRIORITY_EXTENSION_NAME, dev.memory_priority);", + "VkPhysicalDeviceMemoryPriorityFeaturesEXT mem_prio{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PRIORITY_FEATURES_EXT};", + "if (dev.memory_priority) *tail = &mem_prio, tail = &mem_prio.pNext;", + "vkGetPhysicalDeviceFeatures2(dev.physical, &f);", + "dev.memory_priority = dev.memory_priority && mem_prio.memoryPriority;", + "VkDeviceCreateInfo ci{VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO};")] + check(at == sorted(at), "vk_main.cpp does not ask for the extension and its feature with the switch alone, before the device is made") + check(main.count("VK_EXT_MEMORY_PRIORITY_EXTENSION_NAME") == 1 and main.count("&mem_prio") == 2, + "vk_main.cpp names the extension or chains its feature somewhere else") + internal = (VK / "vk_internal.h").read_text(encoding="utf-8") + check("memory_priority = false," in internal and "const memory_priority::Tiers& memory_tiers();" in internal, + "vk_internal.h: the device's flag starts on, or the tiers are not declared") + every = "".join(p.read_text(encoding="utf-8", errors="replace") for p in sorted(VK.glob("*.cpp"))) + check(len(re.findall(r"dev\.memory_priority = ", every)) == 1, "dev.memory_priority is set somewhere but vk_main.cpp's feature check") + + tex = (VK / "vk_texture.cpp").read_text(encoding="utf-8") + create = tex[tex.index("Texture* create(const Key& key"):] + create = create[:create.index("\n}\n")] + at = [create.index(s) for s in ( + "const bool in_textures = compact_on() && !x.sparse && !(as_target && !is_bc(x.format) && !is_astc(x.format));", + "const bool ranked = !in_textures && dev.memory_priority && !x.sparse && !(as_target && !is_bc(x.format) && !is_astc(x.format));", + "} else if (in_textures && create_image(ci, x.image, x.allocation, false, false, true)) {", + "} else if (ranked && create_image(ci, x.image, x.allocation, false, false, true)) {", + "pool_take(pool_key, x.image, x.allocation, x.views)")] + check(at == sorted(at), "create() does not take the textures' pool for its rank only with dev.memory_priority, after the compaction's own line") + check("if (!((compact_on() || dev.memory_priority) && create_image(ci, image, allocation, false, false, true)) &&" in tex, + "a BC1 texture made BC3 again does not come from the textures' pool with the priorities on") + take = block(tex, "bool sparse_take(Texture* tex, VkDeviceSize n, VkDeviceMemory& mem, VkDeviceSize& off) {") + at = [take.index(s) for s in ( + "VkMemoryAllocateInfo ai{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};", + "VkMemoryPriorityAllocateInfoEXT chunk_prio{VK_STRUCTURE_TYPE_MEMORY_PRIORITY_ALLOCATE_INFO_EXT};", + "if (dev.memory_priority) {\n chunk_prio.priority = memory_priority::of(memory_tiers(), memory_priority::Kind::Texture);\n" + " ai.pNext = &chunk_prio;\n }", + "if (vkAllocateMemory(dev.device, &ai, nullptr, &m) != VK_SUCCESS) {")] + check(at == sorted(at) and take.count("ai.pNext") == 1, "a sparse texture's chunks chain a priority without dev.memory_priority, or not the textures'") + check(len(re.findall(r"getenv\(\"RX_VK_MEMORY_PRIORITY\"\)", every)) == 2, + "RX_VK_MEMORY_PRIORITY is read somewhere but vk_main.cpp's ask and vk_vma.cpp's tiers") + docs = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("| `RX_VK_MEMORY_PRIORITY=1` or `,` |" in docs and "`gpu/vk/memory_priority.h`" in docs, + "docs/development.md: RX_VK_MEMORY_PRIORITY is not in the switches table") + return n + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + check((VK / "memory_priority.h").is_file(), "gpu/vk/memory_priority.h is not there") + n = check_source() + with tempfile.TemporaryDirectory(prefix="memory-priority-") as tmp: + c, exe = Path(tmp) / "prio.cpp", Path(tmp) / "prio.exe" + c.write_text(PROGRAM, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(VK), str(c), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + check(r.returncode == 0, "the switch's reading or the tiers:\n" + r.stdout) + print("PASS: memory priorities are off without the switch and the extension; %d create infos in vk_vma.cpp name their tier" % n) + + +if __name__ == "__main__": + main() diff --git a/tests/test_mesa_spilling_rate.py b/tests/test_mesa_spilling_rate.py new file mode 100644 index 0000000..74be5bc --- /dev/null +++ b/tests/test_mesa_spilling_rate.py @@ -0,0 +1,121 @@ +"""shader_spilling_rate=2 for Mesa's Intel shader compiler, set before the driver reads it (gpu/vk/mesa_spilling.h; no game, no GPU). + +(Triforce, 11-22, the performance audit's PIPE-55, after Source A's.) Mesa's Intel compiler builds a program whose +registers do not fit in fewer, larger steps at spilling rate 2: minutes and gigabytes less on the large compute programs. +The Linux build sets the driver's option in its environment before the instance is made, never over a value the player +set; RX_VK_NO_SPILLING_RATE=1 leaves the driver as it is. No other driver reads the option. + +Compiles a small driver against the header and checks: + wanted the Linux build with the switch unset, "" or 0; not with 1 or any other number; never the Windows build; + effective the player's own value wins (3, 1, 2); 2 where wanted and nothing set; nothing where not wanted; + told the log line on Mesa's Intel driver alone; + the name and the rate are the driver's option and 2. + +And reads gpu/vk/vk_main.cpp: in create_instance, under #ifndef _WIN32 and behind the switch, +setenv("shader_spilling_rate", "2", 0) (0: no overwrite) comes before vkCreateInstance(; nothing else sets it; the log +line is keyed on VK_DRIVER_ID_INTEL_OPEN_SOURCE_MESA; and the switches table has the row. + + python -I -B tests/test_mesa_spilling_rate.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +DRIVER = r''' +#include "mesa_spilling.h" +#include +#include +#include + +static int checks, fails; +static void want(const char* what, bool got, bool wanted) { + checks++; + if (got != wanted) fails++, printf("FAIL %s: %d, want %d\n", what, (int)got, (int)wanted); +} +static bool same(const char* a, const char* b) { + return (!a && !b) || (a && b && !std::strcmp(a, b)); +} + +int main() { + using namespace mesa_spilling; + want("the option's name", same(kName, "shader_spilling_rate"), true); + want("the rate", same(kRate, "2"), true); + // the Linux build + want("linux, the switch unset", wanted(false, nullptr), true); + want("linux, the switch empty", wanted(false, ""), true); + want("linux, the switch 0", wanted(false, "0"), true); + want("linux, the switch 1", wanted(false, "1"), false); + want("linux, the switch 2", wanted(false, "2"), false); + // the Windows build sets nothing, whatever the switch + for (const char* off : {(const char*)nullptr, "", "0", "1"}) want("the Windows build", wanted(true, off), false); + // what the driver reads: the player's value first + want("the player's 3 wins", same(effective("3", true), "3"), true); + want("the player's 1 wins", same(effective("1", true), "1"), true); + want("the player's 2 with the switch off", same(effective("2", false), "2"), true); + want("nothing set, wanted: 2", same(effective(nullptr, true), "2"), true); + want("nothing set, not wanted: the driver's own", effective(nullptr, false) == nullptr, true); + // the log's line + want("told on Mesa's Intel driver", told(true), true); + want("not told on any other driver", told(false), false); + if (fails) printf("mesa spilling rate: FAILED (%d of %d checks)\n", fails, checks); + else printf("mesa spilling rate: all %d checks passed\n", checks); + return fails != 0; +} +''' + + +def check_source(): + src = (ROOT / "gpu/vk/vk_main.cpp").read_text(encoding="utf-8") + if '#include "mesa_spilling.h"' not in src: + raise SystemExit("vk_main.cpp does not include mesa_spilling.h") + at = src.find("bool create_instance() {") + if at < 0: + raise SystemExit("vk_main.cpp has no create_instance()") + body = src[at:src.index("\n}\n", at)] + call = 'if (mesa_spilling::wanted(false, getenv("RX_VK_NO_SPILLING_RATE"))) setenv("shader_spilling_rate", "2", 0);' + if body.count(call) != 1: + raise SystemExit("create_instance does not set shader_spilling_rate=2 without overwriting, behind RX_VK_NO_SPILLING_RATE") + if src.count('setenv("shader_spilling_rate"') != 1 or 'putenv("shader_spilling_rate' in src: + raise SystemExit("vk_main.cpp sets shader_spilling_rate somewhere else too") + made = body.index("vkCreateInstance(&ci, nullptr, &dev.instance)") + if body.index(call) > made: + raise SystemExit("create_instance sets the rate after the instance is made: the driver has read its options by then") + # under #ifndef _WIN32: the nearest preprocessor line before the call opens it, the nearest after closes it + before = [l.strip() for l in body[:body.index(call)].split("\n") if l.lstrip().startswith("#")] + after = [l.strip() for l in body[body.index(call):].split("\n") if l.lstrip().startswith("#")] + if not before or before[-1] != "#ifndef _WIN32" or not after or after[0] != "#endif": + raise SystemExit("create_instance: the setenv is not inside an #ifndef _WIN32 block of its own") + device = src[src.index("bool create_device() {"):] + told = device.find("if (mesa_spilling::told(drv.driverID == VK_DRIVER_ID_INTEL_OPEN_SOURCE_MESA)) {") + line = device.find("vk: Mesa's Intel compiler spills at rate %s (driconf shader_spilling_rate: 2 unless set)") + if told < 0 or line < told or line - told > 400 or "getenv(mesa_spilling::kName)" not in device[told:line]: + raise SystemExit("create_device: the rate's log line is not keyed on Mesa's Intel driver id") + if device.index("dev.driver_id = drv.driverID;") > told: + raise SystemExit("create_device: the rate's log line is not beside the driver id") + docs = (ROOT / "docs/development.md").read_text(encoding="utf-8") + if "| `RX_VK_NO_SPILLING_RATE=1` |" not in docs or "`gpu/vk/mesa_spilling.h`" not in docs or "Linux build only" not in docs: + raise SystemExit("docs/development.md: the switches table has no row for RX_VK_NO_SPILLING_RATE") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="g++") + args = parser.parse_args() + check_source() + with tempfile.TemporaryDirectory(prefix="mesa-spilling-") as tmp: + cpp, exe = Path(tmp) / "mesa_spilling_test.cpp", Path(tmp) / "mesa-spilling.exe" + cpp.write_text(DRIVER, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-Wall", "-Wextra", "-Werror", "-I" + str(ROOT / "gpu/vk"), str(cpp), "-o", str(exe)], + check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + print(r.stdout.rstrip()) + if r.returncode: + raise SystemExit("FAIL: the spilling rate's checks failed") + + +if __name__ == "__main__": + main() diff --git a/tests/test_mesh_fallback.py b/tests/test_mesh_fallback.py index 42fee90..cec420a 100644 --- a/tests/test_mesh_fallback.py +++ b/tests/test_mesh_fallback.py @@ -182,7 +182,7 @@ def main(): raise SystemExit("the fallback still makes a record and payload pair for every program") if "fallback_buffers()[&p]" in draw: raise SystemExit("the fallback buffers are still keyed by the program") - if "(!p.fallback || dw().prog == &p)" not in draw or "dw().prog = &p;" not in draw: + if "(!p.fallback || dw().prog == &p)" not in draw or "dwr.prog = &p;" not in draw: # ((11-22, DESC-20) build_descriptors' dw(), taken once) raise SystemExit("a fallback draw reusing its descriptors after a dispatch pushes the dispatch's at the compute bind point") if "VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT" not in VMA.read_text(encoding="utf-8"): raise SystemExit("the allocator does not make the fallback's buffers addressable") diff --git a/tests/test_mirror_bookkeeping.py b/tests/test_mirror_bookkeeping.py new file mode 100644 index 0000000..a4d838a --- /dev/null +++ b/tests/test_mirror_bookkeeping.py @@ -0,0 +1,123 @@ +"""Without mirrors nothing is noted for a write-back that never comes (gpu/vk/vk_memory.cpp wb_notes_off, memory_cpu_owned, +shadow_written_to, guest_cpu_wrote; no game, no GPU). + +(Triforce, 11-22, the performance audit's MEM-51, after Source A's.) The three functions note the CPU's writes for a +mirror's write-back: the replay's own report words, which the write-back leaves out, and the pages the write-back diff's +shadows must read again. Without mirrors (Intel's integrated GPUs, RX_VK_MIRROR=0) no write-back reads them: the lists +grew to 65,536 words and 8 million pages, were dropped and grew again. + +tests/mirror_bookkeeping_test.cpp gets the functions cut from the source and is built twice, from the code as it is and +from the code with the new condition taken out (the code as it was): + - mirrors on: after each of 40 rounds of 3,000 random calls every list is the same in both builds, entry for entry + (size and hash), and so are the host's page look-ups and writes: nothing changes where there are mirrors; + - mirrors off: every list stays empty, guest_cpu_wrote looks no guest page up, guest_cpu_write still writes. The build + of the code as it was fails this case, so the case means something; + - before the device is made mirror_on() is not asked (it decides once, by the card) and everything is noted. + +And reads the source, for the invariant the skip rests on: the two readers (cut_cpu_words, shadows_forget_written) are +called once each, inside memory_end_submission's `if (!writebacks.empty())`; the only writebacks.push_back is in +gpu_buffer_bind after its `!mirror_on()` return; mirror_make is called only after such a return; shadow_make needs a +mirror; mirror_on() reads its switch once. + + python -I -B tests/test_mirror_bookkeeping.py [--cxx clang++] +""" +import argparse +import os +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + check(marker in text, "vk_memory.cpp has no " + marker) + start = text.index(marker) + end = text.index("{", start + len(marker) - 1) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + mem = (VK / "vk_memory.cpp").read_text(encoding="utf-8") + + # the three early returns + off = block(mem, "bool wb_notes_off() {") + check(off == "bool wb_notes_off() { return dev.device != VK_NULL_HANDLE && !mirror_on(); }", "wb_notes_off is not: the device made and no mirrors") + owned = block(mem, "void memory_cpu_owned(uint64_t guest, uint64_t size) {") + check("if (off || !guest || wb_notes_off()) return;" in owned and owned.index("wb_notes_off()") < owned.index("std::lock_guard lk(cpu_words_mu);"), + "memory_cpu_owned notes words without mirrors, or takes its lock first") + to = block(mem, "void shadow_written_to(std::vector& list, const uint32_t* pages, int64_t n) {") + check("if (n <= 0 || !wb_diff_on() || wb_notes_off()) return;" in to and to.index("wb_notes_off()") < to.index("std::lock_guard lk(shadow_written_mu);"), + "shadow_written_to notes pages without mirrors, or takes its lock first") + wrote = block(mem, "void guest_cpu_wrote(uint64_t gpu, uint64_t n) {") + check("if (!n || !wb_diff_on() || wb_notes_off()) return;" in wrote and wrote.index("wb_notes_off()") < wrote.index("host->gpu_guest(a)"), + "guest_cpu_wrote looks guest pages up without mirrors") + + # the invariant: no write-back, and so no reader, without mirrors + end = block(mem, "void memory_end_submission(VkCommandBuffer cmd) {") + wb = block(end, "if (!writebacks.empty()) {") + check(mem.count("cut_cpu_words(") == 2 and "cut_cpu_words(*p, runs);" in wb, "cut_cpu_words is called outside the write-back") + check(mem.count("shadows_forget_written()") == 2 and "shadows_forget_written();" in wb, "shadows_forget_written is called outside the write-back") + bind = block(mem, "BufferRef gpu_buffer_bind(uint64_t gpu, uint64_t size, bool written) {") + gate = "if (!mirror_on() || !mirror_upload || p->mstate == Pool::MIRROR_REFUSED) {" + # (every write-back is queued in gpu_buffer_bind, after its return without mirrors: one push, or (Triforce) the split + # write-back's push of each part beside it) + check(mem.count("writebacks.push_back(") == bind.count("writebacks.push_back(") >= 1 and gate in bind and + "return remember(in_guest({p->buffer, off, size, 0}));" in block(bind, gate) and + bind.index(gate) < bind.index("writebacks.push_back("), "a write-back is queued before gpu_buffer_bind's return without mirrors") + check(mem.count("mirror_make(*p)") == 2 and bind.index(gate) < bind.index("mirror_make(*p)"), "gpu_buffer_bind makes a mirror before its return without mirrors") + begin = block(mem, "void memory_begin_submission() {") + check("if (!mirror_on()) return;" in begin and begin.index("if (!mirror_on()) return;") < begin.index("for (Pool* p : mirror_wanted) mirror_make(*p);"), + "memory_begin_submission makes mirrors before its return without mirrors") + check("if (!p.mirror || !shadow_room(n)) return false;" in block(mem, "bool shadow_make(Pool& p) {"), "a shadow is made for a pool without a mirror") + on = block(mem, "bool mirror_on() {") + check("static const int on = [] {" in on and 'getenv("RX_VK_MIRROR")' in on, "mirror_on() is not fixed for the run") + + cut = "\n".join(block(mem, m) for m in ( + "bool wb_notes_off() {", "void sorted_merge(", "void cpu_words_merge() {", "void memory_cpu_owned(uint64_t guest, uint64_t size) {", + "void shadow_written_to(std::vector& list, const uint32_t* pages, int64_t n) {", + "void shadow_written(const uint32_t* pages, int64_t n) {", "void shadow_written_collected(const uint32_t* pages, int64_t n) {", + "int guest_cpu_write(uint64_t gpu, const void* src, uint64_t n) {", "void guest_cpu_wrote(uint64_t gpu, uint64_t n) {")) + check(cut.count(" || wb_notes_off()") == 3, "the three early returns do not ask wb_notes_off()") + before = cut.replace(" || wb_notes_off()", "") # (the code as it was) + fixture = (ROOT / "tests" / "mirror_bookkeeping_test.cpp").read_text(encoding="utf-8") + env = {k: v for k, v in os.environ.items() if not k.startswith("RX_VK_")} + with tempfile.TemporaryDirectory(prefix="mirror-bookkeeping-") as tmp: + exes = {} + for name, code in (("now", cut), ("before", before)): + src, exe = Path(tmp) / (name + ".cpp"), Path(tmp) / (name + ".exe") + src.write_text(fixture.replace("/* ACTUAL */", code), encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Wno-unused-function", str(src), "-o", str(exe)], check=True) + exes[name] = str(exe) + + def run(name, case): + return subprocess.run([exes[name], case], env=env, capture_output=True, text=True, timeout=300) + + now, was = run("now", "dump"), run("before", "dump") + check(now.returncode == 0 and was.returncode == 0, "with mirrors:\n" + now.stdout + was.stdout) + check(now.stdout == was.stdout and now.stdout.count("\n") == 40 * 7, "with mirrors the lists are not what they were") + r = run("now", "off") + check(r.returncode == 0, "without mirrors:\n" + r.stdout) + check(run("before", "off").returncode != 0, "the code as it was passes the case without mirrors: the case checks nothing") + r = run("now", "early") + check(r.returncode == 0, "before the device is made:\n" + r.stdout) + print("PASS: without mirrors nothing is noted for a write-back and no guest page is looked up; with mirrors every list is " + "what it was") + + +if __name__ == "__main__": + main() diff --git a/tests/test_mirror_budget.py b/tests/test_mirror_budget.py new file mode 100644 index 0000000..9257889 --- /dev/null +++ b/tests/test_mirror_budget.py @@ -0,0 +1,93 @@ +"""The VRAM mirrors' budget by the card (gpu/vk/mirror_budget.h, vk_memory.cpp mirror_budget; no game/GPU). + +The rule is compiled from the header and run on made-up cards (MB in, MB out): discrete 6144 -> 1536, 5992 -> 1498, 4608 -> 576, +4096 -> 512, 8192 -> 2048, 16384 -> 3072, 24576 -> 3072; integrated 6144 -> 768, 6656 -> 832, 8192 -> 2048; video memory not known +-> 3072. The source: mirror_budget reads RX_VK_MIRROR_MB before the rule, asks the rule with the card's device type, and no longer +reads the quality preset's opts.mirror_mb; the docs row says the rule. + + python -I -B tests/test_mirror_budget.py [--cxx clang++] +""" +import argparse +import os +from pathlib import Path +import subprocess +import tempfile + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] + +CASES = [ # (MB of video memory, discrete, MB of mirrors) + (6144, True, 1536), (5992, True, 1498), (4608, True, 576), (4096, True, 512), (8192, True, 2048), (16384, True, 3072), + (24576, True, 3072), (6144, False, 768), (6656, False, 832), (8192, False, 2048), (0, True, 3072), (0, False, 3072), +] + +FIXTURE = r'''#include +#include +#include "mirror_budget.h" +int main(int argc, char** argv) { + for (int i = 1; i + 1 < argc; i += 2) { + const unsigned long long vram = strtoull(argv[i], nullptr, 10) << 20; + std::printf("%llu\n", (unsigned long long)(mkwvk::mirror_size::bytes(vram, atoi(argv[i + 1]) != 0) >> 20)); + } + return 0; +} +''' + + +def fail(message): + raise SystemExit("FAIL: " + message) + + +def check_source(): + mem = (ROOT / "gpu/vk/vk_memory.cpp").read_text(encoding="utf-8") + dev = (ROOT / "docs/development.md").read_text(encoding="utf-8") + body = mem[mem.index("uint64_t mirror_budget() {"):] + body = body[:body.index("\n}\n")] + env = body.index('getenv("RX_VK_MIRROR_MB")') + rule = body.index("mirror_size::bytes(vram, discrete)") + if env > rule: + fail("mirror_budget asks the rule before RX_VK_MIRROR_MB") + if "dev.props.deviceType == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU" not in body: + fail("mirror_budget no longer tells the rule whether the card is discrete") + if "opts.mirror_mb" in body: + fail("mirror_budget reads the quality preset's opts.mirror_mb again") + if "vram / 8" in body: + fail("mirror_budget has a rule of its own besides mirror_budget.h") + if "a quarter of the video memory on a discrete card over 4.5 GB" not in dev: + fail("docs/development.md's RX_VK_MIRROR_MB row does not say the rule") + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--cxx", default="g++") + a = p.parse_args() + try: + check_source() + except ValueError as e: + fail("a text this test reads moved (%s)" % e) + flags = subprocess.BELOW_NORMAL_PRIORITY_CLASS if os.name == "nt" else 0 + with tempfile.TemporaryDirectory(prefix="mirror-budget-") as tmp: + cpp = Path(tmp) / "fixture.cpp" + exe = Path(tmp) / ("fixture" + (".exe" if os.name == "nt" else "")) + cpp.write_text(FIXTURE, encoding="utf-8") + built = subprocess.run([a.cxx, "-std=c++20", "-O1", "-Wall", "-Werror", "-I" + str(ROOT / "gpu/vk"), str(cpp), "-o", str(exe)], + capture_output=True, text=True, creationflags=flags) + if built.returncode: + print(built.stdout + built.stderr) + fail("the fixture did not build") + args = [str(x) for vram, discrete, _ in CASES for x in (vram, int(discrete))] + done = subprocess.run([str(exe)] + args, capture_output=True, text=True, timeout=60, creationflags=flags) + got = done.stdout.split() + if done.returncode or len(got) != len(CASES): + fail("the fixture failed: " + (done.stdout + done.stderr).strip()) + wrong = ["%d MB %s: %s MB, wanted %d" % (v, "discrete" if d else "integrated", g, want) + for (v, d, want), g in zip(CASES, got) if int(g) != want] + if wrong: + fail("\n ".join([""] + wrong)) + print("PASS: the mirrors' budget: a quarter on discrete cards over 4.5 GB (at most 3 GB), an eighth below and on integrated GPUs " + "up to 6.5 GB, 3 GB unknown; RX_VK_MIRROR_MB first (%d cards)" % len(CASES)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_mirror_piece_idle.py b/tests/test_mirror_piece_idle.py new file mode 100644 index 0000000..a403140 --- /dev/null +++ b/tests/test_mirror_piece_idle.py @@ -0,0 +1,200 @@ +"""What the mirrors' pieces hold that no game pool covers, and how much of it no binding reached for a while: counted, +nothing released (gpu/vk/piece_idle.h, vk_memory.cpp piece_idle_for, pieces_touch and the 1200-submission line; no game, +no GPU). + +(Triforce, 11-22, the performance audit's MEM-26, its FIRST STEP only, after Source A's.) A mirror in pieces keeps a +piece's memory until the whole mirror goes, also where no game pool is left. The entry's first step is to count what a +release would find before porting it; the release itself (the unbind, the free after two timelines) is not ported. + +This compiles piece_idle.h and compares it with a model that asks every raw pool about every piece, on 3,000 random +mirrors: pools in any order, overlapping, nested, empty, touching a piece by one byte or ending exactly at its edge; +pieces of 64 KB to 1 MB; a mirror whose size is not a multiple of its piece; random residency and last bindings; the +idle wait 0 (not tracked) or set. held, uncovered and idle must each be the model's. merge() must give disjoint ascending +ranges that cover exactly the bytes the raw pools cover. + +Then it cuts piece_idle_for and pieces_touch out of vk_memory.cpp and runs them over a stand-in pool: with +RX_VK_MIRROR_PIECE_IDLE unset or 0 a binding writes nothing; set, it writes this submission into the pieces of its range +and no other, a whole mirror or a pool with no table is left alone, and a range past the table's end stops at it. + +And reads the source: both places gpu_buffer_bind returns a mirror in pieces touch them, the table is sized once where +res_bits is, the count follows the "mirrors in pieces" line under its own every-1200 test, nothing in the change frees or +unbinds, and the switches table says the release is not ported. + + python -I -B tests/test_mirror_piece_idle.py [--cxx clang++] +""" +import argparse +import os +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + check(text.count(marker) == 1, "vk_memory.cpp does not have once: " + marker) + start = text.index(marker) + end = text.index("{", start + len(marker) - 1) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +PROGRAM = r""" +#include +#include +#include +#include +#include "piece_idle.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { if (fails++ < 10) std::printf("FAIL:%d %s\n", __LINE__, #x); } } while (0) +static uint64_t rnd_state = 0xBB67AE8584CAA73Bull; +static uint64_t rnd() { + rnd_state ^= rnd_state << 13, rnd_state ^= rnd_state >> 7, rnd_state ^= rnd_state << 17; + return rnd_state; +} +// ---- stand-ins for the cut code ---- +static void logf(const char*, ...) {} +struct Pool { + bool mpieces = true; + uint32_t piece_shift = 18; + std::vector piece_last; +}; +uint64_t mirror_epoch = 0; +@@CUT@@ + +int main(int argc, char** argv) { + const uint64_t want_idle = argc > 1 ? (uint64_t)std::atoll(argv[1]) : 0; + // ---- the count against a model that asks every pool about every piece ---- + for (int t = 0; t < 3000; t++) { + const uint32_t shift = 16 + (uint32_t)(rnd() % 5); + const uint64_t piece = 1ull << shift, lo = (0x1000ull + rnd() % 64) << 20; + const uint64_t size = (2 + rnd() % 40) * piece - (rnd() % 3 == 0 ? rnd() % piece : 0); + const uint64_t n = (size + piece - 1) >> shift, now = 5000 + rnd() % 100000, idle_for = t % 3 == 0 ? 0 : 1 + rnd() % 1200; + std::vector res(n); + std::vector last(n); + for (uint64_t i = 0; i < n; i++) res[i] = rnd() % 3 != 0, last[i] = now - rnd() % 2400; + piece_idle::Ranges raw; + for (int k = (int)(rnd() % 12); k > 0; k--) { + uint64_t a = lo - 4 * piece + rnd() % (size + 8 * piece), len = rnd() % 5 == 0 ? rnd() % (6 * piece) : rnd() % (piece / 2 + 1); + if (rnd() % 4 == 0) a = lo + (rnd() % (n + 1)) * piece; // (begins at a piece's edge) + if (rnd() % 4 == 0 && a > lo) len = ((a - lo) / piece + 1) * piece + lo - a; // (ends at one) + if (rnd() % 9 == 0) len = 0; // (an empty one) + raw.push_back({a, a + len}); + } + piece_idle::Count want; + for (uint64_t i = 0; i < n; i++) { + if (!res[i]) continue; + const uint64_t a = lo + i * piece, b = lo + std::min(size, (i + 1) * piece); + want.held += b - a; + bool touched = false; + for (const auto& r : raw) touched = touched || (r.first < r.second && r.first < b && a < r.second); + if (touched) continue; + want.uncovered += b - a; + if (idle_for && now - last[i] >= idle_for) want.idle += b - a; + } + piece_idle::Ranges merged = raw; + piece_idle::merge(merged); + uint64_t covered = 0; + for (size_t k = 0; k < merged.size(); k++) { + CHECK(merged[k].first < merged[k].second && (!k || merged[k - 1].second < merged[k].first)); + covered += merged[k].second - merged[k].first; + } + if (t % 50 == 0) { // (the merged ranges cover the raw pools' bytes, counted 4 KB at a time: the pools here are not page aligned, so by sample) + for (int s = 0; s < 400; s++) { + const uint64_t x = lo - 4 * piece + rnd() % (size + 8 * piece); + bool in_raw = false, in_merged = false; + for (const auto& r : raw) in_raw = in_raw || (x >= r.first && x < r.second); + for (const auto& r : merged) in_merged = in_merged || (x >= r.first && x < r.second); + CHECK(in_raw == in_merged); + } + } + piece_idle::Count got; + piece_idle::count(lo, size, shift, [&](uint64_t i) { return res[i] != 0; }, [&](uint64_t i) { return last[i]; }, merged, now, idle_for, got); + CHECK(got.held == want.held && got.uncovered == want.uncovered && got.idle == want.idle); + CHECK(got.idle <= got.uncovered && got.uncovered <= got.held && (idle_for || !got.idle)); + (void)covered; + } + // ---- the cut code: a binding's note in its pieces ---- + CHECK(piece_idle_for() == want_idle); + Pool p; + p.piece_last.assign(64, 7); + mirror_epoch = 0x100000123ull; // (past 32 bits: the low ones are kept) + pieces_touch(&p, 3 * (1ull << 18) + 5, 2 * (1ull << 18)); // (pieces 3, 4 and 5) + for (size_t i = 0; i < 64; i++) CHECK(p.piece_last[i] == (want_idle && i >= 3 && i <= 5 ? 0x123u : 7u)); + pieces_touch(&p, 62 * (1ull << 18), 40 * (1ull << 18)); // (past the table's end: pieces 62 and 63 only) + CHECK(p.piece_last[62] == (want_idle ? 0x123u : 7u) && p.piece_last[63] == (want_idle ? 0x123u : 7u) && p.piece_last[61] == 7u); + pieces_touch(&p, 10 * (1ull << 18), 0); // (nothing bound) + CHECK(p.piece_last[10] == 7u); + Pool whole; + whole.mpieces = false, whole.piece_last.assign(8, 7); + pieces_touch(&whole, 0, 1 << 20); // (a whole mirror: no pieces) + CHECK(whole.piece_last[0] == 7u); + Pool untracked; + pieces_touch(&untracked, 0, 1 << 20); // (no table: made before the switch could matter) + CHECK(untracked.piece_last.empty()); + if (fails) std::printf("piece idle: FAILED (%d checks)\n", fails); + else std::printf("piece idle (RX_VK_MIRROR_PIECE_IDLE %s): 3,000 mirrors counted as the model counts them; a binding %s\n", + want_idle ? "set" : "unset", want_idle ? "notes its pieces and no other" : "notes nothing"); + return fails != 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + check((VK / "piece_idle.h").exists(), "gpu/vk/piece_idle.h is missing") + mem = (VK / "vk_memory.cpp").read_text(encoding="utf-8") + check('#include "piece_idle.h"' in mem, "vk_memory.cpp does not include piece_idle.h") + idle_for = block(mem, "uint64_t piece_idle_for() {") + check('getenv("RX_VK_MIRROR_PIECE_IDLE")' in idle_for and "static const uint64_t v" in idle_for and "e && atoi(e) > 0 ? (uint64_t)atoi(e) : 0" in idle_for, + "RX_VK_MIRROR_PIECE_IDLE is not read once, off unless a number above 0") + touch = block(mem, "inline void pieces_touch(Pool* p, uint64_t off, uint64_t size) {") + check("if (!piece_idle_for() || !p->mpieces || !size || p->piece_last.empty()) return;" in touch, "pieces_touch does something without the switch") + bind = block(mem, "BufferRef gpu_buffer_bind(uint64_t gpu, uint64_t size, bool written) {") + check(bind.count("pieces_touch(p, off, size);") == 2 and mem.count("pieces_touch(") == 3, "pieces_touch is not called at gpu_buffer_bind's two returns of a mirror") + check("if (pieces_resident(p, off, size)) {\n pieces_touch(p, off, size);" in bind and + bind.rstrip().endswith("return remember({p->mirror, off, size, 0});\n}") and + "pieces_touch(p, off, size);" in bind[bind.rindex("for (uint64_t q = a; q <= b; q++)"):], + "gpu_buffer_bind: a binding given a mirror does not note its pieces before it returns") + check("if (piece_idle_for() && p->piece_last.size() != words * 64) p->piece_last.assign(words * 64, (uint32_t)mirror_epoch);" in mem and + mem.index("if (p->res_bits.size() != words) p->res_bits.assign(words, 0);") < mem.index("p->piece_last.assign(words * 64"), + "the pieces' table is not sized once, where res_bits is") + begin = block(mem, "void memory_begin_submission() {") + order = [begin.find(s) for s in ("if (mirror_pieces_on() && mirror_epoch - at >= 1200) {", "vk: mirrors in pieces: %zu mirrors hold", + "for (const GuestPool& g : guest_pools) game.push_back({g.lo, g.hi});", "piece_idle::merge(game);", + "piece_idle::count(p.lo, p.hi - p.lo, p.piece_shift,", "are pieces no game pool covers now")] + check(min(order) >= 0 and order == sorted(order), "the count is not made after the mirrors-in-pieces line, every 1200 submissions") + # (the first step only: the change frees and unbinds nothing) + header = (VK / "piece_idle.h").read_text(encoding="utf-8") + for word in ("vma_piece_free", "vmaFreeMemory", "piece_binds", "res_bits[", "__atomic_fetch_and"): + check(word not in touch and word not in header, "the count's code touches the pieces' memory or bits (%s)" % word) + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`RX_VK_MIRROR_PIECE_IDLE=`" in dev and "NOT ported" in dev, "docs/development.md: the switch's row does not say the release is not ported") + + base_env = {k: v for k, v in os.environ.items() if not k.startswith("RX_VK_")} + with tempfile.TemporaryDirectory(prefix="piece-idle-") as tmp: + src, exe = Path(tmp) / "idle.cpp", Path(tmp) / "idle.exe" + src.write_text(PROGRAM.replace("@@CUT@@", idle_for + "\n" + touch), encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(VK), str(src), "-o", str(exe)], check=True) + for value in (None, "0", "600"): + env = dict(base_env) if value is None else dict(base_env, RX_VK_MIRROR_PIECE_IDLE=value) + r = subprocess.run([str(exe), value or "0"], env=env, capture_output=True, text=True, timeout=120) + print(r.stdout.rstrip()) + check(r.returncode == 0, "RX_VK_MIRROR_PIECE_IDLE %s: the checks failed" % value) + print("PASS: the pieces no game pool covers are counted as a page-by-page model counts them; nothing is released") + + +if __name__ == "__main__": + main() diff --git a/tests/test_mirror_spare.py b/tests/test_mirror_spare.py new file mode 100644 index 0000000..e93a2b2 --- /dev/null +++ b/tests/test_mirror_spare.py @@ -0,0 +1,106 @@ +"""What the mirrors' sparse buffers cost is counted (gpu/vk/mirror_spare.h, vk_memory.cpp reap_mirrors and mirror_make; +no game, no GPU). + +(Triforce, 11-21, the performance audit's MEM-27, its first step, after Source A's.) A mirror in pieces is a sparse +buffer, and the driver charges by its size to create and to destroy it; the game frees its pools at an area change and +makes them again at the same sizes. Before freed buffers are kept for the next mirror of their size, this counts what +that would save. + +This compiles mirror_spare.h alone and checks the count: creates and destroys with their time, bytes and longest call; +calls over 4 ms; a create is "servable" only when a buffer of exactly its size was freed before it and not matched by an +earlier create, so two creates of one freed size count one. + +And reads the source: the sparse buffer's destroy and create are timed and counted, the destroy with the pool's span +kept when the mirror was handed to the reaper (the pool's own fields are cleared by then), and the 1200-submission line +has the counts. + + python -I -B tests/test_mirror_spare.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +PROGRAM = r""" +#include +#include +#include "mirror_spare.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +int main() { + using namespace mirror_spare; + const uint64_t MB = 1ull << 20; + Counts c; + CHECK(!created(c, 64 * MB, 3.5)); // (the first of its size: nothing freed yet) + CHECK(!created(c, 256 * MB, 15.0)); + CHECK(c.creates == 2 && c.servable == 0 && c.over == 1 && c.longest_create_ms == 15.0 && c.create_bytes == 320 * MB); + destroyed(c, 64 * MB, 4.5); + destroyed(c, 256 * MB, 18.0); + CHECK(c.destroys == 2 && c.over == 3 && c.longest_destroy_ms == 18.0 && c.destroy_ms == 22.5 && c.destroy_bytes == 320 * MB); + CHECK(created(c, 64 * MB, 3.0)); // (an area change later: the same size again) + CHECK(!created(c, 64 * MB, 3.0)); // (one freed buffer serves one create) + CHECK(!created(c, 128 * MB, 6.0)); // (another size: not served by the 256 MB one) + CHECK(created(c, 256 * MB, 14.0)); + CHECK(c.creates == 6 && c.servable == 2 && c.freed.empty()); + destroyed(c, 64 * MB, 1.0), destroyed(c, 64 * MB, 1.0); + CHECK(created(c, 64 * MB, 1.0) && created(c, 64 * MB, 1.0) && !created(c, 64 * MB, 1.0) && c.servable == 4); + CHECK(kLongMs == 4.0); + return fails ? 1 : 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + mem = (VK / "vk_memory.cpp").read_text(encoding="utf-8") + check('#include "mirror_spare.h"' in mem, "vk_memory.cpp does not use the count") + reap = block(mem, "void reap_mirrors() {") + at = [reap.index(p) for p in ("const auto destroy_t0 = std::chrono::steady_clock::now();", + "vkDestroyBuffer(dev.device, dead_mirrors.front().buffer, nullptr);", + "mirror_spare::destroyed(sparse_ct, dead_mirrors.front().sparse_bytes, sparse_ms_since(destroy_t0));", + "vma_piece_free(a);")] + check(at == sorted(at), "the sparse buffer's destroy is not timed around the call alone, or not counted with its size") + check("uint64_t sparse_bytes = 0; };" in mem and "p.mpieces ? p.hi - p.lo : 0});" in mem, + "the mirror's span is not kept for the reaper: the pool's own fields are cleared by then") + free = block(mem, "void mirror_free(Pool& p) {") + check(free.index("dead_mirrors.push_back(") < free.index("p.mpieces = false"), "the span is read after the pool's mirror state is cleared") + make = block(mem, "bool mirror_make(Pool& pool) {") + at = [make.index(p) for p in ("const auto create_t0 = std::chrono::steady_clock::now();", + "const VkResult sparse_made = vkCreateBuffer(dev.device, &bi, nullptr, &sb);", + "mirror_spare::created(sparse_ct, n, sparse_ms_since(create_t0));")] + check(at == sorted(at), "the sparse buffer's create is not timed around the call alone, or not counted") + check("vk: mirrors' sparse buffers so far: %llu made" in mem and "of the creates had a freed" in mem, "the counts are not in the log") + + with tempfile.TemporaryDirectory(prefix="mirror-spare-") as tmp: + c, exe = Path(tmp) / "spare.cpp", Path(tmp) / "spare.exe" + c.write_text(PROGRAM, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(VK), str(c), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + check(r.returncode == 0, "the count as made:\n" + r.stdout) + print("PASS: the mirrors' sparse buffers' creates and destroys are timed, and the creates a freed buffer could serve counted") + + +if __name__ == "__main__": + main() diff --git a/tests/test_missing_nup.py b/tests/test_missing_nup.py index 68169f9..155f921 100644 --- a/tests/test_missing_nup.py +++ b/tests/test_missing_nup.py @@ -24,6 +24,8 @@ class MissingNupTests(unittest.TestCase): def setUp(self): temporary = tempfile.TemporaryDirectory(prefix="hookshot-missing-nup-") self.addCleanup(temporary.cleanup) + # (the performance audit's IO-47: main() locks its work folder; let go before the folder is deleted) + self.addCleanup(make_game.release_work) self.root = Path(temporary.name).resolve() self.assertEqual(self.root.parent, Path(tempfile.gettempdir()).resolve()) self.package = self.root / "selected game \u6f22 [1].nup" diff --git a/tests/test_mods_entry_input.py b/tests/test_mods_entry_input.py index 7e76e71..383a1ee 100644 --- a/tests/test_mods_entry_input.py +++ b/tests/test_mods_entry_input.py @@ -410,7 +410,8 @@ def main(): "the key's Down that came onto Mods is taken again as the entry's own") check('logf("menu: the host\'s word %d for the Mods entry not taken: %s", enter,' in tick, "an enter word the menu does not take leaves no line") - check("title_anim.step(now, sel, std::min(age, mods_entry::kListShownNs));" in + # ((11-20) and the game's own amount of its buttons with it: tests/test_mods_entry_sync.py) + check("title_anim.step(now, sel, std::min(age, mods_entry::kListShownNs), mods_list_shown());" in block(menu, "void draw_title_mods(ui::Ctx& c, bool sel, float cx, float cy, float k, float pitch) {"), "the title's entry does not take the list's age by the game's frames (a slow frame blinks it)") print("PASS: the Mods entry's input: no Down taken twice, held ones repeated, slow frames keep the list") diff --git a/tests/test_mods_entry_sync.py b/tests/test_mods_entry_sync.py new file mode 100644 index 0000000..41aa22a --- /dev/null +++ b/tests/test_mods_entry_sync.py @@ -0,0 +1,191 @@ +"""The Mods entry comes and goes with the game's own buttons, by the game's own amount each frame (gpu/vk/ +mods_entry_anim.h Anim::step, vk_menu.cpp mods_list_shown, draw_title_mods, draw_mods_row and menu_option_set, host/ +oot_menu.c mods_list_tick, vk_api.h HOS_OPT_MODS_ALPHA; no game). + +(Triforce, 11-20) The owner: "fix the mods button it should be synced with the timing of the other button ui so it +seamlessly moves and disappears with them". The entry under a game list's last button was drawn by a clock of the +renderer's own: 15 frames from the first frame the list ran, and gone 150 ms and a fade after the list's last report. A +probe of the title's list showed the game's own buttons' alpha in each button's box (+0x78, a float): 0 for the 1.5 s +the list runs before it shows, a rise over a second as the title opens, 1 to 0 in one frame as Options is chosen, and +15 frames as the list comes back. So the entry stood there a second and a half before the game's buttons and stayed +after them. The host tells that amount each frame now, and the entry shows as much. + +This compiles mods_entry_anim.h alone and checks: with the game's amount told, the entry shows exactly that while the +list is there (nothing while the game shows nothing, however long the list has run; the rise as it is; all of it), it +does not lag behind a rise by a clock of its own, and a list whose reports stop goes as before (150 ms, then the +UnSelect's fade from where it was); not told (an older host), the list's In by the renderer's clock as before. + +And reads the source: the host reads the amount from the last button's box, takes only a number from 0 to 1, tells it +before the frame's HOS_OPT_MODS_LIST, says at once that the list is not the entry's any more when another screen has +the cursor and the game's buttons are hidden, and leaves an entry not shown yet out of the mouse's reach; the renderer +keeps the amount, takes a list report without HOS_MODS_LIST_SHOWN as gone now, gives the title's entry the amount and +fades the System page's Mods and Quit by it. + + python -I -B tests/test_mods_entry_sync.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +PROGRAM = r""" +#include +#include +#include "mods_entry_anim.h" +using namespace mods_entry; +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +static bool near(float a, float b, float tol = 1e-4f) { return std::fabs(a - b) <= tol; } +int main() { + const int64_t t0 = 7000000000LL, ms = 1000000; + { // the title's opening: the list runs 1.5 s at nothing, then the game's own rise over a second + Anim a; + for (int k = 0; k < 90; k++) { // (1.5 s of frames: the game shows nothing of its buttons) + a.step(t0 + k * kFrameNs, false, 0, 0.0f); + CHECK(a.shown == 0.0f); // (the renderer's own clock had it whole after 15 frames) + } + float last = 0; + for (int k = 0; k <= 60; k++) { // (the game's rise: the entry's, frame for frame) + const float game = (float)(k * k) / 3600.0f; + a.step(t0 + (90 + k) * kFrameNs, false, 0, game); + CHECK(near(a.shown, game) && a.shown >= last); + last = a.shown; + } + CHECK(near(a.shown, 1.0f)); + a.step(t0 + 200 * kFrameNs, false, 0, 1.0f); + CHECK(near(a.shown, 1.0f)); + // Options chosen: the game's buttons at nothing on that frame, the entry too + a.step(t0 + 201 * kFrameNs, false, 0, 0.0f); + CHECK(a.shown == 0.0f); + // the list back: 15 frames by the game's amount, not that times a rise of the entry's own + for (int k = 1; k <= 15; k++) { + const float game = curve(kListIn, (float)k); + a.step(t0 + (300 + k) * kFrameNs, false, 0, game); + CHECK(near(a.shown, game)); + } + } + { // a list whose reports stop (its screen went): there for 150 ms, then the UnSelect's fade from where it was + Anim a; + a.step(t0, false, 0, 0.8f); + CHECK(near(a.shown, 0.8f)); + a.step(t0 + 100 * ms, false, 100 * ms, 0.8f); // (within the 150 ms: still the list's) + CHECK(near(a.shown, 0.8f)); + a.step(t0 + 150 * ms + 4 * kFrameNs, false, kListGoneNs + 4 * kFrameNs, 0.8f); + CHECK(near(a.shown, 0.8f * curve(kUnSelect, 4.0f)) && a.shown < 0.8f && a.shown > 0.0f); + a.step(t0 + 500 * ms, false, kListShownNs, 0.8f); + CHECK(a.shown == 0.0f); + } + { // gone at once: a list said to be the entry's no longer has no age to speak of (INT64_MAX, cut to kListShownNs) + Anim a; + a.step(t0, false, 0, 1.0f); + a.step(t0 + kFrameNs, false, kListShownNs, 1.0f); + CHECK(a.shown == 0.0f); + } + { // not told (an older host): the list's In by the renderer's clock, as before + Anim a; + a.step(t0, false, 0); + CHECK(near(a.shown, 0.0f)); + a.step(t0 + 15 * kFrameNs, false, 0); + CHECK(near(a.shown, 1.0f)); + Anim b; + b.step(t0, false, 0, -1.0f); + b.step(t0 + 8 * kFrameNs, false, 0, -1.0f); + CHECK(near(b.shown, curve(kListIn, 8.0f))); + } + { // the amount the overlay draws the game's alpha with: itself in linear light, a^(1/2.2) in code values + CHECK(list_amount(0.0f, false) == 0.0f && near(list_amount(1.0f, false), 1.0f) && near(list_amount(1.0f, true), 1.0f)); + CHECK(near(list_amount(0.25f, true), 0.25f) && near(list_amount(0.25f, false), 0.5325f, 1e-3f)); + CHECK(near(list_amount(0.5f, false), 0.7297f, 1e-3f) && list_amount(-1.0f, false) == 0.0f && near(list_amount(3.0f, false), 1.0f)); + float last = 0; + for (int k = 0; k <= 100; k++) { + const float v = list_amount((float)k / 100.0f, false); + CHECK(v >= last && v >= (float)k / 100.0f - 1e-6f); // (never back, never dimmer than the game's alpha) + last = v; + } + } + { // the cursor's own animation is not the list's: focused at once, whatever the list shows + Anim a; + a.step(t0, true, 0, 0.5f); + a.step(t0 + 2 * kFrameNs, true, 0, 0.5f); + CHECK(near(a.focus, 1.0f) && near(a.shown, 0.5f)); + } + return fails ? 1 : 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + + api = (VK / "vk_api.h").read_text(encoding="utf-8") + check("enum { HOS_OPT_MODS_ALPHA = 162 };" in api, "HOS_OPT_MODS_ALPHA is not an option of its own number") + + host = (ROOT / "host" / "oot_menu.c").read_text(encoding="utf-8") + tick = block(host, "static void mods_list_tick(uint64_t screen, int title, ModsCall *call) {") + read = "if (!peek(peek64(l.b[last] + 0x58) + 0x78, &alpha, 4) || !(alpha >= 0.0f && alpha <= 1.0f)) alpha = 1.0f;" + check(read in tick, "the host does not read the last button's alpha from its box, or takes anything for it") + tell, report = "rx_option_set(HOS_OPT_MODS_ALPHA, (int)(alpha * 1000.0f + 0.5f));", "rx_option_set(HOS_OPT_MODS_LIST, HOS_MODS_LIST_SHOWN" + check(tell in tick and tick.index(read) < tick.index(tell) < tick.index(report), + "the amount is not told before the frame's list report") + gone = "if (alpha <= 0.0f) rx_option_set(HOS_OPT_MODS_LIST, 0);" + cursor = "if (!mods_cursor_button(screen, &pend) && !mods_held.button) {" + check(cursor in tick and gone in tick and tick.index(read) < tick.index(cursor) < tick.index(gone) < tick.index(tell), + "a list hidden under another screen is not said to be gone at once") + check("if (alpha >= 0.5f) InterlockedExchange64(&mods_entry_ns, (LONG64)now);" in tick, + "an entry the game does not show yet is within the mouse's reach") + + menu = (VK / "vk_menu.cpp").read_text(encoding="utf-8") + check("std::atomic mods_list_alpha{-1};" in menu, "the renderer does not keep the game's amount (not told: -1)") + shown = block(menu, "float mods_list_shown() {") + check("return a < 0 ? -1.0f : mods_entry::list_amount((float)a / 1000.0f, dev.hdr);" in shown, + "mods_list_shown is not the host's thousandths as the overlay has to draw them, or -1 while not told") + setter = block(menu, "void menu_option_set(int id, int value) {") + check("if (id == HOS_OPT_MODS_ALPHA) {" in setter and "mods_list_alpha.store(std::clamp(value, 0, 1000), std::memory_order_relaxed);" in setter, + "the renderer does not take HOS_OPT_MODS_ALPHA") + at = setter.index("if (id == HOS_OPT_MODS_LIST) {") + listed = setter[at:setter.index("if (id == HOS_OPT_MODS_ENTER) {")] + check("if (!(value & HOS_MODS_LIST_SHOWN)) {" in listed and "mods_list_ns.store(0, std::memory_order_release);" in listed and + listed.index("mods_list_ns.store(0,") < listed.index("mods_list_bits.store(value,"), + "a list report without HOS_MODS_LIST_SHOWN is not gone at once, or changes what kind of list it was") + title = block(menu, "void draw_title_mods(ui::Ctx& c, bool sel, float cx, float cy, float k, float pitch) {") + check("title_anim.step(now, sel, std::min(age, mods_entry::kListShownNs), mods_list_shown());" in title, + "the title's entry is not given the game's amount") + row = block(menu, "void draw_mods_row(ui::Ctx& c, bool sel) {") + check("const int vtx_row = c.d->VtxBuffer.Size;" in row and "if (const float list = mods_list_shown(); list >= 0 && list < 1)" in row and + "c.d->VtxBuffer[v].col = ui::alpha(c.d->VtxBuffer[v].col, list);" in row and + row.index("const int vtx_row") < row.index("ui::button(c, a, b, names[i], mods_focus[i].shown);") < row.index("list >= 0 && list < 1"), + "the System page's Mods and Quit are not faded by the game's amount") + + with tempfile.TemporaryDirectory(prefix="mods-entry-sync-") as tmp: + src, exe = Path(tmp) / "sync.cpp", Path(tmp) / "sync.exe" + src.write_text(PROGRAM, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(VK), str(src), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + check(r.returncode == 0, "the entry by the game's amount:\n" + r.stdout) + print("PASS: the Mods entry shows as much as the game's own buttons, frame for frame, and goes when they do") + + +if __name__ == "__main__": + main() diff --git a/tests/test_mods_music.py b/tests/test_mods_music.py new file mode 100644 index 0000000..a28f588 --- /dev/null +++ b/tests/test_mods_music.py @@ -0,0 +1,234 @@ +"""The Mods screen's music: the game's shop theme under the Mods screen opened from the title, the title's own music +faded out meanwhile (host/mods_music.h and mods_music.c, host/nn_audio.c render_frame, gpu/vk/vk_menu.cpp +menu_mods_music; no game, no sound device). + +(Triforce, 11-20) The owner: "have the in game mods menu in the title screen play the shop theme quietly in the +background ... it mutes the title screen music's audio while in the mod menu and fades in/out". This compiles +mods_music.h alone and checks the rule, with a made-up tune and a made-up game mix: + +- with no tune (not read, or not in the game's files) a block is the game's as it is, whatever is asked; +- the screen opening: the level rises a sample at a time to 1 over MODS_MUSIC_FADE samples, never stepping; every one + of the game's channels has 1 - level, down to nothing; the tune is in the front pair times level times its gain; +- the tune goes round between the loop points its file names, and a tune with none plays once and is silent after; +- the screen shutting: the level falls back to 0 the same way, the game's own sound is whole again, and the tune is at + its top for the next opening; a shut and silent block is not touched at all; +- a loop outside the tune, or too short to be one, is no loop. + +And reads the source: the sound thread calls it after the game's limiter and before the options' own sounds (which +keep their level) and the Master volume; mods_music.c asks the options (HOS_OPT_MODS_MUSIC) and the Menu sounds row, +reads the shop theme through nn_fs.c (a mod's file where a mod replaces it) on a thread of its own, and takes the +loop from the file; the options answer 2 only for the Mods screen opened from the title's list. + + python -I -B tests/test_mods_music.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +PROGRAM = r""" +#include +#include +#include +#include "mods_music.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +static bool near(float a, float b, float tol = 1e-4f) { return std::fabs(a - b) <= tol; } +constexpr uint32_t kBlock = 240; // (nn_audio.c's block: 5 ms) + +struct Mix { + std::vector ch[6]; + float* p[6]; + explicit Mix(int nch) { + for (int c = 0; c < 6; c++) ch[c].assign(kBlock, c < nch ? 0.5f + 0.1f * (float)c : 0.0f), p[c] = ch[c].data(); + } +}; + +int main() { + // a tune: 2 s, its left the frame's number over its length, its right the negative + const uint32_t frames = 2 * MODS_MUSIC_RATE; + std::vector tune((size_t)frames * 2); + for (uint32_t i = 0; i < frames; i++) tune[2 * (size_t)i] = (float)i / (float)frames, tune[2 * (size_t)i + 1] = -(float)i / (float)frames; + + { // no tune: the game's block as it is + ModsMusic m{}; + Mix x(2); + CHECK(mods_music_block(&m, 1, x.p, 2, x.p[0], x.p[1], kBlock, 1.0f) == 0.0f); + CHECK(x.ch[0][0] == 0.5f && x.ch[1][kBlock - 1] == 0.6f && m.level == 0.0f); + } + + ModsMusic m{}; + m.pcm = tune.data(), m.frames = frames; + mods_music_loop(&m, MODS_MUSIC_RATE / 2, frames); // (round from 0.5 s to its end) + CHECK(m.loop_start == MODS_MUSIC_RATE / 2 && m.loop_end == frames); + + { // shut and silent: not touched + Mix x(6); + CHECK(mods_music_block(&m, 0, x.p, 6, x.p[0], x.p[1], kBlock, 0.35f) == 0.0f); + for (int c = 0; c < 6; c++) CHECK(x.ch[c][0] == 0.5f + 0.1f * (float)c && x.ch[c][kBlock - 1] == 0.5f + 0.1f * (float)c); + CHECK(m.pos == 0); + } + + // the screen opens: the level up a sample at a time, the game's six channels down, the tune in the front pair + const float gain = 0.35f; + float last = 0.0f; + uint32_t played = 0; + for (uint32_t done = 0; done < MODS_MUSIC_FADE + 2 * kBlock; done += kBlock) { + Mix x(6); + const float before = m.level; + const uint32_t pos = m.pos; + const float level = mods_music_block(&m, 1, x.p, 6, x.p[0], x.p[1], kBlock, gain); + CHECK(level >= last && level <= 1.0f && level - before <= (float)kBlock / (float)MODS_MUSIC_FADE + 1e-4f); + for (uint32_t i = 0; i < kBlock; i += 79) { + float l = before + (float)(i + 1) / (float)MODS_MUSIC_FADE; + if (l > 1.0f) l = 1.0f; + for (int c = 2; c < 6; c++) CHECK(near(x.ch[c][i], (0.5f + 0.1f * (float)c) * (1.0f - l))); // (every channel) + const float t = tune[2 * (size_t)(pos + i)]; + CHECK(near(x.ch[0][i], 0.5f * (1.0f - l) + t * l * gain) && near(x.ch[1][i], 0.6f * (1.0f - l) - t * l * gain)); + } + last = level, played += kBlock; + CHECK(m.pos == played); // (no loop yet: 0.6 s in) + } + CHECK(near(m.level, 1.0f, 1e-6f)); + { // all there: the game's own sound is nothing, the tune alone times its gain + Mix x(2); + const uint32_t pos = m.pos; + mods_music_block(&m, 1, x.p, 2, x.p[0], x.p[1], kBlock, gain); + CHECK(near(x.ch[0][10], tune[2 * (size_t)(pos + 10)] * gain) && near(x.ch[1][10], tune[2 * (size_t)(pos + 10) + 1] * gain)); + } + + // round the loop: past the tune's end it is at the loop's start, never past the end + bool wrapped = false; + for (int k = 0; k < 600; k++) { + Mix x(2); + const uint32_t before = m.pos; + mods_music_block(&m, 1, x.p, 2, x.p[0], x.p[1], kBlock, gain); + CHECK(m.pos < frames && m.pos >= (wrapped ? MODS_MUSIC_RATE / 2 : 0u)); + if (m.pos < before) { + wrapped = true; + CHECK(m.pos == MODS_MUSIC_RATE / 2 + (before + kBlock - frames)); + } + } + CHECK(wrapped); + + // the screen shuts: down the same way, then the game's own whole and the tune at its top + last = m.level; + for (uint32_t done = 0; done < MODS_MUSIC_FADE + 2 * kBlock; done += kBlock) { + Mix x(2); + const float before = m.level; + const float level = mods_music_block(&m, 0, x.p, 2, x.p[0], x.p[1], kBlock, gain); + CHECK(level <= last && level >= 0.0f && before - level <= (float)kBlock / (float)MODS_MUSIC_FADE + 1e-4f); + last = level; + } + CHECK(m.level == 0.0f && m.pos == 0); + { + Mix x(2); + mods_music_block(&m, 0, x.p, 2, x.p[0], x.p[1], kBlock, gain); + CHECK(x.ch[0][0] == 0.5f && x.ch[1][kBlock - 1] == 0.6f); + } + { // shut half way through its coming: back down from where it is, no step + for (int k = 0; k < 40; k++) { Mix x(2); mods_music_block(&m, 1, x.p, 2, x.p[0], x.p[1], kBlock, gain); } + const float half = m.level; + CHECK(half > 0.2f && half < 0.5f); + Mix x(2); + CHECK(near(mods_music_block(&m, 0, x.p, 2, x.p[0], x.p[1], kBlock, gain), half - (float)kBlock / (float)MODS_MUSIC_FADE)); + } + + // a tune with no loop: once, then silence, the game's own still down while the screen is open + ModsMusic once{}; + once.pcm = tune.data(), once.frames = MODS_MUSIC_RATE; // (its first second) + mods_music_loop(&once, 0, 0); + CHECK(once.loop_end == 0); + for (int k = 0; k < 400; k++) { Mix x(2); mods_music_block(&once, 1, x.p, 2, x.p[0], x.p[1], kBlock, gain); } + CHECK(once.pos == once.frames && once.level == 1.0f); + { + Mix x(2); + mods_music_block(&once, 1, x.p, 2, x.p[0], x.p[1], kBlock, gain); + CHECK(x.ch[0][5] == 0.0f && x.ch[1][5] == 0.0f); + } + // loops that are none: past the tune (cut to it), the wrong way round, too short + mods_music_loop(&once, 100, 0xfffffff0u); + CHECK(once.loop_start == 100 && once.loop_end == once.frames); + mods_music_loop(&once, 5000, 4000); + CHECK(once.loop_end == 0 && once.loop_start == 0); + mods_music_loop(&once, 1000, 1000 + MODS_MUSIC_RATE / 20); + CHECK(once.loop_end == 0); + return fails ? 1 : 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + host, vk = ROOT / "host", ROOT / "gpu" / "vk" + + audio = (host / "nn_audio.c").read_text(encoding="utf-8") + frame = audio[audio.index("static void render_frame("):] + frame = frame[:frame.index("\n}\n") + 3] + call = "hos_mods_music_frame(channels, nch < 6 ? nch : 6, FRAME);" + check(call in frame, "render_frame does not call the Mods screen's music") + check(frame.index("if (lim_enabled)") < frame.index(call) < frame.index("hos_menu_sound_frame(out[0], out[1], FRAME);") < + frame.index("the Insert menu's Master volume"), + "the music is not after the game's limiter, before the options' sounds and the Master volume") + + music = (host / "mods_music.c").read_text(encoding="utf-8") + tick = block(music, "void hos_mods_music_frame(float *const *channels, int nch, uint32_t frames) {") + check("rx_option(HOS_OPT_MODS_MUSIC, 0)" in tick and "rx_option(HOS_OPT_MENU_SOUND_VOLUME, -1)" in tick, + "the sound thread does not ask the options for the screen and the Menu sounds row") + check("const int want = mode == 2 && volume > 0;" in tick, "the tune is wanted without the Mods screen open, or with the sounds off") + check("mods_music_block(&music, want, channels, nch," in tick, "the block is not the rule's") + check("CreateThread(NULL, 0, load_main" in tick and "atomic_compare_exchange_strong(&loaded, &none, 1)" in tick, + "the tune is not read once, on a thread of its own") + load = block(music, "static DWORD WINAPI load_main(void *arg) {") + check('"Sound/Resource/Stream/BGM_Shop_Market_Mask.bwav"' in music and "hos_fs_rom_file(files[i], FILE_MOST, &size)" in load, + "the shop theme is not read from the game's files through nn_fs.c") + check("hos_menu_sound_decode_long(b, size, &frames, SECONDS_MOST)" in load and "loop_end = u32(b + 0x10 + 0x3c), loop_start = u32(b + 0x10 + 0x40)" in load, + "the tune is not menu_sound.c's decoder's, or its loop is not its file's") + sound = (host / "menu_sound.c").read_text(encoding="utf-8") + check("float *hos_menu_sound_decode(const uint8_t *b, uint64_t size, uint32_t *frames_out) { return decode_most(b, size, frames_out, 10); }" in sound, + "the interface sounds are no longer held to ten seconds") + + api = (vk / "vk_api.h").read_text(encoding="utf-8") + check("enum { HOS_OPT_MODS_MUSIC = 161 };" in api and "HOS_OPT_TEXT_CHAR = 159, HOS_OPT_TEXT_WANTED };" in api, + "HOS_OPT_MODS_MUSIC is not an option of its own number") + replay = (vk / "vk_replay.cpp").read_text(encoding="utf-8") + check("if (id == HOS_OPT_MODS_MUSIC) return menu_mods_music();" in replay, "the renderer does not answer HOS_OPT_MODS_MUSIC") + menu = (vk / "vk_menu.cpp").read_text(encoding="utf-8") + answer = block(menu, "int menu_mods_music() {") + check("if (!mods_title_list()) return 0;" in answer and + "if (open_now.load(std::memory_order_relaxed) && page == kModsPage) return 2;" in answer and + answer.index("mods_title_list()") < answer.index("return 2;"), + "the music is not for the Mods screen opened from the title's list alone") + + with tempfile.TemporaryDirectory(prefix="mods-music-") as tmp: + src, exe = Path(tmp) / "music.cpp", Path(tmp) / "music.exe" + src.write_text(PROGRAM, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(host), str(src), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + check(r.returncode == 0, "the Mods screen's music as made:\n" + r.stdout) + print("PASS: the shop theme comes under the Mods screen opened from the title and the title's music goes, 0.6 s each way") + + +if __name__ == "__main__": + main() diff --git a/tests/test_mods_page_anim.py b/tests/test_mods_page_anim.py new file mode 100644 index 0000000..abb3a13 --- /dev/null +++ b/tests/test_mods_page_anim.py @@ -0,0 +1,133 @@ +"""The Mods screen as it opens: it fades in as the game's lists and windows do (gpu/vk/mods_page_anim.h, vk_menu.cpp +draw_menu and menu_tick; no game). + +(Triforce, 11-20) The owner's "Mod menu fade in". The entry that opens the Mods screen is animated as the game's own (the +title's, and Mods and Quit under the pause menu's System list), and Quit's dialog opens with the game's window's In; the +screen itself came whole on the frame after the press. This compiles mods_page_anim.h alone and checks: + +- the In's alpha: nothing at its start, all of it at frame 15, never going back on the way, slow to start (under a third + half way, as the layouts' slopes 0.02 and 0.14 make it); +- the screen from tick to tick: nothing of it shows before its first step (a drawing between the press and the tick's end + must not show it whole for a frame), its In runs from the tick it came at, it takes input once the In has played, a + screen that goes and comes again plays its In again, and a still shows it as it stands. + +And reads the source for what only the game shows: the screen stepped at every tick's end with whether it is the page +shown; no input passed to it while its In plays; everything draw_menu draws from the page's first vertex faded by the +In's amount, the Full panel left as it is; and the entries that opened it still drawn under it while the In plays, going +as the screen comes. + + python -I -B tests/test_mods_page_anim.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def function(text, head): + start = text.index(head) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +PROGRAM = r""" +#include +#include +#include "mods_page_anim.h" +using namespace mods_page; +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +static bool near(float a, float b, float tol = 0.01f) { return std::fabs(a - b) <= tol; } +int main() { + // the In's alpha + CHECK(kInNs == 15 * kFrameNs); + CHECK(near(in_alpha(0), 0) && near(in_alpha(-5), 0) && near(in_alpha(kInNs), 1) && near(in_alpha(100 * kFrameNs), 1)); + float last = 0; + for (int64_t t = 0; t <= kInNs; t += kFrameNs / 4) { + const float a = in_alpha(t); + CHECK(a >= last - 1e-4f && a >= 0 && a <= 1); + last = a; + } + CHECK(in_alpha(kInNs / 2) > 0.2f && in_alpha(kInNs / 2) < 0.34f); // (slow to start) + CHECK(in_alpha(3 * kFrameNs) < 0.1f && in_alpha(12 * kFrameNs) > 0.6f && in_alpha(14 * kFrameNs) > 0.85f); + + // the screen from tick to tick + const int64_t t0 = 5000000000LL; + Page p; + CHECK(near(p.alpha(t0), 0) && !p.ready(t0)); // (asked for, not stepped yet: nothing of it, no input) + CHECK(near(p.alpha(t0, true), 1)); // (a still: as it stands) + p.step(t0, true); + CHECK(p.shown && p.since_ns == t0); + CHECK(near(p.alpha(t0), 0) && !p.ready(t0)); + p.step(t0 + 4 * kFrameNs, true); // (the ticks after: the same In, not a new one) + CHECK(p.since_ns == t0); + CHECK(p.alpha(t0 + 4 * kFrameNs) > 0 && p.alpha(t0 + 4 * kFrameNs) < 0.2f && !p.ready(t0 + 4 * kFrameNs)); + CHECK(!p.ready(t0 + kInNs - 1) && p.ready(t0 + kInNs) && near(p.alpha(t0 + kInNs), 1)); + CHECK(near(p.alpha(t0 + 4 * kFrameNs, true), 1)); // (a picture that compares) + p.step(t0 + 60 * kFrameNs, false); // (Back: gone at once) + CHECK(!p.shown && p.since_ns == 0 && !p.ready(t0 + 60 * kFrameNs)); + p.step(t0 + 90 * kFrameNs, true); // (opened again: its In again) + CHECK(p.since_ns == t0 + 90 * kFrameNs && near(p.alpha(t0 + 90 * kFrameNs), 0) && !p.ready(t0 + 91 * kFrameNs)); + CHECK(p.ready(t0 + 90 * kFrameNs + kInNs)); + return fails ? 1 : 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + menu = (VK / "vk_menu.cpp").read_text(encoding="utf-8") + + check('#include "mods_page_anim.h"' in menu and "mods_page::Page mods_screen;" in menu, "vk_menu.cpp does not keep the Mods screen's In") + alpha = function(menu, "float mods_screen_alpha() {") + check("page == kModsPage ? mods_screen.alpha(prof_now(), !ease_live || still) : 1.0f" in alpha, + "the In's amount is not the Mods screen's alone, or a still does not show it as it stands") + + tick = function(menu, "void menu_tick() {") + check("mods_screen.step(prof_now(), open_now.load() && page == kModsPage);" in tick, "the screen is not stepped at every tick") + check(tick.index("mods_screen.step(") > tick.index("if (mods_screen.ready(prof_now())) mods_tick("), + "the screen is stepped before the page's input: the tick that opens it would pass input on") + check(tick.count("mods_tick(press, tab_left, tab_right, mouse, moved, ptr, mx, my, changed);") == 1 and + "if (mods_screen.ready(prof_now())) mods_tick(press, tab_left, tab_right, mouse, moved, ptr, mx, my, changed);" in tick, + "the Mods screen takes input while its In plays") + + draw = function(menu, "void draw_menu(ui::Ctx& c, bool perf_under) {") + first, dim = draw.index("const int vtx_page = c.d->VtxBuffer.Size;"), draw.index("ui::system_bg(c)") + check(first < dim, "the page's first vertex is taken after its background: the background would come at once") + fade = "for (int i = vtx_page; mods_in < 1 && i < c.d->VtxBuffer.Size; i++)" + check(fade in draw and "if (i < perf0 || i >= perf1) c.d->VtxBuffer[i].col = ui::alpha(c.d->VtxBuffer[i].col, mods_in);" in draw, + "the page is not faded by the In's amount, or the Full panel fades with it") + check(draw.index("panel_between(c.d);") < draw.index(fade) < draw.index("put_on_grid(c.d, vtx0, cmd0, menu_map, window_clip, perf0, perf1);"), + "the fade is not over the whole page: after the panel's layer is merged, before the page is put on its grid") + check(draw.index(fade) < draw.index("if (menu_map.ox <= 0 && menu_map.oy <= 0 && menu_map.kx == 1) return;"), + "a 16:9 window returns before the fade") + under = draw.index("if (mods_in < 1 && (mods_list_fresh() || mods_title_fading())) {") + check(under < first and "draw_mods_row_shut(c);" in draw[under:first] and "ui::alpha(c.d->VtxBuffer[i].col, 1 - mods_in)" in draw[under:first], + "the entries that opened the screen are not drawn under it, going as it comes, while its In plays") + + with tempfile.TemporaryDirectory(prefix="mods-page-anim-") as tmp: + src, exe = Path(tmp) / "anim.cpp", Path(tmp) / "anim.exe" + src.write_text(PROGRAM, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(VK), str(src), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + check(r.returncode == 0, "the Mods screen's In as made:\n" + r.stdout) + print("PASS: the Mods screen fades in as the game's lists and windows, takes input after, and never shows whole first") + + +if __name__ == "__main__": + main() diff --git a/tests/test_movie_pace.py b/tests/test_movie_pace.py new file mode 100644 index 0000000..45b8fd2 --- /dev/null +++ b/tests/test_movie_pace.py @@ -0,0 +1,303 @@ +"""While a movie plays at a frame rate cap of exactly 60 the frames are paced by 240, with the game's limiter and its time +step following, and the cap is back a quarter second after the movie player's last update (host/movie_pace.h, +host/nn_vi.c movie_pace_decide, pace_fps, realtime_step and hos_vi_native_presented; no game). + +(Triforce, 11-22, the performance audit's PRESENT-30, after Source A's.) The game's movie player takes one picture an +update at most and its updates follow the frame loop, so at the 60 cap every frame that came late was a picture the +decoder skipped. HOS_MOVIE_PACE=0 keeps the cap through movies, as before. + +This compiles movie_pace.h alone (when the pace is raised; the limiter raised, kept, and put back unless the game wrote +over it; the game's rate slot brought back down), reads the source (the decision once a present, before the period; +realtime_step by the pace and no longer by the cap; the Steady row and the vsync event still by the cap; the switch +read once and in the switches table), and runs nn_vi.c's own fps_max, movie_seen, movie_plays, pace_fps, +movie_pace_decide and realtime_step, cut out of the file, on a clock of its own: +- the decision: 240 only at a cap of 60 with a movie player's update in the last 250 ms; not with HOS_MOVIE_PACE=0, not + with the Steady frame rate row on (by its environment switch or its options line), not at a cap of 45, 120 or 240; + still 240 at 249 ms after the update and the cap again at 251; one line in the log at the start and one at the end; +- the step, through a movie between two stretches at 60: the limiter goes 60, 240, 60; with the frame-time hook the game + is given each frame's own time during the movie (1/240 s) and none before or after, its rate slot staying 60; without + the hook the slot follows the pace to 240 and comes back to 60 with it, where the step goes on moving it. + + python -I -B tests/test_movie_pace.py [--cc clang] +""" +import argparse +import os +import re +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +HOST = ROOT / "host" +RUN_ALL_ARGS = ["--cc", "clang"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(source, marker): + """From marker to the first line that starts with a closing brace""" + start = source.index(marker) + return source[start:start + re.search(r"^}", source[start:], re.M).end()] + + +RULES = r""" +#include +#include "movie_pace.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +int main(void) { + /* movie_pace_raise(on, cap, steady, movie) */ + CHECK(MOVIE_PACE_FPS == 240); + CHECK(movie_pace_raise(1, 60, 0, 1)); + CHECK(!movie_pace_raise(1, 60, 0, 0)); /* no movie */ + CHECK(!movie_pace_raise(0, 60, 0, 1)); /* HOS_MOVIE_PACE=0 */ + CHECK(!movie_pace_raise(1, 60, 1, 1)); /* the Steady frame rate row */ + for (int cap = 20; cap <= 240; cap++) CHECK(movie_pace_raise(1, cap, 0, 1) == (cap == 60)); /* a cap of exactly 60 */ + CHECK(movie_pace_fps(1, 60) == 240 && movie_pace_fps(0, 60) == 60 && movie_pace_fps(0, 144) == 144 && movie_pace_fps(0, 45) == 45); + + /* the game's limiter: 60, raised through a movie, back after it */ + MovieLimit m = {0, 0}; + CHECK(movie_pace_limit(&m, 60, 60) == 60 && !m.before); + CHECK(movie_pace_limit(&m, 60, 240) == 240 && m.before == 60); + CHECK(movie_pace_limit(&m, 240, 240) == 240 && m.before == 60); /* (already there: nothing to do, the old value kept) */ + CHECK(movie_pace_limit(&m, 240, 60) == 60 && !m.before && !m.raised); + CHECK(movie_pace_limit(&m, 60, 60) == 60); + /* the game lowered it during the movie (its own controller): raised again, and what it was before the first raise kept */ + CHECK(movie_pace_limit(&m, 60, 240) == 240 && movie_pace_limit(&m, 90, 240) == 240 && m.before == 60); + CHECK(movie_pace_limit(&m, 240, 60) == 60); + /* the game wrote another value over the raised one: left as the game has it */ + CHECK(movie_pace_limit(&m, 60, 240) == 240); + CHECK(movie_pace_limit(&m, 300, 240) == 300 && movie_pace_limit(&m, 300, 60) == 300 && !m.before); + /* no limiter (0 or less): never touched */ + CHECK(movie_pace_limit(&m, 0, 240) == 0 && movie_pace_limit(&m, -1, 240) == -1 && movie_pace_limit(&m, 0, 60) == 0 && !m.before); + /* a cap above 60 for good: raised and never lowered, as before */ + MovieLimit c = {0, 0}; + for (int i = 0; i < 5; i++) CHECK(movie_pace_limit(&c, i ? 144 : 60, 144) == 144); + /* a cap under 60: left alone */ + MovieLimit low = {0, 0}; + CHECK(movie_pace_limit(&low, 60, 45) == 60 && !low.before); + + /* movie_pace_slot(slot, the pace before, the pace now) */ + CHECK(movie_pace_slot(240, 240, 60) == 60 && movie_pace_slot(100, 240, 60) == 60); /* the raise ended: back to 60 */ + CHECK(movie_pace_slot(60, 240, 60) == 60 && movie_pace_slot(40, 240, 60) == 40); /* (60 or less: the step's own) */ + CHECK(movie_pace_slot(240, 240, 240) == 240 && movie_pace_slot(240, 60, 240) == 240);/* during it, and at its start */ + CHECK(movie_pace_slot(240, 60, 60) == 240 && movie_pace_slot(300, 240, 60) == 300); /* (not the step's: left alone) */ + return fails ? 1 : 0; +} +""" + +FIXTURE = r'''#define _CRT_SECURE_NO_WARNINGS +#include +#include +#include +#include +#include +#include "step_steady.h" +#include "pace_steady.h" +#include "movie_pace.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +typedef long long LONG64; +static int64_t clock_ns = 1000000000LL; +int64_t hos_now_ns(void) { return clock_ns; } +static char log_text[1 << 15]; +static void hos_log(const char *fmt, ...) { + va_list ap; + const size_t n = strlen(log_text); + va_start(ap, fmt); + if (n + 600 < sizeof log_text) vsnprintf(log_text + n, 500, fmt, ap), strcat(log_text, "\n"); + va_end(ap); +} +static int logged(const char *text) { + int n = 0; + for (const char *p = log_text; (p = strstr(p, text)) != NULL; p += strlen(text)) n++; + return n; +} +static int64_t pace_period_ns; /* (the period the presents are paced by: hos_vi_native_presented) */ +static uint8_t guest[0x4000]; +#define GP(a) ((void *)(guest + (a))) +#define MOD 0x1000u +int rx_option(int id, int otherwise) { (void)id; return otherwise; } +static int steady_row; /* (the options file's steady_frame_rate) */ +int hos_ini_option(const char *key, int otherwise) { return strcmp(key, "steady_frame_rate") ? otherwise : steady_row; } +enum { HOS_OPT_DISPLAY_CAP = 60 }; +static volatile long long presents; +volatile uint64_t hos_gfx_module = MOD; +volatile int hos_fixed_resolution, hos_dynres_runs; +volatile uint32_t hos_fixed_scale_bits; +volatile float hos_text_scale; +volatile uint64_t hos_step_probe, hos_res_module; +volatile uint32_t hos_frame_seconds_bits; +@HOST@ +/* one frame: the movie player's update when a movie plays, the frame's time, then what the present does in this order */ +static void present(int movie, int64_t frame_ns) { + if (movie) movie_seen(); + clock_ns += frame_ns; + movie_pace_decide(); + pace_period_ns = 1000000000LL / pace_fps(); + realtime_step(clock_ns); +} +static float seconds_given(void) { /* (what the frame-time hook reads: 0 when the game steps by its own rate) */ + const uint32_t bits = hos_frame_seconds_bits; + float f; + memcpy(&f, &bits, 4); + return bits ? f : 0.0f; +} +static int near(float a, double b) { return a > b * 0.999 && a < b * 1.001; } +int main(int argc, char **argv) { + const char *mode = argc > 1 ? argv[1] : ""; + const int cap = argc > 2 ? atoi(argv[2]) : 60, raised = argc > 3 ? atoi(argv[3]) : 240; + const int64_t ms = 1000000; + int *limit = (int *)GP(MOD + 0xfc), *rate = (int *)GP(MOD + 0x2c8); + steady_row = getenv("SI_STEADY_ROW") ? atoi(getenv("SI_STEADY_ROW")) : 0; + *limit = 60, *rate = 60; + if (!strcmp(mode, "decide")) { + CHECK(pace_fps() == cap); + movie_pace_decide(); + CHECK(pace_fps() == cap && fps_max() == cap); /* no movie player has updated */ + for (int movie = 1; movie <= 2; movie++) { + movie_seen(); + clock_ns += 4 * ms; + movie_pace_decide(); + CHECK(pace_fps() == raised && fps_max() == cap); /* a movie plays (the cap itself never moves) */ + CHECK(pace_fps() == raised); /* (asked again within the present: the same) */ + clock_ns += 245 * ms; /* 249 ms after the update: still */ + movie_pace_decide(); + CHECK(pace_fps() == raised); + clock_ns += 2 * ms; /* 251 ms: the cap again */ + movie_pace_decide(); + CHECK(pace_fps() == cap); + CHECK(logged("vi: a movie plays at the 60 cap: frames paced at 240") == (raised != cap ? movie : 0)); + CHECK(logged("vi: the movie is over: frames paced at the 60 cap again") == (raised != cap ? movie : 0)); + clock_ns += 5000 * ms; + movie_pace_decide(); + CHECK(pace_fps() == cap); + } + } else { + const int hook = !strcmp(mode, "step"), off = !strcmp(mode, "step-off"); + hos_step_probe = hook || off ? 0x2000 : 0; /* (the lifted code sets it when the frame-time hook is in the build) */ + for (int i = 0; i < 120; i++) { /* before the movie: 60 */ + present(0, 1000000000LL / 60); + CHECK(pace_fps() == 60 && *limit == 60 && *rate == 60 && seconds_given() == 0.0f); + } + for (int i = 0; i < 480; i++) { /* the movie: its player updates every frame */ + present(1, 1000000000LL / pace_fps()); + if (off) { + CHECK(pace_fps() == 60 && *limit == 60 && *rate == 60 && seconds_given() == 0.0f); + continue; + } + CHECK(pace_fps() == 240 && pace_period_ns == 1000000000LL / 240 && *limit == 240); + if (hook && i > 20) CHECK(near(seconds_given(), 1.0 / 240) && *rate == 60); /* (each frame's own time; the rate the engine's) */ + if (!hook && i > 20) CHECK(seconds_given() == 0.0f && *rate == 240); /* (no hook: the rate up to the pace) */ + } + int still = 0; + for (int i = 0; i < 200; i++) { /* its player has stopped: a quarter second more, then 60 */ + present(0, 1000000000LL / pace_fps()); + still += pace_fps() == 240; + } + CHECK(off ? still == 0 : still >= 55 && still <= 60); /* (250 ms of frames 4.17 ms apart) */ + CHECK(pace_fps() == 60 && pace_period_ns == 1000000000LL / 60); + CHECK(*limit == 60); /* the limiter where it was */ + CHECK(*rate == 60 && seconds_given() == 0.0f); /* the game steps by its own rate again, and the step still moves it */ + for (int i = 0; i < 30; i++) present(0, 1000000000LL / 40); + CHECK(*rate >= 36 && *rate <= 41 && *limit == 60); /* (frames of 25 ms: the rate follows them, a hitch repaid slowly) */ + CHECK(logged("vi: a movie plays at the 60 cap") == !off && logged("vi: the movie is over") == !off); + CHECK(!logged("no real-time step")); + } + if (fails) fputs(log_text, stdout); + printf("%s: %s %d %d\n", fails ? "FAILED" : "PASS", mode, cap, raised); + return fails ? 1 : 0; +} +''' + +# (mode, the cap expected, the pace expected while a movie plays, the environment) +CASES = [ + ("decide", 60, 240, {}), + ("decide", 60, 240, {"HOS_MOVIE_PACE": "1"}), + ("decide", 60, 60, {"HOS_MOVIE_PACE": "0"}), # the switch: the cap kept + ("decide", 60, 60, {"HOS_FPS_STEADY": "1"}), # the Steady frame rate row, by its switch + ("decide", 60, 60, {"HOS_FPS_STEADY": "2"}), + ("decide", 60, 60, {"SI_STEADY_ROW": "1"}), # ... and by its line in the options file + ("decide", 60, 240, {"SI_STEADY_ROW": "2", "HOS_FPS_STEADY": "0"}), # (the switch over the line, as steady_period reads it) + ("decide", 60, 240, {"SI_STEADY_ROW": "7"}), # (a value that is no row: off) + ("decide", 45, 45, {"HOS_FPS_MAX": "45"}), # any other cap: never + ("decide", 120, 120, {"HOS_FPS_MAX": "120"}), + ("decide", 240, 240, {"HOS_FPS_MAX": "240"}), + ("step", 60, 240, {}), + ("step-nohook", 60, 240, {}), + ("step", 60, 240, {"HOS_STEP_STEADY": "0"}), # (the mean of five in place of the steady step) + ("step-off", 60, 60, {"HOS_MOVIE_PACE": "0"}), +] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cc", default="clang") + args = parser.parse_args() + vi = (HOST / "nn_vi.c").read_text(encoding="utf-8") + + check('#include "movie_pace.h"' in vi, "nn_vi.c does not take movie_pace.h") + check("return movie_pace_fps(movie_raise, fps_max());" in block(vi, "static int pace_fps(void) {"), "pace_fps is not the rule's") + decide = block(vi, "static void movie_pace_decide(void) {") + check('if (on < 0) on = !(getenv("HOS_MOVIE_PACE") && *getenv("HOS_MOVIE_PACE") && atoi(getenv("HOS_MOVIE_PACE")) == 0);' in decide, + "HOS_MOVIE_PACE is not read once, on unless 0") + check("const int raise = movie_pace_raise(on, fps_max(), steady != PACE_STEADY_OFF, movie_plays());" in decide + and "movie_raise = raise;" in decide, "the decision is not the rule's, from the cap, the Steady row and the movie's stamp") + steady = block(vi, "static int64_t steady_period(int64_t base, int64_t now) {") + for read in ('getenv("HOS_FPS_STEADY");', 'e && *e ? atoi(e) : hos_ini_option("steady_frame_rate", PACE_STEADY_OFF);'): + check(read in decide and read in steady, "the decision does not read the Steady frame rate row as steady_period does: " + read) + check("if (cap != fps_max()) {" in steady and "pace_fps()" not in steady, "the Steady frame rate row no longer goes by the cap itself") + check("return seen && hos_now_ns() - seen < 250000000LL;" in block(vi, "static int movie_plays(void) {"), + "a movie no longer plays for a quarter second after its player's update") + check("movie_seen();" in block(vi, "uint32_t hos_movie_frames(uint64_t player) {")[:400], "a movie player's update is not noted") + + presented = block(vi, "void hos_vi_native_presented(void) {") + order = [presented.index(s) for s in ("const int interval = fps_max() < 60 ? 1 : present_interval;", "movie_pace_decide();", + "const int64_t period = steady_period(1000000000LL / pace_fps() * interval, now);", + "pace_period_ns = period;", "realtime_step(now);")] + check(order == sorted(order) and presented.count("movie_pace_decide();") == 1, + "the present does not decide once, before the period it paces by and the step") + step = block(vi, "static void realtime_step(int64_t now) {") + check("fps_max()" not in step and step.count("pace_fps()") == 1 and "const int pace = pace_fps();" in step, + "realtime_step does not go by the pace, asked once") + for line in ("const int was = *limit, to = movie_pace_limit(&raised, was, pace);", "if (to != was) *limit = to;", + "const int was = *slot, back = movie_pace_slot(was, before, pace);", "if (back != was) *slot = back;", + "if (!rx_option(0, 1) && pace == 60) {", "const int timed = pace > 60 && timed_step && hos_step_probe;", + "const int top = timed ? 60 : pace;", "if (*slot < 15 || *slot > (pace > 60 ? pace : 60)) {", + "if (paced && paced != pace) { count = next = 0; carry = 0; step_steady_reset(&steady); }"): + check(line in step, "realtime_step lost: " + line) + check(step.index("const int pace = pace_fps();") < step.index("movie_pace_slot(") < step.index("movie_pace_limit(") + < step.index("if (!rx_option(0, 1) && pace == 60) {"), "the slot and the limiter are not settled before the per-frame speed's early return") + vsync = block(vi, "static DWORD WINAPI vsync_main(void *p) {") + check("1000000000LL / fps_max()" in vsync and "pace_fps" not in vsync, "the vsync event no longer goes by the cap itself") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`HOS_MOVIE_PACE=0`" in dev, "docs/development.md: HOS_MOVIE_PACE=0 is not in the switches table") + + movie = vi[vi.index("static volatile LONG64 movie_seen_ns;"):vi.index("static int pace_handed(void) {")] + host = "\n".join([block(vi, "static int fps_cap_of("), block(vi, "static int fps_max(void) {"), movie, "static int movie_raise;", + block(vi, "static int pace_fps(void) {"), decide, block(vi, "static float text_scale("), + block(vi, "static int realtime_rate("), step]) + with tempfile.TemporaryDirectory(prefix="movie-pace-") as tmp: + src, exe = Path(tmp) / "rules.c", Path(tmp) / "rules.exe" + src.write_text(RULES, encoding="utf-8") + subprocess.run([args.cc, "-std=gnu11", "-O1", "-Wall", "-Wextra", "-Werror", "-I", str(HOST), str(src), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + check(r.returncode == 0, "the movie pace's rules:\n" + r.stdout) + src, exe = Path(tmp) / "fixture.c", Path(tmp) / "fixture.exe" + src.write_text(FIXTURE.replace("@HOST@", host), encoding="utf-8") + subprocess.run([args.cc, "-std=gnu11", "-O1", "-Wall", "-Wextra", "-Werror", "-Wno-unused-function", "-I", str(HOST), str(src), + "-o", str(exe), "-lm"], check=True) + for mode, cap, raised, env in CASES: + e = {k: v for k, v in os.environ.items() if not k.startswith(("HOS_", "SI_"))} + e.update(env) + r = subprocess.run([str(exe), mode, str(cap), str(raised)], capture_output=True, text=True, env=e) + check(r.returncode == 0 and "PASS: %s %d %d" % (mode, cap, raised) in r.stdout, + "%s at a cap of %d with %s:\n%s%s" % (mode, cap, env or "nothing set", r.stdout, r.stderr)) + print("PASS: a movie at the 60 cap is paced at 240 with the limiter and the time step following, and the cap is back after it") + + +if __name__ == "__main__": + main() diff --git a/tests/test_msync_pass.py b/tests/test_msync_pass.py new file mode 100644 index 0000000..25fa75f --- /dev/null +++ b/tests/test_msync_pass.py @@ -0,0 +1,275 @@ +"""A mirror sync's pages from the captures go by one compute dispatch, in order with the copies (gpu/vk/msync_pass.h, +vk_memory.cpp; no game, no GPU). + +(Triforce, 11-21, the performance audit's MEM-48, after Source A's.) A sync copied the pages that come from a capture's +buffer into the pool's VRAM mirror as copy regions, a page or a few each, 700 to 1,300 a frame. The write-back diff's +pass is a copy when its shadow is its destination, so a sync's regions go as that pass's chunks in one dispatch. + +This compiles msync_pass.h with wbd_split cut out of vk_memory.cpp and checks: the switch (NVIDIA when not set); which +runs go as chunks; that the chunks of runs, applied as the shader applies them, leave the destination as plain copies +would, byte for byte and nothing else touched, for addresses alike within 16 bytes and not; a vector chunk's addresses +on 16 bytes and no chunk past 64 KB; the table ring (a table from an atom of its own at atoms 64, 128 and 256, a block +not taken again while the GPU still reads it, a table too big refused); the barrier's rule. + +And reads the source: the capture buffers are made with the pass's usage; mirror_sync adds, flushes ahead of the live +runs and notes its copies; the flush writes and flushes the table before its dispatch, marks a checkpoint and notes the +pass; every copy into a mirror in the uploads is noted for the barrier; the checkpoint kind and the switches row. + + python -I -B tests/test_msync_pass.py [--cxx clang++] +""" +import argparse +import re +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, head): + at = text.index(head) + return text[at:text.index("\n}\n", at) + 3] + + +def in_order(text, parts, why): + at = 0 + for p in parts: # (each after the one before it: a part may be there more than once) + at = text.find(p, at) + check(at >= 0, why + " (missing, or out of order: " + p + ")") + at += len(p) + + +PROGRAM = r""" +#include +#include +#include +#include +#include +#include "msync_pass.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +// (vk_memory.cpp's own, cut out by the test) +@WBD_SPLIT@ +struct Chunk { uint64_t src, dst; uint32_t count; bool vecs; }; +static uint32_t rnd_state = 12345; +static uint32_t rnd() { return rnd_state = rnd_state * 1664525u + 1013904223u; } + +// Runs (source offset, destination offset, size) between a source at src_base and a destination at dst_base, as chunks and +// as copies: the same bytes +static void copies_alike(uint64_t src_base, uint64_t dst_base, const std::vector& runs) { + const size_t n = 1u << 20; + std::vector src(n), want(n), got(n); + for (size_t i = 0; i < n; i++) src[i] = (uint8_t)rnd(), want[i] = got[i] = (uint8_t)rnd(); + std::vector chunks; + for (size_t i = 0; i + 2 < runs.size() + 0 && i < runs.size(); i += 3) { + const uint64_t so = runs[i], d0 = runs[i + 1], size = runs[i + 2]; + CHECK(msync_pass::words(src_base + so, dst_base + d0, size)); + std::memcpy(&want[d0], &src[so], size); + uint64_t bytes = 0; + msync_pass::run_chunks(src_base + so, dst_base + d0, size, + [](uint64_t at, uint64_t sz, auto&& e) { wbd_split(at, sz, e); }, + [&](uint64_t s, uint64_t d, uint32_t count, bool vecs) { + chunks.push_back({s, d, count, vecs}); + bytes += (uint64_t)count * (vecs ? 16 : 4); + }); + CHECK(bytes == size); + } + for (const Chunk& c : chunks) { + const uint64_t bytes = (uint64_t)c.count * (c.vecs ? 16 : 4); + CHECK(c.count != 0 && bytes <= msync_pass::kChunkBytes); + if (c.vecs) CHECK(!(c.src & 15) && !(c.dst & 15)); + else CHECK(!(c.src & 3) && !(c.dst & 3)); + CHECK(c.src >= src_base && c.src + bytes <= src_base + n && c.dst >= dst_base && c.dst + bytes <= dst_base + n); + // (the shader: a vector or a word written where it differs, the source's value: a copy) + const uint32_t step = c.vecs ? 16 : 4; + for (uint32_t i = 0; i < c.count; i++) { + const uint8_t *s = &src[c.src - src_base + (uint64_t)i * step]; + uint8_t* d = &got[c.dst - dst_base + (uint64_t)i * step]; + if (std::memcmp(s, d, step)) std::memcpy(d, s, step); + } + } + CHECK(got == want); +} + +int main() { + using namespace msync_pass; + CHECK(on(nullptr, true) && !on(nullptr, false) && on("", true) && !on("", false)); + CHECK(on("1", false) && on("1", true) && !on("0", true) && !on("0", false)); + CHECK(words(0x1000, 0x2000, 4096) && words(0x1004, 0x2008, 4)); + CHECK(!words(0x1000, 0x2000, 0) && !words(0x1001, 0x2000, 4096) && !words(0x1000, 0x2002, 4096) && !words(0x1000, 0x2000, 4094)); + + // whole pages, both bases on 16 bytes; pages of a mirror whose base is 8 off the source's; odd word runs + copies_alike(0x10000000, 0x20000000, {0, 0x3000, 4096, 0x1000, 0x8000, 3 * 4096, 0x40000, 0x80000, 5 * 65536 + 4096}); + copies_alike(0x10000000, 0x20000008, {0, 0x3000, 4096, 0x1000, 0x8000, 3 * 4096, 0x40000, 0x80000, 2 * 65536 + 4096}); + copies_alike(0x10000004, 0x20000004, {4, 0x3004, 4096, 0x100c, 0x800c, 40, 0x2000, 0x9000, 4, 0x4008, 0xa008, 65536 + 24}); + copies_alike(0x10000000, 0x20000000, {0x10, 0x3004, 4096 + 8, 0x40004, 0x80008, 3 * 65536}); + for (int k = 0; k < 40; k++) { // (many small runs side by side, as a sync's dirty pages) + std::vector runs; + uint64_t so = 0, d0 = 0; + for (int i = 0; i < 24 && d0 < (1u << 19); i++) { + const uint64_t size = (1 + rnd() % 8) * 4096; + d0 += (rnd() % 4) * 4096; + runs.insert(runs.end(), {so, d0, size}); + so += size, d0 += size; + } + copies_alike(0x10000000 + (rnd() % 4) * 4, 0x20000000 + (rnd() % 4) * 4, runs); + } + + // the tables' ring + for (const uint64_t atom : {64u, 128u, 256u}) { + Ring r; + uint64_t at = 99; + CHECK(r.get(0, atom, 5, 4, &at) < 0 && r.get(kBlockBytes + 32, atom, 5, 4, &at) < 0 && r.get(32, 0, 5, 4, &at) < 0); + int i = r.get(96, atom, 5, 4, &at); // (the first block, not made yet: the caller's to make) + CHECK(i == 0 && at == 0 && !r.b[0].made && r.b[0].value == 5 && r.b[0].fill == 96); + r.b[0].made = true; + i = r.get(32, atom, 5, 4, &at); // (the next table: from an atom of its own) + CHECK(i == 0 && at == (96 + atom - 1) / atom * atom && at % atom == 0 && at >= 96); + const uint64_t second = at; + i = r.get(32, atom, 7, 4, &at); // (a later recording reads the block too: its value follows) + CHECK(i == 0 && at % atom == 0 && at >= second + 32 && r.b[0].value == 7); + i = r.get(kBlockBytes, atom, 8, 4, &at); // (no room left in it: the next block) + CHECK(i == 1 && at == 0 && r.b[1].value == 8); + r.b[1].made = true; + for (int k = 2; k < kBlocks; k++) { + i = r.get(kBlockBytes, atom, 8 + k, 4, &at); + CHECK(i == k && at == 0); + r.b[k].made = true; + } + CHECK(r.get(kBlockBytes, atom, 30, 6, &at) < 0); // (block 0 is read by recording 7, the GPU at 6: none) + CHECK(r.cur == kBlocks - 1); // (and nothing moved) + i = r.get(kBlockBytes, atom, 30, 7, &at); // (done with 7: block 0 again) + CHECK(i == 0 && at == 0 && r.b[0].value == 30 && r.b[0].fill == kBlockBytes); + CHECK(r.get(32, atom, 31, 7, &at) < 0); // (block 1, read by recording 8, is next) + } + { // a block the caller could not make is asked for again + Ring r; + uint64_t at = 0; + CHECK(r.get(64, 64, 1, 0, &at) == 0 && !r.b[0].made); + CHECK(r.get(64, 64, 1, 0, &at) == 0 && !r.b[0].made && at == 64); + } + + // the barrier + CHECK(!barrier_first(0, kCopies) && !barrier_first(0, kPass)); // (nothing wrote it before) + CHECK(!barrier_first(kCopies, kCopies)); // (copies after copies: in order, as before) + CHECK(barrier_first(kCopies, kPass) && barrier_first(kPass, kPass) && barrier_first(kCopies | kPass, kPass)); + CHECK(barrier_first(kPass, kCopies) && barrier_first(kCopies | kPass, kCopies)); + return fails ? 1 : 0; +} +""" + + +def copies_into_mirrors(mem): + """(line number, line, the six lines before) of every vkCmdCopyBuffer whose destination, its third argument, is a mirror""" + lines = mem.splitlines() + found = [] + for i, line in enumerate(lines): + at = line.find("vkCmdCopyBuffer(") + if at < 0 or line.lstrip().startswith("//"): + continue + depth, args, cur = 0, [], "" + for ch in line[at + len("vkCmdCopyBuffer("):]: + if ch == "(": + depth += 1 + elif ch == ")": + if depth == 0: + break + depth -= 1 + if ch == "," and depth == 0: + args.append(cur.strip()) + cur = "" + else: + cur += ch + if len(args) >= 3 and re.search(r"(\.|->)mirror$", args[2]): + found.append((i + 1, line, "\n".join(lines[max(0, i - 6):i]))) + return found + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + mem = (VK / "vk_memory.cpp").read_text(encoding="utf-8").replace("\r\n", "\n") + + check('#include "msync_pass.h"' in mem, "vk_memory.cpp does not use msync_pass.h") + check('msync_pass::on(getenv("RX_VK_MSYNC_PASS"), dev.props.vendorID == 0x10DE)' in mem, "the switch is not the rule's, on NVIDIA when not set") + usage = block(mem, "VkBufferUsageFlags cap_usage() {") + check("msync_pass_on() ? VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT : 0" in usage, + "a capture buffer is not made for the pass to read by its address") + # ((Triforce, 11-22) since MEM-39 every capture buffer is made by cap_buf_make, cached or (lab, RX_VK_CAP_UNCACHED=1) + # not: both with the pass's usage, and no capture buffer is made anywhere else) + make = block(mem, "bool cap_buf_make(uint64_t size, VkBuffer& buffer, VmaAllocation& alloc, void** map, bool own) {") + check("create_buffer_uncached(size, cap_usage()," in make and "create_buffer_cached(size, cap_usage()," in make and + mem.count("create_buffer_cached(size, cap_usage(),") == 1 and mem.count("create_buffer_uncached(") == 1 and + "create_buffer_cached(size, VK_BUFFER_USAGE_TRANSFER_SRC_BIT" not in mem, + "a capture buffer is made without the pass's usage: the pass would read a buffer that has no address") + + sync = block(mem, "void mirror_sync(Pool& p) {") + in_order(sync, ["UpWrites up(p);", 'const bool pass = msync_pass_on() && !lab_ab::on("nomsyncpass");', + "clip_resident(p, rs);", "if (pass && msync_pass_add(p, buf, rs)) {", "up.write(msync_pass::kCopies);", + "vkCmdCopyBuffer(upload_command_buffer(), buf, p.mirror, (uint32_t)rs.size(), rs.data());", + "msync_pass_flush(up);", "clip_resident(p, runs);", "up.write(msync_pass::kCopies);", + "vkCmdCopyBuffer(upload_command_buffer(), p.buffer, p.mirror, (uint32_t)runs.size(), runs.data());"], + "mirror_sync does not hand a capture's clipped runs to the pass, flush it ahead of the live runs, or note its copies") + + add = block(mem, "bool msync_pass_add(Pool& p, VkBuffer buf, const std::vector& rs) {") + in_order(add, ["if (rs.empty() || !p.mirror || !buf || !wbd_pipe_ready()) return false;", "if (!src || !dst) return false;", + "if (!msync_pass::words(src + r.srcOffset, dst + r.dstOffset, r.size)) return false;", + "msync_pass::run_chunks(src + r.srcOffset, dst + r.dstOffset, r.size,", "wbd_split(at, size, e);", + "msync_chunks.push_back({s, d, d, count, vecs ? 1u : 0u});", "msync_copies.push_back({buf, p.mirror, rs});"], + "msync_pass_add does not refuse a run that is not whole words before it takes any, or its chunks' shadow is not their destination") + flush = block(mem, "void msync_pass_flush(UpWrites& up) {") + in_order(flush, ["MsyncBlock* b = msync_block_get(bytes, at);", "if (!b) {", "up.write(msync_pass::kCopies);", + "vkCmdCopyBuffer(cmd, c.src, c.dst, (uint32_t)c.rs.size(), c.rs.data());", "st_msync_fallbacks++;", + "memcpy(b->map + at, msync_chunks.data(), bytes);", "flush_mapped_range(b->alloc, at, bytes);", + "up.write(msync_pass::kPass);", "checkpoint_mark(cmd, n, 's');", + "vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, wbd_pipe.pipe);", + "pc{b->address + at, n, 0};", "vkCmdDispatch(cmd, std::min(n, 65535u), (n + 65534u) / 65535u, 1);", + "msync_chunks.clear();", "msync_copies.clear();"], + "msync_pass_flush: a sync with no table room is not copied, or the table is not written and flushed, the barrier's " + "note made and the checkpoint marked before the dispatch") + get = block(mem, "MsyncBlock* msync_block_get(uint64_t bytes, uint64_t& at) {") + in_order(get, ["std::max(sizeof(WbdChunk), dev.props.limits.nonCoherentAtomSize);", + "msync_ring.get(bytes, align, mem_rec_value, mem_completed, &at);", "if (!msync_ring.b[i].made) {", + "msync_ring.b[i].made = true;"], "msync_block_get does not take its tables from the ring by the atom and the GPU's progress") + + ups = block(mem, "struct UpWrites {") + in_order(ups, ["if (!msync_pass_on()) return;", "cmd = upload_command_buffer();", + "prior = p.up_cmd == cmd && p.up_value == mem_rec_value ? p.up_kinds : 0;", + "if (msync_pass::barrier_first(prior, kind)) {", "full_barrier(cmd,", "prior = 0", "mine |= kind;", + "p.up_cmd = cmd, p.up_value = mem_rec_value, p.up_kinds = (uint8_t)(prior | mine);"], + "UpWrites does not put a full barrier before a write the rule asks one for, or does not keep what wrote the mirror") + # (every copy into a mirror in the uploads: noted, so that a pass before or after it in that command buffer is ordered) + writes = copies_into_mirrors(mem) + check(len(writes) >= 8, "the copies into mirrors were not found (%d): the scan no longer reads the source" % len(writes)) + for n, line, before in writes: + noted = ("mirror_up_copy(" in line or "mirror_up_copy(" in before or "up.write(msync_pass::kCopies);" in before or + "up_kinds = msync_pass::kCopies;" in before) + check(noted, "vk_memory.cpp:%d copies into a mirror without a note for the syncs' pass: a dispatch before it in the same " + "upload command buffer is not ordered with it\n %s" % (n, line.strip())) + check("msync_pass_report();" in block(mem, "void memory_end_submission(VkCommandBuffer cmd) {"), "the pass's 600-submit report is not made") + + kinds = (VK / "checkpoint_kinds.h").read_text(encoding="utf-8") + check(kinds.count("case 's':") == 2, "checkpoint_kinds.h does not name the pass's marker and what follows it") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`RX_VK_MSYNC_PASS=0`" in dev, "docs/development.md: RX_VK_MSYNC_PASS is not in the switches table") + + with tempfile.TemporaryDirectory(prefix="msync-pass-") as tmp: + c, exe = Path(tmp) / "pass.cpp", Path(tmp) / "pass.exe" + c.write_text(PROGRAM.replace("@WBD_SPLIT@", block(mem, "template void wbd_split(uint64_t at, uint64_t size, F&& emit) {")), + encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(VK), str(c), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + check(r.returncode == 0, "the rules as made:\n" + r.stdout) + print("PASS: a mirror sync's pages from the captures go by one dispatch, in order with the copies") + + +if __name__ == "__main__": + main() diff --git a/tests/test_nvn_host_tables.py b/tests/test_nvn_host_tables.py index 66d565d..3379582 100644 --- a/tests/test_nvn_host_tables.py +++ b/tests/test_nvn_host_tables.py @@ -82,6 +82,11 @@ static LONG64 InterlockedAdd64(volatile LONG64 *v, LONG64 d) { return *v += d; } static LONG64 InterlockedIncrement64(volatile LONG64 *v) { return ++*v; } static LONG64 InterlockedDecrement64(volatile LONG64 *v) { return --*v; } static void hos_log(const char *f, ...) { (void)f; } +/* ((Triforce, OS-04) the tables allocate through host/host_alloc.h's three, which wait at Windows' commit limit before they + answer NULL (tests/test_commit_limit.py): here the C library's own */ +#define hos_xmalloc malloc +#define hos_xcalloc calloc +#define hos_xrealloc realloc static jmp_buf died; static int dying; static void die(const char *fmt, ...) { (void)fmt; if (dying) longjmp(died, 1); fprintf(stderr, "unexpected die\n"); exit(3); } diff --git a/tests/test_obj_cache_size.py b/tests/test_obj_cache_size.py new file mode 100644 index 0000000..47703d6 --- /dev/null +++ b/tests/test_obj_cache_size.py @@ -0,0 +1,226 @@ +"""The per-thread cache of the nn::os object lookups: 16,384 entries, four ways a set (host/nn_os_sync.c obj_lookup_cached, +HOS_OBJ_CACHE). + +The game's record pass under the graphics front end locks and unlocks some 4,000 per-slot mutexes 0x80 apart; the +direct-mapped cache's multiply-shift index folded those keys onto few entries and missed at nearly every step. The code from +the ObjWay type to the end of obj_nockpt (the switch's parse, the four-way cache, the counts' flush, obj and obj_nockpt with +the direct-mapped cache) is cut out unchanged and compiled with stand-ins: an object table that answers every lookup as +obj_lookup does (an object a key, re-kinded in place, buried ones reused for other keys), a clock and a log. + +Checked: HOS_OBJ_CACHE's values (unset, empty or not a number 16,384; 0 the direct-mapped cache; else 256 to 65,536, down to +a power of two); a 4,000-mutex walk at a 0x80 stride misses under 100 times on its third pass by default, most of the 4,000 +with =0, all of them with =256, and both lookups of each step with HOS_NO_OBJ_CACHE=1; 200,000 lookups among buried, re-kinded and reused objects each +answer the table's object of that key and kind; the counts' log line every 10 s, from obj only; obj_nockpt neither logs nor +flushes (it runs under host locks: tests/test_multiwait_wake.py); the docs rows. + + python -I -B tests/test_obj_cache_size.py [--cxx clang++] +""" +import argparse +import os +import re +import subprocess +import tempfile +from pathlib import Path + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def fail(message): + raise SystemExit("FAIL: " + message) + + +HARNESS = r""" +#include +#include +#include +#include +#include + +typedef struct Obj { uint64_t key; int kind; struct Obj *spare; } Obj; +static int64_t fake_now = 1; +static int64_t hos_now_ns(void) { return fake_now; } +static char logged[512]; +static int nlogged; +static void hos_log(const char *fmt, ...) { + va_list ap; + va_start(ap, fmt); + vsnprintf(logged, sizeof logged, fmt, ap); + va_end(ap); + nlogged++; +} +static void hos_vcore_checkpoint(void) {} + +/* (the stand-in table: keys from a fixed pool, an object a key, as obj_lookup answers) */ +#define POOL 8192 +static uint64_t pool[POOL]; +static int npool; +static Obj *live[POOL]; +static Obj *spares; +static long slow; +static int pool_index(uint64_t key) { + for (int i = 0; i < npool; i++) if (pool[i] == key) return i; + if (npool == POOL) { fprintf(stderr, "pool full\n"); exit(2); } + pool[npool] = key; + return npool++; +} +static int last_index = -1; +static Obj *obj_lookup(uint64_t key, int kind) { + slow++; + const int i = last_index >= 0 && pool[last_index] == key ? last_index : pool_index(key); + Obj *o = live[i]; + if (o && o->kind == kind) return o; + if (o) { /* (the address initialized again as another kind) */ + __atomic_store_n(&o->kind, kind, __ATOMIC_RELEASE); + return o; + } + if ((o = spares)) spares = o->spare; + else o = (Obj *)calloc(1, sizeof *o); + __atomic_store_n(&o->key, key, __ATOMIC_RELAXED); + __atomic_store_n(&o->kind, kind, __ATOMIC_RELEASE); + live[i] = o; + return o; +} +static void bury(int i) { /* (the object goes to the spares, to be reused for another address) */ + Obj *o = live[i]; + if (!o) return; + __atomic_store_n(&o->kind, 0, __ATOMIC_RELEASE); + live[i] = NULL; + o->spare = spares; + spares = o; +} + +CUT + +static uint64_t rng = 0x9E3779B97F4A7C15ULL; +static uint64_t next_rand(void) { rng ^= rng << 13; rng ^= rng >> 7; rng ^= rng << 17; return rng; } + +int main(int argc, char **argv) { + const char *mode = argc > 1 ? argv[1] : ""; + if (!strcmp(mode, "entries")) { + printf("%d\n", obj_cache_entries()); + return 0; + } + if (!strcmp(mode, "walk")) { + long misses = 0; + for (int pass = 0; pass < 3; pass++) { + const long before = slow; + for (int i = 0; i < 4000; i++) { + const uint64_t key = 0x8a3f1200ULL + (uint64_t)i * 0x80; + Obj *o = obj(key, 1); + if (!o || o->key != key || o->kind != 1) { printf("wrong object\n"); return 1; } + Obj *again = obj(key, 1); /* (lock, then unlock: the second is a hit) */ + if (again != o) { printf("wrong object\n"); return 1; } + } + misses = slow - before; + } + printf("%ld\n", misses); + return 0; + } + if (!strcmp(mode, "random")) { + for (int i = 0; i < 6000; i++) + pool_index(0x80000000ULL + (next_rand() % 0x1000000) * 16 + (i % 3) * 0x80000000ULL); + long wrong = 0; + for (long n = 0; n < 200000; n++) { + const uint64_t r = next_rand(); + const int i = (int)(r % 1500 + (r >> 20) % 7 * ((r >> 40) % 2 ? 600 : 0)) % npool; + if ((r >> 32) % 97 == 0) { bury(i); continue; } + const int kind = (r >> 48) % 13 == 0 ? 2 + (int)((r >> 52) % 3) : 1; + last_index = i; + Obj *o = obj(pool[i], kind); + last_index = -1; + if (o != live[i] || o->key != pool[i] || o->kind != kind) wrong++; + } + printf("%ld wrong of 200000, %ld to the table\n", wrong, slow); + return wrong != 0; + } + if (!strcmp(mode, "log")) { + for (int round = 0; round < 3; round++) { + for (int n = 0; n < 70000; n++) obj(0x81000000ULL, 1); + obj(0x82000000ULL + (uint64_t)round * 0x1000, 1); /* (a miss past 65,536 lookups: due) */ + obj(0x81000000ULL, 1); /* (the flush, in obj) */ + if (round == 0 && nlogged) { printf("logged at the first flush\n"); return 1; } + if (round == 0) fake_now += 11000000000LL; /* (the third round within 10 s of the second: no line) */ + } + printf("%d|%s\n", nlogged, logged); + return 0; + } + return 1; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + source = (ROOT / "host" / "nn_os_sync.c").read_text(encoding="utf-8") + start = "typedef struct { uint64_t key; Obj *o; } ObjWay;" + if start not in source: + fail("host/nn_os_sync.c has no four-way object cache (ObjWay)") + nockpt = block(source, "static Obj *obj_nockpt(uint64_t key, int kind) {") + cut = source[source.index(start):source.index(nockpt) + len(nockpt)] + # (the source's own rules) + if "hos_log" in nockpt or "obj_cache_flush" in nockpt or "hos_log" in block(source, "static Obj *obj_lookup_cached("): + fail("obj_nockpt or the cache logs or flushes: it runs under host locks") + obj = block(source, "static Obj *obj(uint64_t key, int kind) {") + if "if (obj_cache_due) obj_cache_flush();" not in obj or obj.index("hos_vcore_checkpoint();") > obj.index("obj_cache_flush"): + fail("obj: the counts are not flushed there, after the checkpoint") + if 'getenv("HOS_NO_OBJ_CACHE")' not in nockpt or "if (entries) return obj_lookup_cached(key, kind, entries);" not in nockpt: + fail("obj_nockpt: HOS_NO_OBJ_CACHE before both caches, the four-way one unless HOS_OBJ_CACHE=0") + docs = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + if "`HOS_OBJ_CACHE=`" not in docs or "`HOS_NO_OBJ_CACHE=1`" not in docs: + fail("docs/development.md: rows for HOS_OBJ_CACHE and HOS_NO_OBJ_CACHE") + with tempfile.TemporaryDirectory(prefix="obj-cache-") as tmp: + c, exe = Path(tmp) / "obj_cache.c", Path(tmp) / "obj_cache.exe" + c.write_text(HARNESS.replace("CUT", cut), encoding="utf-8") + subprocess.run([args.cxx, "-x", "c", "-std=gnu11", "-O1", "-Wall", "-Wextra", "-Werror", "-Wno-unused-function", + str(c), "-o", str(exe)], check=True) + + def run(mode, **env): + e = {k: v for k, v in os.environ.items() if k not in ("HOS_OBJ_CACHE", "HOS_NO_OBJ_CACHE")} + e.update(env) + r = subprocess.run([str(exe), mode], capture_output=True, text=True, env=e, timeout=120) + if r.returncode: + fail("%s %s: %s%s" % (mode, env, r.stdout, r.stderr)) + return r.stdout.strip() + for value, want in ((None, 16384), ("", 16384), ("abc", 16384), ("-5", 16384), ("12k", 16384), ("0", 0), ("256", 256), + ("300", 256), ("100", 256), ("16384", 16384), ("20000", 16384), ("65536", 65536), + ("99999999", 65536)): + got = int(run("entries", **({} if value is None else {"HOS_OBJ_CACHE": value}))) + if got != want: + fail("HOS_OBJ_CACHE=%r: %d entries, not %d" % (value, got, want)) + print("ok HOS_OBJ_CACHE: unset 16,384; 0 the direct-mapped cache; 256 to 65,536, a power of two") + walk = int(run("walk")) + fixed, small, off = int(run("walk", HOS_OBJ_CACHE="0")), int(run("walk", HOS_OBJ_CACHE="256")), int(run("walk", HOS_NO_OBJ_CACHE="1")) + print(" a 4,000-mutex walk at 0x80, misses on its third pass: %d by default, %d with =0, %d with =256, %d with " + "HOS_NO_OBJ_CACHE=1" % (walk, fixed, small, off)) + if walk >= 100: + fail("the walk missed %d times on its third pass (under 100 wanted)" % walk) + if fixed < 3000 or small != 4000 or off != 8000: + fail("the walk's controls: =0 %d (over 3,000 wanted), =256 %d (4,000 wanted), HOS_NO_OBJ_CACHE=1 %d (8,000: both lookups a step)" % (fixed, small, off)) + print("ok the walk: under 100 misses a pass; its controls miss as before") + for env in ({}, {"HOS_OBJ_CACHE": "256"}, {"HOS_OBJ_CACHE": "0"}): + print("ok random lookups %s: %s" % (env or "(default)", run("random", **env))) + n, line = run("log").split("|", 1) + m = re.fullmatch(r"sync: object cache \(16384 entries, 4 ways a set; HOS_OBJ_CACHE\): (\d+) lookups, (\d+) missed " + r"\((\d+\.\d\d) %\) in (\d+) s", line) + if int(n) != 1 or not m or int(m.group(1)) < 140000 or int(m.group(2)) > 10 or m.group(4) != "11": + fail("the log line: %s times, the last %r" % (n, line)) + print("ok the log line every 10 s, from obj: " + line) + print("PASS: the four-way object cache") + + +if __name__ == "__main__": + main() diff --git a/tests/test_occ_answer.py b/tests/test_occ_answer.py index 1455fc8..402ee43 100644 --- a/tests/test_occ_answer.py +++ b/tests/test_occ_answer.py @@ -130,7 +130,7 @@ def check_source(src): block(replay, "void occ_slot_begin(Recording& r) {"), ["if (!occ_host_reset) vkCmdResetQueryPool(r.cmd, occ_pool, (uint32_t)r.slot * kOccPerSlot, kOccPerSlot), sl.used = 0;", "sl.serial = occ_serial_next.fetch_add(1, std::memory_order_relaxed) + 1;", "sl.armed = true;"]) - submit = block(replay, "void submit_recording(Recording& r, bool signal_value = true, uint64_t signal_at = 0) {") + submit = block(replay, "void submit_recording(Recording& r, bool signal_value = true, uint64_t signal_at = 0, bool part = false) {") in_order("vk_replay.cpp submit_recording (sealed after dev.present_mu is let go: the seal is declared before the lock)", submit, ["seq = ++rec_seq;", "struct OccSeal {", "if (occ_live && rec.submitted) occ_seal(rec, at);", "} occ_sealed{r, seq};", "std::lock_guard lk(dev.present_mu); // (one queue"]) @@ -167,9 +167,9 @@ def check_source(src): "occ_writers.word(addr, now), sy = occ_writers.word(addr + 16, now);", "if (sx == occ::Writers::kNone && sy == occ::Writers::kNone) return;", 'const bool active = occ_env_real || lab_ab::on("occq");', - "if (active && (sx == occ::Writers::kWrittenSrc || sy == occ::Writers::kWrittenSrc)) {", + "if (active && (sx == occ::Writers::kWrittenSrc || sy == occ::Writers::kWrittenSrc || gated)) {", "host->gpu_read(addr, &x, 8);", "*enabled = mode == 0 ? x == y : x != y;", - "occ::cond_decide(*enabled, sx, sy, active, occ_cond_on);", "occ_st.cond(c.kind);", + "occ::cond_decide(*enabled, sx, sy, active, occ_cond_on, occ_gate_on);", "occ_st.cond(c.kind);", "*enabled = c.enabled;", "if (c.kind == occ::Cond::kWrittenOff) occ_cond_off = &s;", "if (occ_log_cond && seen++ % 499 == 0 && logged < 300) {"]) in_order("vk_replay.cpp occ_rewritten (another counter type's report, or a small clear: the records leave Writers)", @@ -204,10 +204,10 @@ def check_source(src): cond = replay_fn[replay_fn.index("case NOP_RENDER_COND: {"):] in_order("vk_replay.cpp NOP_RENDER_COND (not the worker's: the replay's and the dry pass's decide, the segments take it)", cond[:2400], - ["if (worker) {", "par_conds[cond_at] != 0", "break;", "s.render_enabled = r.mode == 0 ? x == y : x != y;", + ["if (worker) {", "par_conds[cond_at] : 1;", "s.render_enabled = v == 1;", "occ_seg_off = v == 2 ? &s : nullptr;", "break;", "s.render_enabled = r.mode == 0 ? x == y : x != y;", "if (occ_live) {", "if (r.snapped) occ_cond_off = nullptr;", "else occ_cond(s, r.addr, (int)r.mode, x, y, &s.render_enabled);", "if (always) s.render_enabled = true;", - "if (dry) par_conds.push_back(s.render_enabled ? 1 : 0);"]) + "if (dry) {", "const uint8_t v = s.render_enabled ? 1 : occ_live && occ_cond_off == &s ? 2 : 0;", "par_conds.push_back(v);"]) enable = replay_fn[replay_fn.index("case NOP_RENDER_ENABLE:"):replay_fn.index("case NOP_RENDER_COND: {")] need("vk_replay.cpp NOP_RENDER_ENABLE (a condition's skipping ends)", enable, ["if (occ_live && !worker && occ_cond_off == &s) occ_cond_off = nullptr;"]) @@ -223,7 +223,8 @@ def check_source(src): ["if (occ_live && !worker) occ_skip_note(s, true);", "if (s.render_enabled && !dry) {"]) in_order("vk_replay.cpp replay() (a skipped draw or dispatch counted, not a worker's)", replay_fn, ["h.op == NOP_DISPATCH || h.op == NOP_DISPATCH_INDIRECT) {", "if (occ_live && !worker) occ_skip_note(s, false);", - "if (dry) {", "if (++dry_work % 64 == 0)"]) + "if (occ_live && !dry && h.op != NOP_DISPATCH && h.op != NOP_DISPATCH_INDIRECT) occ_gate_note(s, worker);", + "if (dry) {\n const bool compute", "if (++dry_work % par_point_every() == 0)"]) handle = block(replay, "const uint8_t* replay_handle(VkCommandBuffer& cmd, const uint8_t* h, bool may_stop = false) {") in_order("vk_replay.cpp replay_handle (the replay's counting given up before a dry pass; its reports placed at cb's)", handle, ["const uint64_t cb = ++stream_cbs;", "if (occ_live) occ_serial_give_up(cmd), occ_dry_cb = cb;", "rmode = Mode::Dry;"]) @@ -261,6 +262,9 @@ def check_source(src): other = max(replay.rfind("\nvoid ", 0, m.start()), replay.rfind("\nbool ", 0, m.start()), replay.rfind("\nuint64_t ", 0, m.start()), replay.rfind("\nint ", 0, m.start())) init_fn = replay.rfind("\nbool occ_init() {", 0, m.start()) + line = replay[replay.rfind("\n", 0, m.start()) + 1:m.start()] + if line.strip().startswith("if (occ_live && "): + continue # (behind the switch in the line itself: begin_segment_slot's reset command in the job) if max(fn, init_fn) < other: raise SystemExit("vk_replay.cpp: " + call + " outside the occ_ functions") @@ -270,16 +274,19 @@ def check_source(src): draw = (src / "gpu/vk/vk_draw.cpp").read_text(encoding="utf-8").replace("\r\n", "\n") in_order("vk_draw.cpp end_pass (the query ends in its pass)", block(draw, "void end_pass(VkCommandBuffer cmd) {"), - ["if (!pass().open) return;", "if (occ_live) occ_pass_end(cmd);", "vkCmdEndRendering(cmd);"]) + ["Pass& ps = pass();", "if (!ps.open) return;", "if (occ_live) occ_pass_end(cmd);", "vkCmdEndRendering(cmd);"]) d = block(draw, "void draw(VkCommandBuffer cmd, NvnState& s, const DrawArgs& d) {") + # ((Triforce, 11-22, the performance audit's GPUWORK-01) a draw that writes colour is told to the count right before a query + # may begin: counted, and with RX_VK_OCC_DEPTH_ONLY=1 the count given up (tests/test_occ_depth_only.py runs it) in_order("vk_draw.cpp draw", d, ["if (occ_live) occ_draw_enter();", "begin_pass(cmd, at);", - "if (occ_live && pass().open) occ_draw_begin(cmd);"]) - issued = d.count("st_draws++, pass().draws++;") - if issued != d.count("st_draws++, pass().draws++;\n") or issued != d.count("if (occ_live) occ_draw_issued();") or not issued: + "if (occ_live && ps.open && (writes & 0xFF)) occ_draw_colour(cmd);", + "if (occ_live && ps.open) occ_draw_begin(cmd);"]) + issued = d.count("st_draws++, ps.draws++;") + if issued != d.count("st_draws++, ps.draws++;\n") or issued != d.count("if (occ_live) occ_draw_issued();") or not issued: raise SystemExit("vk_draw.cpp draw: every draw it records (st_draws++) says so to the count (occ_draw_issued)") dev = (src / "docs/development.md").read_text(encoding="utf-8") - for switch in ("RX_VK_OCC_QUERY=1", "RX_VK_OCC_QUERY_SKIP", "RX_VK_OCC_COND=0", "RX_VK_OCC_LOG_COND=1"): + for switch in ("RX_VK_OCC_QUERY=0", "RX_VK_OCC_QUERY_SKIP", "RX_VK_OCC_COND=0", "RX_VK_OCC_LOG_COND=1"): if "`" + switch not in dev: raise SystemExit("docs/development.md: " + switch + " is not in the switches table") if "RX_VK_OCC_QUERY_FP" in dev: diff --git a/tests/test_occ_depth_only.py b/tests/test_occ_depth_only.py new file mode 100644 index 0000000..4f2b7c8 --- /dev/null +++ b/tests/test_occ_depth_only.py @@ -0,0 +1,203 @@ +"""Only depth-only occlusion windows counted, behind a switch that is off (gpu/vk/occ_answer.h Counter::colour_draw; +vk_replay.cpp occ_draw_colour, occ_depth_only; vk_draw.cpp draw; no game, no GPU). + +(Triforce, 11-22, the performance audit's GPUWORK-01, after Source A's.) The game's occlusion tests draw depth only. A +query here opens at a window's first draw in the pass and stays open to the report or the pass's end, so it can enclose +draws that write colour, and a query's end waits for the work it encloses. Source A counts a run only when its draws +write no colour. Here: every draw that writes a colour target is told to the count before a query may begin. That is +counted for the occlusion line whether or not the rule is on (the windows that met one, the colour draws recorded under +an open query). With RX_VK_OCC_DEPTH_ONLY=1, or in the lab's "occdepth" phases, the count is given up there: its open +query ends before the colour draw, no query begins again until the next reset, and its report is incomplete, so the +record keeps the placeholder and what it gates is drawn. Off by default: it culls fewer objects. + +Runs tests/occ_answer_test.cpp (the header's cases, the colour ones among them: tests/test_occ_answer.py runs it too, +after source checks of its own). + +Cuts occ_depth_only, occ_draw_colour, occ_draw_begin and occ_close out of vk_replay.cpp and records windows through them +on a stand-in command buffer: + - off: a window of a depth draw, a colour draw and a depth draw is one query over all three, measured, as before, and + the window and its colour draw under the query are counted; + - on (the switch, and the lab's knob): the query ends before the colour draw, none begins after it, the report is + incomplete for that reason; the next window, depth only, is measured again; + - a colour draw first in its window, on: no query at all for that window; + - 64 colour draws under one query, off: counted in one step of 64. + +Reads the source: draw tells a colour draw (its write mask's colour bits) to the count right before occ_draw_begin, only +in an open pass with the queries live; the switch is read once and is off unless set; the report's breakdown has the +reason; the line is logged; vk_draw.h declares the hook; the switches table has the row. + + python -I -B tests/test_occ_depth_only.py [--cxx clang++] +""" +import argparse +import os +import subprocess +import tempfile +from pathlib import Path + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] + +FIXTURE = r""" +#include "occ_answer.h" +#include +#include +#include +#include +#include +#include +#define CHECK(x) do { if (!(x)) { std::printf("line %d: %s (events: %s)\n", __LINE__, #x, events.c_str()); return 1; } } while (0) + +namespace occ = mkwvk::occ; +using VkCommandBuffer = int; +using VkQueryPool = int; +using VkQueryControlFlags = uint32_t; +namespace lab_ab { +bool knob = false; +bool on(const char* name) { return knob && !strcmp(name, "occdepth"); } +} +std::string events; // (what was recorded: B a query begun, E ended, D a depth draw, C a colour draw) +void vkCmdBeginQuery(VkCommandBuffer, VkQueryPool, uint32_t i, VkQueryControlFlags) { events += "B" + std::to_string(i) + " "; } +void vkCmdEndQuery(VkCommandBuffer, VkQueryPool, uint32_t i) { events += "E" + std::to_string(i) + " "; } +constexpr uint32_t kOccPerSlot = 1024; +struct OccSlot { uint32_t used = 0; } occ_slots[2]; +VkQueryPool occ_pool = 1; +VkQueryControlFlags occ_flags = 0; +std::atomic occ_keep{false}; +thread_local bool occ_open_here = false; +static occ::Counter counter; +occ::Counter& occ_ctr() { return counter; } +static const uint64_t kSerial = 7; +uint64_t occ_serial_of(VkCommandBuffer, int* slot) { *slot = 0; return kSerial; } +/* CUT */ + +static const VkCommandBuffer cmd = 5; +static void draw(bool colour) { // (vk_draw.cpp draw's order) + counter.draw_enter(occ_keep); + if (colour) occ_draw_colour(cmd); + occ_draw_begin(cmd); + events += colour ? "C " : "D "; + counter.draw_issued(occ_keep); +} +static occ::Counter::Outcome report(occ::Rep* r) { // (vk_replay.cpp's report: the open query ends, then the count) + bool keep_now = false; + occ_close(counter, cmd); + return counter.report(occ_keep, kSerial, occ_slots[0].used, r, &keep_now); +} + +int main(int argc, char** argv) { + const std::string mode = argc > 1 ? argv[1] : ""; + const bool rule = mode != "off"; + if (mode == "knob") lab_ab::knob = true; + CHECK(occ_depth_only() == rule); + occ::Rep r; + // a window with a colour draw between two depth draws + counter.reset(0, kSerial, occ_slots[0].used); + draw(false), draw(true), draw(false); + const occ::Counter::Outcome first = report(&r); + if (!rule) { + CHECK(events == "B0 D C D E0 " && first == occ::Counter::kMeasured && r.first == 0 && r.n == 1); + CHECK(occ_colour_windows == 1 && occ_colour_lost == 0 && counter.why == occ::Counter::kNone); + // 64 colour draws under one query: counted in one step + events.clear(); + counter.reset(0, kSerial, occ_slots[0].used); + draw(false); + for (int i = 0; i < 62; i++) draw(true); + CHECK(occ_colour_open == 0); + draw(true); // (the 64th of this thread, the first window's one among them) + CHECK(occ_colour_open == 64 && occ_colour_windows == 2 && report(&r) == occ::Counter::kMeasured && r.n == 1); + std::printf("ok off\n"); + return 0; + } + CHECK(events == "B0 D E0 C D " && first == occ::Counter::kIncomplete && counter.why == occ::Counter::kColour); + CHECK(occ_colour_windows == 1 && occ_colour_lost == 1 && !occ_open_here && occ_slots[0].used == 1); + // the next window, depth only: measured, with a query of its own + events.clear(); + counter.reset(0, kSerial, occ_slots[0].used); + draw(false), draw(false); + CHECK(report(&r) == occ::Counter::kMeasured && events == "B1 D D E1 " && r.first == 1 && r.n == 1 && occ_colour_lost == 1); + // a colour draw first in its window: no query at all + events.clear(); + counter.reset(0, kSerial, occ_slots[0].used); + draw(true), draw(false), draw(true); + CHECK(report(&r) == occ::Counter::kIncomplete && events == "C D C " && counter.why == occ::Counter::kColour && occ_colour_lost == 2); + CHECK(occ_slots[0].used == 2 && occ_colour_windows == 2); + std::printf("ok %s\n", mode.c_str()); + return 0; +} +""" + + +def block(text, marker): + start = text.index(marker) + at = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[at] == "{") - (text[at] == "}") + at += 1 + return text[start:at] + + +def need(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def ordered(text, parts, why): + last = 0 + for p in parts: + k = text.find(p, last) + need(k >= 0, why + " (missing, or out of order: " + p + ")") + last = k + len(p) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + replay = (ROOT / "gpu/vk/vk_replay.cpp").read_text(encoding="utf-8") + draw = (ROOT / "gpu/vk/vk_draw.cpp").read_text(encoding="utf-8") + only = block(replay, "bool occ_depth_only() {") + need('static const bool env = getenv("RX_VK_OCC_DEPTH_ONLY") && atoi(getenv("RX_VK_OCC_DEPTH_ONLY")) != 0;' in only and + 'return env || lab_ab::on("occdepth");' in only, "the switch is not RX_VK_OCC_DEPTH_ONLY, off unless set, with the lab's occdepth knob") + colour = block(replay, "void occ_draw_colour(VkCommandBuffer cmd) {") + ordered(colour, ["c.colour_draw(occ_keep.load(std::memory_order_relaxed), occ_depth_only());", "if (!m.lose) return;", + "occ_close(c, cmd);", "c.lose_colour();"], "occ_draw_colour does not end the open query and give the count up only under the rule") + ordered(replay, ["void occ_close(occ::Counter& c, VkCommandBuffer cmd) {", "c.why == occ::Counter::kColour ? occ_st.inc_colour", + "bool occ_depth_only() {", "void occ_present_tick() {", "occ_st.colour_line(line, sizeof line, 600, occ_depth_only());", + "void occ_draw_colour(VkCommandBuffer cmd) {", "void occ_draw_begin(VkCommandBuffer cmd) {"], + "vk_replay.cpp: the report's breakdown, the line or the hook is missing") + body = block(draw, "void draw(VkCommandBuffer cmd, NvnState& s, const DrawArgs& d) {") + ordered(body, ["uint16_t writes;", "writes = draw_hazards(cmd, p, s, at);", "begin_pass(cmd, at);", + "if (occ_live && ps.open && (writes & 0xFF)) occ_draw_colour(cmd);\n if (occ_live && ps.open) occ_draw_begin(cmd);"], + "draw does not tell a colour draw to the count right before a query may begin") + need(body.count("occ_draw_colour(") == 1, "draw tells a colour draw to the count in more than one place") + hazards = block(draw, "uint16_t draw_hazards(") + need("if (at.color[i] && target_write_mask(p, s, i)) writes |= (uint16_t)(1u << i);" in hazards and "writes |= 1u << 8;" in hazards, + "the write mask's colour bits are no longer bits 0 to 7 with the depth's at 8") + need("void occ_draw_colour(VkCommandBuffer cmd);" in (ROOT / "gpu/vk/vk_draw.h").read_text(encoding="utf-8"), "vk_draw.h does not declare the hook") + docs = (ROOT / "docs/development.md").read_text(encoding="utf-8") + need("`RX_VK_OCC_DEPTH_ONLY=1`" in docs and "occdepth" in docs, "docs/development.md: the switches table has no RX_VK_OCC_DEPTH_ONLY row") + cut = replay[replay.index("bool occ_depth_only() {"):replay.index("/// Once a present: the sealed recordings done read")] + cut += block(replay, "void occ_close(occ::Counter& c, VkCommandBuffer cmd) {") + "\n" + colour + "\n" + \ + block(replay, "void occ_draw_begin(VkCommandBuffer cmd) {") + env = {k: v for k, v in os.environ.items() if not k.startswith("RX_VK_")} + flags = ["-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-D_CRT_SECURE_NO_WARNINGS", "-I" + str(ROOT / "gpu/vk")] + with tempfile.TemporaryDirectory(prefix="occ-depth-only-") as tmp: + tmp = Path(tmp) + header = tmp / "occ_answer_test.exe" + subprocess.run([args.cxx, *flags, str(ROOT / "tests/occ_answer_test.cpp"), "-o", str(header)], check=True) + subprocess.run([str(header)], check=True, env=env, timeout=120) + cpp, exe = tmp / "fixture.cpp", tmp / "fixture.exe" + cpp.write_text(FIXTURE.replace("/* CUT */", cut), encoding="utf-8") + subprocess.run([args.cxx, *flags, str(cpp), "-o", str(exe)], check=True) + for mode, more in (("off", {}), ("on", {"RX_VK_OCC_DEPTH_ONLY": "1"}), ("knob", {}), ("off", {"RX_VK_OCC_DEPTH_ONLY": "0"})): + r = subprocess.run([str(exe), mode], capture_output=True, text=True, timeout=60, env={**env, **more}) + need(r.returncode == 0 and r.stdout.strip() == "ok " + mode, "the windows recorded (%s): %s" % (mode, r.stdout.strip() or r.returncode)) + print(r.stdout.strip()) + print("PASS: a window that meets a colour draw is counted for the line, and given up only with RX_VK_OCC_DEPTH_ONLY=1 " + "(off by default)") + + +if __name__ == "__main__": + main() diff --git a/tests/test_occ_gate.py b/tests/test_occ_gate.py new file mode 100644 index 0000000..72258df --- /dev/null +++ b/tests/test_occ_gate.py @@ -0,0 +1,240 @@ +"""A samples-passed count over a draw that a written count's condition left out (on by default, RX_VK_OCC_GATE=0: off; +gpu/vk/occ_answer.h Counter::gate, Writers::kGatedSrc, cond_decide; vk_replay.cpp occ_gate_note). + +Tickets 11-21 (Forest Temple items and trees, black interiors; RX_VK_OCC_QUERY=0 showed them): the game draws an object it +thinks visible under a render condition on its last count, between a reset and the next report. When that count was a 0 +the condition skips the draw, NVN counts nothing for it and neither do the queries: the next count is 0 again because of the +skip, the next condition skips again, and the object never comes back. On the console the game's CPU reads the 0 after its +frame wait and draws the object's proxy instead; here most counts land after the CPU has read the placeholder (4096), so +nothing breaks the loop. Such a count is marked gated (still written, for the CPU); with RX_VK_OCC_GATE=1 a condition on it +renders. What must hold, without the game or a GPU: +- occ_answer.h (compiled and run): gate only while counting; a reset clears it; the cumulative count after a report (keep) + stays gated until the reset; the report carries it; complete writes a gated count (0 too) and marks the word gated; + word() says so; cond_decide renders on it with the switch on, reads memory with it off, a pending word still renders; the + stats line counts them. A frame loop: with the switch off an object hidden once stays hidden after it shows again (the + tickets' loop); with it on it is drawn again within two frames, and a hidden object is still skipped every other frame. +- vk_replay.cpp: the switch read once in occ_init (off unless set), occ_gate_note on draws only (not dispatches, not the dry + pass) after occ_skip_note, the serial replay's state from occ_cond_off and a segment's from par_conds (2: off from a + written count), occ_seg_off cleared at a render enable and a segment's edge; occ_cond reads a gated word again and passes + the switch to cond_decide. +- docs/development.md: the switch in the table. + + python -I -B tests/test_occ_gate.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] + +TEST_CPP = r''' +#include "occ_answer.h" +#include +#include +#include +using namespace mkwvk::occ; +using S = Writers::Src; +static int failures = 0; +#define CHECK(x) do { if (!(x)) { std::fprintf(stderr, "%s:%d: CHECK failed: %s\n", __FILE__, __LINE__, #x); failures++; } } while (0) + +// One object the game draws under a condition on its last count, its draw counted for the next report (the tickets' loop). +// visible[f]: whether it shows in frame f when drawn. The count of frame f lands before frame f + 1's condition. Returns +// the frames it was drawn in. +static std::vector loop(const std::vector& visible, bool gate_on) { + Writers w; + Stats st; + std::vector drawn; + const uint64_t slot_serial = 9; + for (int f = 0; f < (int)visible.size(); f++) { + const uint64_t prev = 0x1000 + 32 * (uint64_t)((f + 1) % 2), addr = 0x1000 + 32 * (uint64_t)(f % 2); // (a ring of two) + bool enabled = true; + if (f) { // (the condition: render if the count is not 0; memory holds the last written count) + const S sx = w.word(prev, f); + const Cond c = cond_decide(w.value(prev) != 0, sx, S::kNone, true, true, gate_on); + st.cond(c.kind); + enabled = c.enabled; + } + Counter k; + k.reset(0, slot_serial, 0); + uint64_t pairs[2] = {0, 1}; + uint32_t used = 0; + if (enabled) { // (draw(): its query begun, the draw recorded) + k.draw_enter(false); + CHECK(k.begin(false, slot_serial)); + k.draw_issued(false), used = 1, pairs[0] = visible[f] ? 500 : 0, drawn.push_back(f); + } else { // (a draw the condition skips never goes into draw(): replay()'s occ_gate_note) + k.gate(false); + } + Rep r; + bool keep_now = false; + CHECK(k.report(false, slot_serial, used, &r, &keep_now) == Counter::kMeasured && !keep_now); + CHECK(r.gated == !enabled); + r.addr = addr, r.pos = 100 + (uint64_t)f, r.present = f; + w.placed(addr, r.pos, f, true, kVisible); + uint64_t mem = kVisible; + complete({r}, pairs, used, true, w, f, st, [&](uint64_t, uint64_t expect, uint64_t count) { + if (mem != expect) return false; + mem = count; + return true; + }, [](uint64_t, uint64_t) {}); + CHECK(mem == (enabled && visible[f] ? 500u : 0u)); // (a gated count is written too: the CPU reads it) + CHECK(w.word(addr, f) == (enabled ? S::kWrittenSrc : S::kGatedSrc)); + } + return drawn; +} + +int main() { + { // the counter: gated only while counting, cleared by a reset, kept past a report while counting goes on (keep) + Counter k; + k.gate(false); + CHECK(!k.gated); + k.reset(0, 5, 0); + k.gate(false); + CHECK(k.gated); + Rep r; + bool kn = false; + CHECK(k.report(false, 5, 0, &r, &kn) == Counter::kMeasured && r.gated); + k.reset(0, 5, 0); + CHECK(!k.gated); + CHECK(k.report(false, 5, 0, &r, &kn) == Counter::kMeasured && !r.gated); + k.gate(false); // (reported, no keep: not counting) + CHECK(!k.gated); + k.gate(true); // (keep: the counting goes on past the report, the next cumulative count is gated) + CHECK(k.gated); + CHECK(k.report(true, 5, 0, &r, &kn) == Counter::kMeasured && r.gated); + k.segment(); + CHECK(!k.gated); + } + { // Writers and cond_decide + Writers w; + w.placed(0x40, 1, 0, true, kVisible); + w.wrote(0x40, 1, 0, true); + CHECK(w.word(0x40, 0) == S::kGatedSrc && w.value(0x40) == 0); + w.placed(0x40, 2, 0, true, kVisible); // (a new report there: not gated, pending) + CHECK(w.word(0x40, 0) == S::kPendingSrc); + w.wrote(0x40, 2, 0); + CHECK(w.word(0x40, 0) == S::kWrittenSrc); + CHECK(cond_decide(false, S::kGatedSrc, S::kNone, true, true, true).enabled); + CHECK(cond_decide(false, S::kGatedSrc, S::kNone, true, true, true).kind == Cond::kGated); + CHECK(cond_decide(false, S::kNone, S::kGatedSrc, true, true, true).kind == Cond::kGated); + CHECK(!cond_decide(false, S::kGatedSrc, S::kNone, true, true, false).enabled); // (off: a written count, from memory) + CHECK(cond_decide(false, S::kGatedSrc, S::kNone, true, true, false).kind == Cond::kWrittenOff); + CHECK(cond_decide(true, S::kGatedSrc, S::kNone, true, true).kind == Cond::kWrittenOn); // (the default: off) + CHECK(cond_decide(false, S::kGatedSrc, S::kPendingSrc, true, true, false).kind == Cond::kUnknown); + CHECK(!cond_decide(false, S::kGatedSrc, S::kNone, false, true, true).enabled); // (the knob's off phase: as before) + CHECK(cond_decide(false, S::kWrittenSrc, S::kNone, true, true, true).kind == Cond::kWrittenOff); // (not gated: skips) + CHECK(std::string(src_name(S::kGatedSrc)).find("written") == 0); + } + { // the frame loop of the tickets: hidden for frames 2-3, shows again from frame 4 + std::vector vis(12, true); + vis[2] = vis[3] = false; + const std::vector off = loop(vis, false); + CHECK(off.size() == 3 && off.back() == 2); // (drawn 0-2, then never again: the skip's 0 keeps it hidden) + const std::vector on = loop(vis, true); + CHECK(std::find(on.begin(), on.end(), 3) == on.end()); // (frame 2's real 0 still skips frame 3, as on the console) + for (int f = 4; f < 12; f++) CHECK(std::find(on.begin(), on.end(), f) != on.end()); // (back as it shows) + std::vector hidden(12, false); + const std::vector h = loop(hidden, true); + CHECK(h == (std::vector{0, 2, 4, 6, 8, 10})); // (still skipped every other frame while it stays hidden) + } + { // the stats line + Stats s; + s.gated = 5, s.gated_written = 4, s.gated_zeros = 3, s.cond_gated_read = 2, s.cond_gated = 1; + s.cond(Cond::kGated); + CHECK(s.cond_gated == 2 && s.conds == 1); + char line[2048]; + CHECK(s.line(line, sizeof line, 600, 1) < (int)sizeof line); + CHECK(std::string(line).find("counts over a draw such a skip left out: 5 counted whole, 4 written (3 of them 0), 2 " + "conditions read one (2 rendered: RX_VK_OCC_GATE=1)") != std::string::npos); + } + if (failures) { + std::fprintf(stderr, "occ_gate: %d failed\n", failures); + return 1; + } + std::puts("occ_gate: all cases pass"); + return 0; +} +''' + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def in_order(where, text, needles): + at = [text.index(n) if n in text else -1 for n in needles] + if -1 in at or at != sorted(at): + raise SystemExit(where + ": not in this order: " + " / ".join(needles)) + + +def check_source(src): + replay = (src / "gpu/vk/vk_replay.cpp").read_text(encoding="utf-8").replace("\r\n", "\n") + in_order("vk_replay.cpp occ_init (RX_VK_OCC_GATE: on unless 0)", block(replay, "bool occ_init() {"), + ['occ_log_cond = getenv("RX_VK_OCC_LOG_COND")', + 'occ_gate_on = !(getenv("RX_VK_OCC_GATE") && atoi(getenv("RX_VK_OCC_GATE")) == 0);']) + if "bool occ_gate_on = true;" not in replay: + raise SystemExit("vk_replay.cpp: occ_gate_on is not on by default") + in_order("vk_replay.cpp occ_gate_note (a draw off from a written count's condition gates the thread's counting)", + block(replay, "void occ_gate_note(const NvnState& s, bool worker) {"), + ["if (s.render_enabled || s.raster_discard || (worker ? occ_seg_off : occ_cond_off) != &s) return;", + "occ_ctr().gate(occ_keep.load(std::memory_order_relaxed));"]) + in_order("vk_replay.cpp occ_cond (a gated word read again; the switch to cond_decide; counted)", + block(replay, "void occ_cond(const NvnState& s, uint64_t addr, int mode, uint64_t& x, uint64_t& y, bool* enabled) {"), + ["const bool gated = sx == occ::Writers::kGatedSrc || sy == occ::Writers::kGatedSrc;", + "sy == occ::Writers::kWrittenSrc || gated)) {", "host->gpu_read(addr, &x, 8);", + "occ::cond_decide(*enabled, sx, sy, active, occ_cond_on, occ_gate_on);", "occ_st.cond_gated_read += active && gated ? 1 : 0;", + "if (c.kind == occ::Cond::kWrittenOff) occ_cond_off = &s;"]) + in_order("vk_replay.cpp occ_report_measure (gated counts counted)", + block(replay, "void occ_report_measure(VkCommandBuffer cmd, uint64_t addr, uint64_t pos, bool writable) {"), + ["case occ::Counter::kMeasured: occ_st.measured++, occ_st.gated += rep.gated ? 1 : 0; break;"]) + in_order("vk_replay.cpp occ_segment_edge (a segment's state cleared)", block(replay, "void occ_segment_edge(VkCommandBuffer cmd) {"), + ["c.segment();", "occ_seg_off = nullptr;"]) + replay_fn = block(replay, "void replay(VkCommandBuffer& cmd, NvnState& s, const uint8_t* p, const uint8_t* end = nullptr) {") + enable = replay_fn[replay_fn.index("case NOP_RENDER_ENABLE:"):replay_fn.index("case NOP_RENDER_COND: {")] + in_order("vk_replay.cpp NOP_RENDER_ENABLE (both threads' states end)", enable, + ["if (occ_live && !worker && occ_cond_off == &s) occ_cond_off = nullptr;", + "if (occ_live && worker && occ_seg_off == &s) occ_seg_off = nullptr;"]) + cond = replay_fn[replay_fn.index("case NOP_RENDER_COND: {"):] + in_order("vk_replay.cpp NOP_RENDER_COND (par_conds: 1 on, 0 off, 2 off from a written count)", cond[:3000], + ["if (worker) {", "const uint8_t v = cond_at < par_conds_n.load(std::memory_order_acquire) ? par_conds[cond_at] : 1;", + "s.render_enabled = v == 1;", "occ_seg_off = v == 2 ? &s : nullptr;", "break;", + "else occ_cond(s, r.addr, (int)r.mode, x, y, &s.render_enabled);", + "const uint8_t v = s.render_enabled ? 1 : occ_live && occ_cond_off == &s ? 2 : 0;", "par_conds.push_back(v);"]) + if "par_conds[cond_at] != 0" in replay: + raise SystemExit("vk_replay.cpp: a segment takes a condition's 2 (off from a written count) for on") + in_order("vk_replay.cpp replay() (draws only, the serial replay's and a segment's, after the skip's note)", replay_fn, + ["h.op == NOP_DISPATCH || h.op == NOP_DISPATCH_INDIRECT) {", "if (occ_live && !worker) occ_skip_note(s, false);", + "if (occ_live && !dry && h.op != NOP_DISPATCH && h.op != NOP_DISPATCH_INDIRECT) occ_gate_note(s, worker);"]) + if replay.count("occ_gate_note(") != 2: # (its definition and replay()'s call) + raise SystemExit("vk_replay.cpp: occ_gate_note is called somewhere other than replay()'s draws") + dev = (src / "docs/development.md").read_text(encoding="utf-8") + if "`RX_VK_OCC_GATE=0`" not in dev: + raise SystemExit("docs/development.md: RX_VK_OCC_GATE=0 is not in the switches table") + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--source", type=Path, default=ROOT) + p.add_argument("--cxx", default="clang++") + a = p.parse_args() + check_source(a.source) + with tempfile.TemporaryDirectory(prefix="occ-gate-") as temp: + cpp, exe = Path(temp) / "occ_gate_test.cpp", Path(temp) / "occ_gate_test.exe" + cpp.write_text(TEST_CPP, encoding="utf-8") + flags = ["-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-D_CRT_SECURE_NO_WARNINGS", "-I" + str(a.source / "gpu/vk")] + subprocess.run([a.cxx, *flags, str(cpp), "-o", str(exe)], check=True) + subprocess.run([str(exe)], check=True) + print("occ_gate: ok") + + +if __name__ == "__main__": + main() diff --git a/tests/test_occ_lean.py b/tests/test_occ_lean.py new file mode 100644 index 0000000..8f5ce39 --- /dev/null +++ b/tests/test_occ_lean.py @@ -0,0 +1,180 @@ +"""The occlusion proxies' census (gpu/vk/occ_lean.h, RX_VK_OCC_LEAN=3; the performance audit's DESC-82, after Source B's: its +first step; no game, no GPU). + +The rules, compiled and run: only 3 turns the census on (1 and 2, the lean pipelines and their check, are not built); each +refusal class refuses, in its order, and a draw with none is lean; the census counts draws and fragment descriptors by +class from several threads, its line gives them a frame and a draw, and the counts start again. Each rule taken out in turn +(a mutation) fails. Then the sources: draw() notes a draw only when an occlusion query is open on its thread after +occ_draw_begin and the draw writes no colour, with the switch; the note reads the fragment stage's Shader::Info and the +state's alpha to coverage and alpha test; the line every 600 presents; and nothing that makes a pipeline, its key or its +descriptors reads the census. + + python -I -B tests/test_occ_lean.py [--cxx clang++] +""" +import argparse +import os +import subprocess +import tempfile +from pathlib import Path + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def in_order(text, parts, where): + at = 0 + for part in parts: + found = text.find(part, at) + if found < 0: + raise SystemExit("%s: %r is missing, or not after the part before it (update the test with the code)" % (where, part)) + at = found + len(part) + + +TEST = r''' +#include +#include +#include +#include +#include "occ_lean.h" +#define CHECK(x) do { if (!(x)) { std::fprintf(stderr, "FAIL:%d %s\n", __LINE__, #x); std::exit(1); } } while (0) +using namespace occ_lean; +int main() { + CHECK(mode_of(nullptr) == 0 && mode_of("") == 0 && mode_of("0") == 0 && mode_of("1") == 0 && mode_of("2") == 0 && mode_of("3") == 3); + CHECK(mode_of("x") == 0 && mode_of("4") == 0); + Facts f; + CHECK(classify(f) == kLean); + // (each refusal refuses, and the first that applies names the class) + { Facts g; g.has_fragment = false; g.depth_out = true; CHECK(classify(g) == kNoFragment); } + { Facts g; g.depth_out = true; g.stores = true; CHECK(classify(g) == kDepth); } + { Facts g; g.sample_mask = true; CHECK(classify(g) == kSampleMask); } + { Facts g; g.demote = true; g.a2c = true; CHECK(classify(g) == kDemote); } + { Facts g; g.stores = true; CHECK(classify(g) == kStores); } + { Facts g; g.image_writes = true; CHECK(classify(g) == kImageWrites); } + { Facts g; g.atomics = true; CHECK(classify(g) == kAtomics); } + { Facts g; g.global = true; CHECK(classify(g) == kGlobal); } + { Facts g; g.a2c = true; g.alpha_test = true; CHECK(classify(g) == kA2c); } + { Facts g; g.alpha_test = true; CHECK(classify(g) == kAlphaTest); } + // (the counts, from two threads, and the line) + Census c; + std::thread t([&c] { for (int i = 0; i < 1000; i++) c.note(kLean, 3); }); + for (int i = 0; i < 1000; i++) c.note(kLean, 3); + t.join(); + for (int i = 0; i < 600; i++) c.note(kDemote, 5); + CHECK(c.draws[kLean].load() == 2000 && c.descriptors[kLean].load() == 6000 && c.draws[kDemote].load() == 600); + char line[1024]; + CHECK(c.line(line, sizeof line, 600) > 0); + CHECK(std::strstr(line, "600 presents): 4 colour-off counted draws a frame; lean 3 a frame with 3.00 fragment descriptors each")); + CHECK(std::strstr(line, "; demote 1 (5.00 descriptors each)") && !std::strstr(line, "stores")); + CHECK(c.draws[kLean].load() == 0 && c.descriptors[kDemote].load() == 0); + CHECK(c.line(line, sizeof line, 600) > 0 && std::strstr(line, ": 0 colour-off counted draws a frame; lean 0 a frame with 0.00")); + char small[40]; + c.note(kStores, 1); + CHECK(c.line(small, sizeof small, 1) > 0 && std::strlen(small) < sizeof small); + std::puts("PASS occ_lean.h: the switch (3 only), each refusal in its order, the counts from two threads, the line, the reset"); + return 0; +} +''' + +MUTATIONS = [ + ("only the census is built", "return e && *e && std::atoi(e) == 3 ? 3 : 0;", "return e && *e ? std::atoi(e) : 0;"), + ("no fragment stage first", " if (!f.has_fragment) return kNoFragment;\n", ""), + ("depth out refuses", " if (f.depth_out) return kDepth;\n", ""), + ("demote refuses", " if (f.demote) return kDemote;\n", ""), + ("stores refuse", " if (f.stores) return kStores;\n", ""), + ("alpha to coverage refuses", " if (f.a2c) return kA2c;\n", ""), + ("the alpha test refuses", " if (f.alpha_test) return kAlphaTest;\n", ""), + ("the counts start again", "d[i] = draws[i].exchange(0, std::memory_order_relaxed)", "d[i] = draws[i].load(std::memory_order_relaxed)"), +] + + +def check_source(draw, draw_h, replay, dev): + d = block(draw, "void draw(VkCommandBuffer cmd, NvnState& s, const DrawArgs& d) {") + in_order(d, ["if (occ_live && ps.open && (writes & 0xFF)) occ_draw_colour(cmd);", "if (occ_live && ps.open) occ_draw_begin(cmd);", + "if (occ_live && ps.open && !(writes & 0xFF) && occ_lean_census_on() && occ_counting()) occ_lean_note(p, s);"], + "vk_draw.cpp draw(): the census after the query may begin, for colour-off draws it counts") + if d.count("occ_lean") != 2: + raise SystemExit("vk_draw.cpp draw(): the census is asked in a place this test does not know") + note = block(draw, "void occ_lean_note(const Program& p, const NvnState& s) {") + in_order(note, ["if (st.vk_stage == VK_SHADER_STAGE_FRAGMENT_BIT) fs = &st;", "f.has_fragment = fs != nullptr;", + "f.depth_out = i.stores_frag_depth, f.sample_mask = i.stores_sample_mask, f.demote = i.uses_demote_to_helper_invocation;", + "f.stores = i.stores_global_memory;", "f.stores = f.stores || d.is_written", + "f.image_writes = i.uses_typeless_image_writes;", "f.atomics = i.uses_global_increment", "i.uses_atomic_image_u32;", + "f.global = i.uses_global_memory;", "f.a2c = s.ms.a2c != 0;", + "f.alpha_test = s.color.alpha_func != 0 && s.color.alpha_func != NVN_ALPHA_FUNC_ALWAYS;", + "occ_lean::census.note(occ_lean::classify(f), descriptors);"], "vk_draw.cpp occ_lean_note") + on = block(draw, "bool occ_lean_census_on() {") + if 'occ_lean::mode_of(e) == 3' not in on or 'getenv("RX_VK_OCC_LEAN")' not in on: + raise SystemExit("vk_draw.cpp occ_lean_census_on: not RX_VK_OCC_LEAN=3 read once") + if "bool occ_counting();" not in draw_h or "bool occ_lean_census_on();" not in draw_h: + raise SystemExit("vk_draw.h: occ_counting and occ_lean_census_on are not declared") + if "bool occ_counting() { return occ_ctr().open >= 0; }" not in replay: + raise SystemExit("vk_replay.cpp: occ_counting is not whether the thread's count has a query open") + tick = block(replay, "void status_tick() {") + in_order(tick, ["if (occ_lean_census_on()) {", "if (++presents % 600 == 0) {", "occ_lean::census.line(line, sizeof line, 600);"], + "vk_replay.cpp status_tick: the census's line every 600 presents") + # (no pipeline, key or descriptor reads it: the census changes nothing drawn) + for marker in ("uint64_t pipeline_key(const Program& p, const NvnState& s, const Targets& at, VkPrimitiveTopology topo) {", + "VkPipeline build_graphics(const Program& p, const NvnState& s, const Targets& at, VkPrimitiveTopology topo, int part, bool* cold) {", + "void build_descriptors(const NvnState& s, const Program& p, bool keep = false, bool scene = false) {"): + if marker not in draw: + raise SystemExit("vk_draw.cpp: %r moved (update the test with the code)" % marker) + if "occ_lean" in block(draw, marker): + raise SystemExit("vk_draw.cpp %s reads the census: it must change no pipeline" % marker) + if "RX_VK_OCC_LEAN" not in dev: + raise SystemExit("docs/development.md: RX_VK_OCC_LEAN is not in the switches table") + + +def build(cxx, d, name, include): + cpp, exe = d / (name + ".cpp"), d / (name + ".exe") + cpp.write_text(TEST, encoding="utf-8") + r = subprocess.run([cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-Wno-unused-function", "-Wno-format-truncation", + "-pthread", "-I" + str(include), str(cpp), "-o", str(exe)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + return (exe, "") if r.returncode == 0 else (None, r.stdout) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--cxx", default=os.environ.get("CXX", "g++")) + a = ap.parse_args() + check_source((VK / "vk_draw.cpp").read_text(encoding="utf-8"), (VK / "vk_draw.h").read_text(encoding="utf-8"), + (VK / "vk_replay.cpp").read_text(encoding="utf-8"), (ROOT / "docs" / "development.md").read_text(encoding="utf-8")) + print("PASS sources: the census in draw() for colour-off draws a query counts, the note's facts, the switch, the line, " + "no pipeline reads it, the row") + header = (VK / "occ_lean.h").read_text(encoding="utf-8") + with tempfile.TemporaryDirectory(prefix="occ-lean-") as tmp: + d = Path(tmp) + exe, out = build(a.cxx, d, "lean", VK) + if not exe: + raise SystemExit("the build failed:\n" + out) + r = subprocess.run([str(exe)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=60) + if r.returncode: + raise SystemExit("the test failed:\n" + r.stdout) + print(r.stdout.strip()) + for i, (what, old, new) in enumerate(MUTATIONS): + if header.count(old) != 1: + raise SystemExit("mutation %r: its text is not once in occ_lean.h (update the test with the header)" % what) + inc = d / ("mutant%d" % i) + inc.mkdir() + (inc / "occ_lean.h").write_text(header.replace(old, new), encoding="utf-8") + exe, out = build(a.cxx, d, "mutant%d" % i, inc) + if exe: + r = subprocess.run([str(exe)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=60) + if r.returncode == 0: + raise SystemExit("mutation %r: the test still passed without it" % what) + print("PASS mutations: %d rules of the census, each taken out, fail the test" % len(MUTATIONS)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_oot_menu_linux_half.py b/tests/test_oot_menu_linux_half.py index 0ba27e2..b211c26 100644 --- a/tests/test_oot_menu_linux_half.py +++ b/tests/test_oot_menu_linux_half.py @@ -1,12 +1,25 @@ -"""host/oot_menu.c's two halves: what other host files call is defined in both. +"""host/oot_menu.c's two halves: what other host files call is defined in both, and benchmark mode presses through the start +in both. The file is one half on Windows (#if defined(_WIN32)) and another elsewhere (#else). A function the Windows half added for the rest of the host (v0.5.0's hos_oot_mods_down, nn_hid.c) and not the other half stopped every Linux build at the link (11-18). This test lists the non-static functions each half defines and the ones host/*.c outside it call, and checks that every one called from outside is in the non-Windows half too. + +(Triforce, 11-22) Benchmark mode's presses through the start's screens (bench_menu) were the Windows half's alone, and on Linux +"Find my best settings" stood on the title screen (#bug-tracker, 11-21). The other half is compiled here (on Windows with +HOS_OOT_MENU_NO_MOUSE, as tests/test_oot_menu_mouse.py does) against a fake guest memory holding a screen's list of entries +(+136 the head, +144 the first node, +156 the entry's offset from its node; entry+24 the button, its focus at +56, its hit box +at *(button+0x58)) and the screen's focus manager (*(*(screen+32)+96)): with benchmark mode on, a screen with a focused button +shown is handed to hos_bench_menu with its vtable, the focused entry and the count, and its A is pressed; no focused button, a +hidden box, no focus manager or benchmark mode off: nothing. The half still has none of the mouse, Quit or Mods. """ import re +import shutil +import subprocess +import sys +import tempfile import unittest from pathlib import Path @@ -35,6 +48,139 @@ class OotMenuHalvesTest(unittest.TestCase): missing = sorted(called - other_defs) self.assertEqual(missing, [], "defined only in oot_menu.c's Windows half, but called from other host files: %s" % missing) + def test_other_half_has_no_mouse_quit_or_mods(self): + _, other = halves(SRC.read_text(encoding="utf-8")) + other = re.sub(r"/\*.*?\*/", "", other, flags=re.S) # (its comments name the Windows half's) + for name in ("hos_mouse_game_pointer", "hos_mouse_game_menu", "mods_list_tick", "quit_entry", "mods_entry"): + self.assertNotRegex(other, r"\b%s" % name, "the non-Windows half has %s" % name) + + +HOS_H = r""" +#pragma once +#include +#include +typedef struct Ctx { uint64_t x[32]; uint64_t vreg[32][2]; } Ctx; +extern uint8_t *fake_heap; +#define FAKE_BASE 0x100000ULL +#define HOS_GUEST_SPACE (FAKE_BASE + (4ULL << 20)) +#define GP(a) ((void *)(fake_heap + ((uint64_t)(a) - FAKE_BASE))) +void hos_log(const char *fmt, ...); +int64_t hos_now_ns(void); +""" +VK_API_H = "#pragma once\nenum { HOS_OPT_GAME_OPTIONS = 48 };\n" +BENCH_H = r""" +#pragma once +#include +enum { HOS_BENCH_MENU_NONE, HOS_BENCH_MENU_PRESS_A }; +int hos_bench_on(void); +int hos_bench_menu(uint64_t screen, uint64_t vt, int focus, int focused_entry, int entries, int *entry); +""" +FIXTURE = r""" +#include "hos.h" +#include "hos_bench.h" +#include +#include +#include +uint8_t *fake_heap; +static uint64_t heap_top = FAKE_BASE + 0x1000; +int64_t hos_now_ns(void) { return 1000000000LL; } +void hos_log(const char *fmt, ...) { (void)fmt; } +void rx_option_set(int id, int value) { (void)id, (void)value; } +void hos_oot_menu_tick(Ctx *c, uint64_t screen); +static int bench, asked, asked_entry, asked_entries; +static uint64_t asked_screen, asked_vt; +int hos_bench_on(void) { return bench; } +int hos_bench_menu(uint64_t screen, uint64_t vt, int focus, int focused_entry, int entries, int *entry) { + (void)entry; + asked++, asked_screen = screen, asked_vt = vt, asked_entry = focused_entry, asked_entries = entries; + return focus ? HOS_BENCH_MENU_PRESS_A : HOS_BENCH_MENU_NONE; +} +static char taps[256]; +void hos_hid_tap(const char *press) { strncat(taps, press, sizeof taps - strlen(taps) - 2), strcat(taps, ";"); } +static uint64_t alloc(size_t n) { const uint64_t a = heap_top; heap_top += (n + 15) & ~(size_t)15; return a; } +static void put64(uint64_t a, uint64_t v) { memcpy(GP(a), &v, 8); } +static uint64_t get64(uint64_t a) { uint64_t v; memcpy(&v, GP(a), 8); return v; } +static void putf(uint64_t a, float f) { memcpy(GP(a), &f, 4); } +/* a screen of n buttons (vtable vt) with its focus manager: the buttons' boxes shown, none focused */ +static uint64_t button[8]; +static uint64_t screen_of(uint64_t vt, int n) { + const uint64_t s = alloc(256), ui = alloc(128); + put64(s, vt), put64(s + 32, ui), put64(ui + 96, alloc(16)); + const int32_t off = 0; + memcpy(GP(s + 156), &off, 4); + uint64_t prev = s + 144; + for (int i = 0; i < n; i++) { + const uint64_t node = alloc(64), b = alloc(128), box = alloc(0x110); + put64(prev, node), put64(node + 24, b), put64(b + 0x58, box); + putf(box + 0x28, 1), putf(box + 0x38, 1), putf(box + 0x4c, 0), putf(box + 0x50, -60.f * i); + putf(box + 0x84, -200), putf(box + 0x88, -30), putf(box + 0x8c, 400), putf(box + 0x90, 60); + button[i] = b, prev = node + 8; + } + put64(prev, s + 136); + return s; +} +static int fails; +#define CHECK(x) do { if (!(x)) { printf("FAIL line %d: %s\n", __LINE__, #x); fails++; } } while (0) +int main(void) { + fake_heap = calloc(1, 4 << 20); + Ctx ctx; + memset(&ctx, 0, sizeof ctx); + const uint64_t title_vt = 0x84c106e8ULL, title = screen_of(title_vt, 4); + bench = 1; + hos_oot_menu_tick(&ctx, title); /* (no button focused yet: the title still fading in) */ + CHECK(asked == 0 && !taps[0]); + *(uint8_t *)GP(button[2] + 56) = 1; + bench = 0; + hos_oot_menu_tick(&ctx, title); /* (benchmark mode off: a player's menus, nothing pressed) */ + CHECK(asked == 0 && !taps[0]); + bench = 1; + hos_oot_menu_tick(&ctx, title); + CHECK(asked == 1 && asked_screen == title && asked_vt == title_vt && asked_entry == 2 && asked_entries == 4); + CHECK(!strcmp(taps, "0:A/120;")); + /* (the focused button's box hidden: its page not shown) */ + taps[0] = 0; + putf(get64(button[2] + 0x58) + 0x28, 0); + hos_oot_menu_tick(&ctx, title); + CHECK(asked == 1 && !taps[0]); + /* (another screen, its focus manager gone: it is not taking the player's input) */ + const uint64_t files = screen_of(0x84c0eba0ULL, 3); + *(uint8_t *)GP(button[0] + 56) = 1; + put64(get64(files + 32) + 96, 0); + hos_oot_menu_tick(&ctx, files); + CHECK(asked == 1 && !taps[0]); + put64(get64(files + 32) + 96, alloc(16)); + hos_oot_menu_tick(&ctx, files); + CHECK(asked == 2 && asked_screen == files && asked_entry == 0 && asked_entries == 3 && !strcmp(taps, "0:A/120;")); + if (!fails) printf("PASS\n"); + return fails != 0; +} +""" + + +class OotMenuBenchOtherHalfTest(unittest.TestCase): + def test_bench_presses_through_the_start_without_the_mouse(self): + compiler = shutil.which("gcc") or shutil.which("clang") + if not compiler: + self.skipTest("no C compiler") + with tempfile.TemporaryDirectory(prefix="triforce-oot-menu-bench-") as tmp: + tmp = Path(tmp) + (tmp / "host").mkdir() + (tmp / "gpu" / "vk").mkdir(parents=True) + (tmp / "host" / "hos.h").write_text(HOS_H, encoding="utf-8") + (tmp / "gpu" / "vk" / "vk_api.h").write_text(VK_API_H, encoding="utf-8") + (tmp / "host" / "hos_bench.h").write_text(BENCH_H, encoding="utf-8") + shutil.copy(SRC, tmp / "host" / "oot_menu.c") + shutil.copy(ROOT / "host" / "guest_bounds.h", tmp / "host" / "guest_bounds.h") + (tmp / "host" / "t.c").write_text(FIXTURE, encoding="utf-8") + binary = tmp / "t.exe" + other_half = ["-DHOS_OOT_MENU_NO_MOUSE"] if sys.platform == "win32" else [] + build = subprocess.run([compiler, "-std=gnu11", "-Wall", "-Werror", "-O1", "-g"] + other_half + + [str(tmp / "host" / "t.c"), str(tmp / "host" / "oot_menu.c"), "-o", str(binary)], + text=True, capture_output=True) + self.assertEqual(build.returncode, 0, build.stderr) + run = subprocess.run([str(binary)], text=True, capture_output=True, timeout=30) + self.assertEqual(run.returncode, 0, run.stdout + run.stderr) + if __name__ == "__main__": raise SystemExit(unittest.main(verbosity=1)) diff --git a/tests/test_oot_menu_mouse.py b/tests/test_oot_menu_mouse.py index 1a9295d..4b3165b 100644 --- a/tests/test_oot_menu_mouse.py +++ b/tests/test_oot_menu_mouse.py @@ -62,6 +62,7 @@ enum { HOS_OPT_GAME_OPTIONS = 48 }; enum { HOS_OPT_MODS_LIST = 150, HOS_OPT_MODS_X0, HOS_OPT_MODS_X1, HOS_OPT_MODS_Y0, HOS_OPT_MODS_Y1, HOS_OPT_MODS_PITCH, HOS_OPT_MODS_ENTER, HOS_OPT_MODS_SELECTED, HOS_OPT_MODS_PRESS }; enum { HOS_MODS_LIST_SHOWN = 1, HOS_MODS_LIST_LAST_FOCUSED = 2, HOS_MODS_LIST_TITLE = 4 }; +enum { HOS_OPT_MODS_ALPHA = 162 }; """ FIXTURE = r""" @@ -160,6 +161,7 @@ static uint64_t button_new(uint64_t s, float x, float y, float w, float h) { const uint64_t entry = alloc(64), button = alloc(128), box = alloc(0x200); put64(entry + 24, button), put64(entry + 8, s + 136), put64(button + 0x58, box); putf(box + 0x28, 1), putf(box + 0x38, 1), putf(box + 0x4c, x), putf(box + 0x50, y); + putf(box + 0x78, 1); /* ((11-20) the button's alpha, as the game has it for a list that shows: mods_list_tick) */ putf(box + 0x84, -w / 2), putf(box + 0x88, -h / 2), putf(box + 0x8c, w), putf(box + 0x90, h); uint64_t node = s + 144; /* (the link to the head: the last node's +8, or the first's at +144) */ while (get64(node) != s + 136) node = get64(node) + 8; @@ -536,6 +538,7 @@ int64_t hos_now_ns(void) { return now_ns; } void hos_log(const char *fmt, ...) { (void)fmt; } static int told, told_id, told_value; void rx_option_set(int id, int value) { told++, told_id = id, told_value = value; } +void hos_hid_tap(const char *press) { (void)press; } /* ((Triforce, 11-22) benchmark mode's presses, both halves': off here) */ void hos_oot_menu_focus(Ctx *c, uint64_t screen, uint64_t pane, uint64_t lr); void hos_oot_menu_tick(Ctx *c, uint64_t screen); int hos_oot_options_front(void); diff --git a/tests/test_oot_shadow_step.py b/tests/test_oot_shadow_step.py new file mode 100644 index 0000000..bef3d11 --- /dev/null +++ b/tests/test_oot_shadow_step.py @@ -0,0 +1,137 @@ +"""A character's or an object's shadow map's side is rounded up to a multiple of 64 (host/oot_shadow_step.h, host/ +oot_shadow_step.c, nx2/oot_patches.py patch_shadow_step, HOS_OOT_SHADOW_STEP; no game). + +(Triforce, 11-21, the performance audit's GAME-05, after Source B's.) The game sizes each nearby character's or object's +shadow map by its size on screen, so while anything moves each frame asks for depth textures of sides not seen before; +the render targets' pool gives an image back only to a target of exactly its shape. + +1. The rule, host/oot_shadow_step.h compiled alone: a side goes up to the next multiple of the step and a multiple stays; + sides from 384 to 1,920 become 25 sizes; no side crosses 768 (the renderer's rule for maps of 768 and less); at most + 63 more texels; step 0 or 1 leaves the side; the switch's values. +2. The host's function (source): the step is the rule's of HOS_OOT_SHADOW_STEP, read once and logged; the answer the rule's. +3. The lift's patch: the one read of the side in f_802640a0 goes through the host, in either form the lifter writes a + register in, once, with its declaration; the same line in another function is not touched; a text patched before is + left as it is; a text without the function, with the read twice, or without the include is not touched; it is among + the supported build's patches, with a warning when its place is not found. + + python -I -B tests/test_oot_shadow_step.py [--cc clang] +""" +import argparse +import subprocess +import sys +import tempfile +from pathlib import Path + +RUN_ALL_ARGS = ["--cc", "clang"] + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "nx2")) + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +PROGRAM = r""" +#include +#include +#include "oot_shadow_step.h" +static int fails; +#define CHECK(x) do { if (!(x)) { printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +int main(void) { + CHECK(oot_shadow_side(385, 64) == 448 && oot_shadow_side(448, 64) == 448 && oot_shadow_side(449, 64) == 512); + CHECK(oot_shadow_side(777, 64) == 832 && oot_shadow_side(2400, 64) == 2432 && oot_shadow_side(1920, 64) == 1920); + CHECK(oot_shadow_side(500, 0) == 500 && oot_shadow_side(500, 1) == 500 && oot_shadow_side(500, 256) == 512); + CHECK(oot_shadow_side(0, 64) == 0 && oot_shadow_side(1, 64) == 64); + int sizes = 0; + uint32_t last = 0; + for (uint32_t s = 384; s <= 1920; s++) { + const uint32_t r = oot_shadow_side(s, 64); + CHECK(r >= s && r - s <= 63 && r % 64 == 0); + CHECK((s <= 768) == (r <= 768)); /* (no map crosses the renderer's line at 768) */ + if (r != last) sizes++, last = r; + } + CHECK(sizes == 25); + /* the switch */ + CHECK(oot_shadow_step(NULL) == 64 && oot_shadow_step("") == 64 && oot_shadow_step("abc") == 64 && oot_shadow_step("64x") == 64); + CHECK(oot_shadow_step("0") == 1 && oot_shadow_step("1") == 1); + CHECK(oot_shadow_step("2") == 2 && oot_shadow_step("32") == 32 && oot_shadow_step("128") == 128 && oot_shadow_step("256") == 256); + CHECK(oot_shadow_step("48") == 64 && oot_shadow_step("512") == 64 && oot_shadow_step("99999999") == 64); + return fails ? 1 : 0; +} +""" + + +def check_source(): + host = (ROOT / "host" / "oot_shadow_step.c").read_text(encoding="utf-8") + side = block(host, "uint32_t hos_oot_shadow_side(uint32_t side) {") + check('s = (int)oot_shadow_step(getenv("HOS_OOT_SHADOW_STEP"));' in side and "const uint32_t out = oot_shadow_side(side, (uint32_t)s);" in side and + "return out;" in side, "the host's answer is not the rule's of the switch's step") + check("__atomic_compare_exchange_n(&step, &unset, s," in side and 'hos_log("oot: shadow maps\' sides rounded up to a multiple of %d' in side, + "the step is not kept once, or leaves no line") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`HOS_OOT_SHADOW_STEP=`" in dev, "docs/development.md: HOS_OOT_SHADOW_STEP is not in the switches table") + + +def check_patch(): + import oot_patches as o + head, mark, decl = o.SHADOW_STEP_HEAD, o.SHADOW_STEP_MARK, o.SHADOW_STEP_DECL + check("f_802640a0" in head, "the function is not the one that sizes a shadow map") + check(len(o.SHADOW_STEP_FORMS) == 2, "the two forms a register is written in are not both looked for") + inc = '#include "a64.h"\n' + for old, new in o.SHADOW_STEP_FORMS: + check("+ 1256" in old and "recomp_load32(" in old and new.count("hos_oot_shadow_side(") == 1 and old.rstrip(";")[old.index("=") + 1:] in new, + "the new text does not round the game's own read of the side at +0x4e8") + other = "void f_80264300(Ctx *__restrict A64_NOESC c) {\n " + old + "\n}\n" # (the same read in another function) + text = inc + head + " c->x[21]=c->x[0];\n " + old + "\n c->x[11]=c->x[10];\n}\n" + other + out, found = o.patch_shadow_step(text) + check(found == 1 and out.count(new + mark) == 1 and out.count(old) == 1 and out.count(decl) == 1 and out.endswith(other), + "patch_shadow_step: the read is not made through the host once, or another function's is") + check(out.index(decl) < out.index(head), "patch_shadow_step: the host's function is not declared before its use") + again, found = o.patch_shadow_step(out) + check(found == 1 and again == out, "patch_shadow_step: a text patched before is patched again") + for what, bad in (("without the function", inc + other), + ("with the read twice in the function", inc + head + " " + old + "\n " + old + "\n}\n"), + ("with another offset", inc + head + " " + old.replace("1256", "1260") + "\n}\n"), + ("without the include to declare at", head + " " + old + "\n}\n")): + got, found = o.patch_shadow_step(bad) + check(not found and got == bad, "patch_shadow_step: a text %s is touched" % what) + src = (ROOT / "nx2" / "oot_patches.py").read_text(encoding="utf-8") + apply = src[src.index("def apply(gen, *, protect_hearts=False, required=False):"):] + at = 0 + for part in ("shadow_step_sites = 0", "if protect_hearts: # (found by address: the supported build's functions)", + "stepped, found = patch_shadow_step(out)", "shadow_step_sites += found", "out = stepped", "if out != s:", + "if protect_hearts and shadow_step_sites != 1:"): + found = apply.find(part, at) + check(found >= 0, "oot_patches.py apply: %r is missing, or not after the part before it" % part) + at = found + len(part) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cc", default="clang") + args = parser.parse_args() + check_source() + check_patch() + with tempfile.TemporaryDirectory(prefix="oot-shadow-step-") as tmp: + c, exe = Path(tmp) / "step.c", Path(tmp) / "step.exe" + c.write_text(PROGRAM, encoding="utf-8") + subprocess.run([args.cc, "-std=c11", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(ROOT / "host"), str(c), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True, timeout=60) + check(r.returncode == 0, "the rule as made:\n" + r.stdout) + print("PASS: a shadow map's side is rounded up to a multiple of 64, through the host: 25 sizes from 384 to 1,920") + + +if __name__ == "__main__": + main() diff --git a/tests/test_oot_tex_compact.py b/tests/test_oot_tex_compact.py index 225a2a7..d0ba8b3 100644 --- a/tests/test_oot_tex_compact.py +++ b/tests/test_oot_tex_compact.py @@ -9,6 +9,10 @@ 3. The lift's patch: the one call of the step in the compaction thread's handler is made through the host, once, with its declaration; a text patched before is left as it is; a text with the call twice, or without it, is not touched; and it is among the supported build's patches, with a warning when its place is not found. +4. (Triforce) The ask (host/oot_tex_compact.h, compiled alone): HOS_OOT_TEX_ASK_MS's values (unset 1000, 0 every ask, off + none); a gap lets the first ask through, then none until the gap has passed since the last one let through. The host's + hos_oot_tex_compact_due reads the switch and asks the rule; the lift's patch adds the host's test to the manager's ask + (f_80797240) once, with its declaration, and leaves other text alone; apply() makes it with the supported build's patches. python -I -B tests/test_oot_tex_compact.py [--cc gcc] """ @@ -107,6 +111,77 @@ def check_patch(): "if protect_hearts and tex_compact_sites != 1:"], "oot_patches.py apply") +ASK_TEST = r""" +#include +#include +#include +#include "oot_tex_compact.h" +#define CHECK(x) do { if (!(x)) { fprintf(stderr, "FAIL:%d %s\n", __LINE__, #x); exit(1); } } while (0) +int main(void) { + /* (the switch) */ + CHECK(hos_oot_tex_ask_ms(NULL) == 1000 && hos_oot_tex_ask_ms("") == 1000 && hos_oot_tex_ask_ms("x") == 1000); + CHECK(hos_oot_tex_ask_ms("12x") == 1000 && hos_oot_tex_ask_ms("0") == 0 && hos_oot_tex_ask_ms("250") == 250); + CHECK(hos_oot_tex_ask_ms("off") == -1 && hos_oot_tex_ask_ms("-1") == -1 && hos_oot_tex_ask_ms("-50") == -1); + CHECK(hos_oot_tex_ask_ms("off2") == 1000 && hos_oot_tex_ask_ms("99999999") == 3600000); + /* (0: every ask; -1: none) */ + int64_t last = -1; + for (int i = 0; i < 5; i++) CHECK(hos_oot_tex_ask_due(0, i, &last)); + last = -1; + for (int i = 0; i < 5; i++) CHECK(!hos_oot_tex_ask_due(-1, (int64_t)i * 2000000000LL, &last)); + /* (1000 ms: the first ask, then none for a second after it, then the next) */ + const int64_t ms = 1000000; + last = -1; + CHECK(hos_oot_tex_ask_due(1000, 5 * ms, &last) && last == 5 * ms); + CHECK(!hos_oot_tex_ask_due(1000, 6 * ms, &last) && !hos_oot_tex_ask_due(1000, 1004 * ms, &last) && last == 5 * ms); + CHECK(hos_oot_tex_ask_due(1000, 1005 * ms, &last) && last == 1005 * ms); + CHECK(!hos_oot_tex_ask_due(1000, 2004 * ms, &last) && hos_oot_tex_ask_due(1000, 2100 * ms, &last)); + /* (a second of asks at 120 frames a second: one let through) */ + int let = 0; + last = -1; + for (int f = 0; f < 120; f++) let += hos_oot_tex_ask_due(1000, 10000 * ms + (int64_t)f * 8333333, &last); + CHECK(let == 1); + puts("PASS ask: unset 1000, 0 every ask, off none; one ask a gap, the first at once"); + return 0; +} +""" + + +def check_ask(host): + due = block(host, "int hos_oot_tex_compact_due(void) {") + in_order(due, ['getenv("HOS_OOT_TEX_ASK_MS")', "hos_oot_tex_ask_ms(e)", "const int64_t now = hos_now_ns();", + "const int due = hos_oot_tex_ask_due(ms, now, &last);", "return due;"], "oot_tex_compact.c hos_oot_tex_compact_due") + if '#include "oot_tex_compact.h"' not in host: + raise SystemExit("oot_tex_compact.c: the rule's header is not included") + import oot_patches + head, old, new, decl = oot_patches.TEX_ASK_HEAD, oot_patches.TEX_ASK_OLD, oot_patches.TEX_ASK_NEW, oot_patches.TEX_ASK_DECL + if (head != "void f_80797240(Ctx *__restrict A64_NOESC c) {\n" or "c->x[19] + 960" not in old + or "if (!((c->x[8] >> 1) & 1)) { goto L_80797288; }" not in old + or "if (!((c->x[8] >> 1) & 1) || !hos_oot_tex_compact_due()) { goto L_80797288; }" not in new): + raise SystemExit("oot_patches.py: the ask's test is not the manager's step's test of +960's bit 1") + other = "void f_80000000(Ctx *__restrict A64_NOESC c) {\n" + old + "\n}\n" + text = '#include "a64.h"\n' + other + head + " c->x[8]=recomp_load8(c,c->x[0] + 960);\n" + old + "\nL_80797288:;\n}\n" + other + out, found = oot_patches.patch_tex_ask(text) + if found != 1 or out.count(new) != 1 or out.count(decl) != 1 or out.count(old) != 2 or out.replace(new, old).replace(decl, "") != text: + raise SystemExit("patch_tex_ask: the manager's ask is not made through the host once, or other text changed") + if out.index(new) < out.index(head): + raise SystemExit("patch_tex_ask: another function's text was patched") + again, found = oot_patches.patch_tex_ask(out) + if found != 1 or again != out: + raise SystemExit("patch_tex_ask: a text patched before is patched again") + for bad in ('#include "a64.h"\n' + other, '#include "a64.h"\n' + head + old + "\n" + old + "\n}\n", head + old + "\n}\n"): + out, found = oot_patches.patch_tex_ask(bad) + if found or out != bad: + raise SystemExit("patch_tex_ask: a text without the manager's one test and the one include is touched") + src = (ROOT / "nx2" / "oot_patches.py").read_text(encoding="utf-8") + apply = src[src.index("def apply(gen, *, protect_hearts=False, required=False):"):] + in_order(apply, ["if protect_hearts: # (found by address: the supported build's functions)", + "asked, found = patch_tex_ask(out)", "tex_ask_sites += found", "out = asked", + "if protect_hearts and tex_ask_sites != 1:"], "oot_patches.py apply") + docs = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + if "`HOS_OOT_TEX_ASK_MS=`" not in docs: + raise SystemExit("docs/development.md: no HOS_OOT_TEX_ASK_MS row") + + def main(): p = argparse.ArgumentParser(description=__doc__) p.add_argument("--cc", default="gcc") @@ -114,6 +189,7 @@ def main(): host = (ROOT / "host" / "oot_tex_compact.c").read_text(encoding="utf-8") check_source(host) check_patch() + check_ask(host) print("PASS sources: only an idle compaction is held, as the planner's nothing to move; the lift's one call through the host") rule = block(host, "int hos_oot_tex_compact_runs(int ms, int64_t now, int64_t next) {") with tempfile.TemporaryDirectory(prefix="oot-tex-compact-") as tmp: @@ -139,6 +215,14 @@ def main(): if ok: # (a mutant that does not build has failed too) raise SystemExit("mutation %r: the rule's test still passed without it" % what) print("PASS mutations: %d parts of the rule, each taken out, fail its test" % len(MUTATIONS)) + c, exe = d / "ask.c", d / "ask.exe" + c.write_text(ASK_TEST, encoding="utf-8") + subprocess.run([a.cc, "-std=gnu11", "-O1", "-Wall", "-Wextra", "-Werror", "-I", str(ROOT / "host"), str(c), "-o", str(exe)], + check=True) + r = subprocess.run([str(exe)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=60) + if r.returncode: + raise SystemExit("the ask's test failed:\n" + r.stdout) + print(r.stdout.strip()) if __name__ == "__main__": diff --git a/tests/test_open_file_vcore.py b/tests/test_open_file_vcore.py new file mode 100644 index 0000000..ea10347 --- /dev/null +++ b/tests/test_open_file_vcore.py @@ -0,0 +1,71 @@ +"""nn::fs::OpenFile gives its virtual core up before the path is worked out, and takes it back on every way out +(host/nn_fs.c OpenFile, host/prompts.c hos_prompts_font_file; no game). + +(Triforce, 11-21, the performance audit's IO-24, after Source A's.) OpenFile gave the core up only around the host's +own open. Working the path out reaches the button prompts' builder, which builds a file at its first open, a quarter of +a second and more, while the other openers wait for it: each of them held a virtual core the whole time. + +Reads the source, and walks OpenFile's text: the core is given up after the guest's path is read and before host_path; +between the give-up and the host's open no guest memory is read or written but through RET; every return on the way +has the take-back right before its RET; the open's own take-back is still there; with the switch the core is given up +around CreateFileA alone, as before; the prompts' font hook gives the core up around the builder, not inside it. + + python -I -B tests/test_open_file_vcore.py +""" +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def main(): + fs = (ROOT / "host" / "nn_fs.c").read_text(encoding="utf-8") + body = block(fs, "HLE(_ZN2nn2fs8OpenFileEPNS0_10FileHandleEPKci) {") + code = re.sub(r"/\*.*?\*/", " ", body, flags=re.S) + give, path, create = code.index("int away = open_keep_core ? 0 : hos_vcore_away();"), code.index("host_path(g, path"), code.index("CreateFileA(") + check(code.index("const char *g = gstr(ARG(1));") < give < path < create, + "the core is not given up after the guest's path is read and before host_path") + check('open_keep_core = getenv("HOS_OPEN_KEEP_CORE") != NULL;' in code and + "if (open_keep_core) away = hos_vcore_away();" in code[give:create], + "the switch does not keep the core until the host's open, as before") + away_part = code[give:create] + # (no guest memory touched while the core is away, except the answer: each RET is right after a take-back) + check("gstr(" not in away_part and "G64(" not in away_part and "G32(" not in away_part and "recomp_" not in away_part, + "guest memory is read or written while the core is away") + returns = [m.start() for m in re.finditer(r"\bRET\(", away_part)] + check(len(returns) == 4, "OpenFile has %d ways out before its open: this test knows 4" % len(returns)) + for at in returns: + before = away_part[:at].rstrip() + check(before.endswith("if (away) hos_vcore_back();"), "a return before the open answers the game without taking the core back: ..." + before[-80:]) + after = code[create:] + check(after.index("if (away) hos_vcore_back();") < after.index("INVALID_HANDLE_VALUE"), "the core is not taken back right after the host's open") + check(code.count("hos_vcore_away()") == 2 and code.count("hos_vcore_back()") == 5, "a give-up or take-back this test does not know") + prompts = (ROOT / "host" / "prompts.c").read_text(encoding="utf-8") + font = block(prompts, "int hos_prompts_font_file(const char *key, char *out, size_t cap) {") + at = [font.index(p) for p in ("if (strcmp(key, FONT_PATH)) return 0;", "const int away = hos_vcore_away();", + "hos_prompts_romfs(FONT_PATH, out, cap);", "if (away) hos_vcore_back();", "return found;")] + check(at == sorted(at), "the font hook does not give the core up around the builder") + romfs = block(prompts, "int hos_prompts_romfs(const char *key, char *out, size_t cap) {") + check("hos_vcore_away" not in romfs, "the core is given up inside the builder, where its lock is held") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`HOS_OPEN_KEEP_CORE=1`" in dev, "docs/development.md: HOS_OPEN_KEEP_CORE=1 is not in the switches table") + print("PASS: OpenFile gives its core up before the path is worked out and takes it back on every way out") + + +if __name__ == "__main__": + main() diff --git a/tests/test_order_wake.py b/tests/test_order_wake.py new file mode 100644 index 0000000..f328ba0 --- /dev/null +++ b/tests/test_order_wake.py @@ -0,0 +1,256 @@ +"""The order thread woken when the CPU signals an event while a queue is stopped at a wait (gpu/vk/vk_replay.cpp +order_wake_on, sync_watch, event_signal_queued, the hold loop of wait; no game, no GPU). + +(Triforce, 11-22, the performance audit's SYNC-59, after Source B's.) A queue stopped at a wait goes on when a submit +finds the wait met. A signal of the CPU's wakes no one: the stop waits for the order thread's next Sleep(1), or for a +game thread's wait that yields up to 3 ms and then forces it. RX_VK_ORDER_WAKE=1 (off unless set; the lab knob +orderwake): the order thread waits on an auto-reset event, 1 ms at most, that the signal sets. + +This cuts order_wake_on, sync_watch (its endless loop bounded) and the signal's part of event_signal_queued out of the +source and runs them over stand-ins, once with the switch unset and once with RX_VK_ORDER_WAKE=1: + - unset: the order thread sleeps 1 ms a turn and never waits on the event; a signal sets nothing; + - on: it waits on the event, 1 ms at most, and never sleeps; a signal sets the event exactly when a queue is stopped, + after the signal is queued and marked, holding no lock (ev_cpu_mu is free at the SetEvent); + - either way the order thread makes the same looks, in the same turns, from the same stopped queues (burst_begin, + schedule, burst_end, update_blocked, and the forced drain past hold_ms): the script's decisions print the same; + - on, a turn the event woke counts as a look, and as a resume when it let work on. + +And reads the source: the hold loop of wait lets the met waits on inside its sync_order_mu block when a signal is +pending and the switch is on, with the four calls sync_settle makes, before it asks what is left; the order line counts +the wakes; the switches table has the row. + + python -I -B tests/test_order_wake.py [--cxx clang++] +""" +import argparse +import os +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + check(text.count(marker) == 1, "vk_replay.cpp does not have once: " + marker) + start = text.index(marker) + end = text.index("{", start + len(marker) - 1) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +FIXTURE = r""" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) + +// ---- stand-ins ---- +typedef void* HANDLE; +#define FALSE 0 +#define WAIT_OBJECT_0 0u +#define WAIT_TIMEOUT 258u +static int the_thread; +static HANDLE GetCurrentThread() { return &the_thread; } +static void SetThreadDescription(HANDLE, const wchar_t*) {} +static int the_event, events_made = 0; +static HANDLE CreateEventA(void*, int manual, int initial, const char*) { + CHECK(!manual && !initial); // (auto-reset, not signalled) + events_made++; + return &the_event; +} +static bool event_set = false; +static int sleeps = 0, event_waits = 0, sets = 0, sets_locked = 0, sets_before_store = 0; +static std::string trace; // (what the order thread did, turn by turn) +static void Sleep(unsigned ms) { CHECK(ms == 1); sleeps++; } +static unsigned WaitForSingleObject(HANDLE h, unsigned ms) { + CHECK(h == &the_event && ms == 1); + event_waits++; + const bool was = event_set; + event_set = false; // (auto-reset) + return was ? WAIT_OBJECT_0 : WAIT_TIMEOUT; +} +namespace lab_ab { bool on(const char*) { return false; } } +static void logf(const char*, ...) {} +static bool async_on() { return false; } +// (a lock that can say whether it is held, asked from the thread that may hold it) +struct Mu { + bool held = false; + void lock() { held = true; } + void unlock() { held = false; } + bool try_lock() { return held ? false : (held = true); } +}; +Mu ev_cpu_mu; +std::vector> ev_cpu_pending; +std::atomic ev_cpu_any{false}; +std::atomic ev_cpu_signals{0}; // ((Triforce, 11-22, SYNC-29) the signal's part counts it for the reads' gate +static int SetEvent(HANDLE h) { + CHECK(h == &the_event); + sets++, event_set = true; + if (!ev_cpu_mu.try_lock()) sets_locked++; // (the signalling thread holds no lock at the wake) + else ev_cpu_mu.unlock(); + if (!ev_cpu_any.load() || ev_cpu_pending.empty()) sets_before_store++; + return 1; +} +// (the order's state: the script says, turn by turn, how many queues are stopped and whether their wait is met) +std::atomic blocked_n{0}; +Mu sync_order_mu; +std::chrono::steady_clock::time_point blocked_since; +int64_t st_resumed = 0, st_wake_looks = 0, st_wake_resumes = 0; +std::atomic st_wake_sets{0}; +std::atomic order_wake_event{nullptr}; +static bool met = false, stopped_long = false; +static int hold_ms() { return stopped_long ? -1 : 1000000; } // (past the hold or far from it, whatever the clock says) +static void burst_begin(uint64_t v) { CHECK(v == 0); trace += 'b'; } +static void schedule() { + trace += 's'; + if (met && blocked_n.load()) st_resumed++, blocked_n = 0, trace += '+'; +} +static void burst_end() { trace += 'e'; } +static void update_blocked() { trace += 'u'; } +static void drain(int why) { CHECK(why == 3); trace += 'D'; blocked_n = 0; } +static int turns = 0, turn_now = 0; +static void script(int turn); + +@@ORDER_WAKE_ON@@ +@@SYNC_WATCH@@ +void signal_part(uint64_t gpu, uint32_t value) { +@@SIGNAL@@ +} + +static uint64_t rnd_state = 0x6A09E667F3BCC909ull; +static uint64_t rnd() { + rnd_state ^= rnd_state << 13, rnd_state ^= rnd_state >> 7, rnd_state ^= rnd_state << 17; + return rnd_state; +} +static int signals = 0, signals_while_stopped = 0; +// before each turn of the order thread: the game's threads stop queues, signal events, hold the order's lock +static bool lock_held = false; +static void script(int turn) { + trace += '|'; + if (lock_held) sync_order_mu.unlock(), lock_held = false; + const uint64_t r = rnd(); + if (!blocked_n.load() && r % 3 == 0) blocked_n = 1 + (int)(r >> 8) % 2; // (a queue stops at a wait) + met = (r >> 16) % 3 == 0, stopped_long = (r >> 24) % 11 == 0; + if ((r >> 32) % 2 == 0) { // (the CPU signals an event) + signals++, signals_while_stopped += blocked_n.load() != 0; + signal_part(0x1000 + (uint64_t)turn, (uint32_t)turn); + met = met || (r >> 40) % 2 == 0; + } + if ((r >> 48) % 13 == 0) sync_order_mu.lock(), lock_held = true; // (a submit holds the order: the thread looks next turn) +} + +int main(int argc, char** argv) { + const bool on = argc > 1 && std::string(argv[1]) == "on"; + turns = 20000; + CHECK(order_wake_on() == on); + sync_watch(); + CHECK(turn_now == turns && events_made == 1 && order_wake_event.load() == &the_event); + CHECK((int)ev_cpu_pending.size() == signals && ev_cpu_any.load()); // (every signal queued and marked, on or off) + CHECK((int)ev_cpu_signals.load() == signals); // ((Triforce, 11-22, SYNC-29) and counted, on or off) + if (on) { + CHECK(sleeps == 0 && event_waits == turns); + CHECK(sets == signals_while_stopped && sets > 100 && st_wake_sets.load() == sets); // (set exactly while a queue is stopped) + CHECK(sets_locked == 0 && sets_before_store == 0); + CHECK(st_wake_looks > 0 && st_wake_resumes > 0 && st_wake_resumes <= st_wake_looks && st_wake_looks <= sets); + } else { + CHECK(sleeps == turns && event_waits == 0 && sets == 0 && st_wake_sets.load() == 0 && st_wake_looks == 0 && st_wake_resumes == 0); + } + uint64_t h = 1469598103934665603ull; + for (char c : trace) h = (h ^ (uint8_t)c) * 1099511628211ull; + std::printf("decisions %zu %016llx resumed %lld\n", trace.size(), (unsigned long long)h, (long long)st_resumed); + std::printf("%s: %d turns, %d sleeps, %d waits on the event, %d signals (%d while a queue was stopped), %d sets, %lld looks after a wake, %lld let work on\n", + on ? "on" : "unset", turns, sleeps, event_waits, signals, signals_while_stopped, sets, (long long)st_wake_looks, (long long)st_wake_resumes); + return fails != 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + rep = (VK / "vk_replay.cpp").read_text(encoding="utf-8") + + on = block(rep, "bool order_wake_on() {") + check('getenv("RX_VK_ORDER_WAKE")' in on and "static const bool env" in on and "e && atoi(e) != 0" in on and + 'return env || lab_ab::on("orderwake");' in on, "RX_VK_ORDER_WAKE is not read once, off unless set, with the lab knob orderwake") + watch = block(rep, "void sync_watch() {") + order = [watch.find(s) for s in ( + "const HANDLE wake = CreateEventA(nullptr, FALSE, FALSE, nullptr);", "order_wake_event.store(wake, std::memory_order_release);", + "for (;;) {", "if (wake && order_wake_on()) woken = WaitForSingleObject(wake, 1) == WAIT_OBJECT_0;", "else Sleep(1);", + "if (!blocked_n.load()) continue;", "std::unique_lock lk(sync_order_mu, std::try_to_lock);", "if (!lk) continue;", + "burst_begin(0);", "schedule();", "burst_end();", "update_blocked();", + "if (woken) st_wake_looks++, st_wake_resumes += st_resumed != resumed0;", "if (blocked_n.load() && held > hold_ms()) {", "drain(3);")] + check(min(order) >= 0 and order == sorted(order), "sync_watch: the event, the wait or the sleep, the look and the forced drain are not there in that order") + # (the part of event_signal_queued without the replay thread: its first statement, a block) + signal = block(rep, "void event_signal_queued(uint64_t gpu, uint32_t value) {\n if (!async_on()) {").split("\n", 1)[1] + order = [signal.find(s) for s in ( + "ev_cpu_pending.emplace_back(gpu, value);", "ev_cpu_any.store(true, std::memory_order_release);\n }", + "if (blocked_n.load(std::memory_order_relaxed) && order_wake_on())", "SetEvent(wake), st_wake_sets.fetch_add(1, std::memory_order_relaxed);", + "return;")] + check(min(order) >= 0 and order == sorted(order), "event_signal_queued does not wake the order thread after the signal is queued and marked, its lock let go") + for lock in ("sync_order_mu", "replay_mu", "schedule(", "burst_begin(", "drain("): + check(lock not in signal, "event_signal_queued's wake takes the order's lock or replays on the signalling thread (%s)" % lock) + check(rep.count("SetEvent(wake)") == 1 and "SetEvent(wake)" in signal, "the wake is set elsewhere than in event_signal_queued") + + # the hold loop of a game thread's wait: the met waits go on inside its sync_order_mu block, none forced + look = block(rep, "if (ev_cpu_any.load(std::memory_order_acquire) && order_wake_on()) {") + order = [look.find(s) for s in ("burst_begin(0);", "schedule();", "burst_end();", "update_blocked();", + "st_wake_looks++, st_wake_resumes += st_resumed != resumed0;")] + check(min(order) >= 0 and order == sorted(order) and "drain(" not in look, "the hold loop's look is not sync_settle's four calls, or forces a wait") + at = rep.index(look) + before, after = rep[max(0, at - 420):at], rep[at + len(look):at + len(look) + 200] + check("std::lock_guard lk(sync_order_mu);" in before and before.rindex("std::lock_guard lk(sync_order_mu);") > before.rfind("}"), + "the hold loop's look is not inside its sync_order_mu block") + check(after.lstrip().startswith("needs = oldest_unfinished() <= f;"), "the hold loop no longer asks what is left right after its look") + settle = block(rep, "void sync_settle() {") + for s in ("burst_begin(0);", "schedule();", "burst_end();", "update_blocked();"): + check(s in settle, "sync_settle changed: the hold loop's look copies it") + stats = block(rep, "void log_order_stats() {") + check("st_wake_sets.load(std::memory_order_relaxed)" in stats and "st_wake_resumes" in stats, "the order line does not count the wakes") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`RX_VK_ORDER_WAKE=1`" in dev and "`orderwake`" in dev, "docs/development.md: RX_VK_ORDER_WAKE=1 is not in the switches table") + + loop = " for (;;) {\n" + check(watch.count(loop) == 1, "sync_watch has no single endless loop") + bounded = watch.replace(loop, " for (turn_now = 0; turn_now < turns; turn_now++) {\n script(turn_now);\n") + body = signal # (the non-async part of event_signal_queued, as a function's body) + src_text = FIXTURE.replace("@@ORDER_WAKE_ON@@", on).replace("@@SYNC_WATCH@@", bounded).replace("@@SIGNAL@@", body) + base_env = {k: v for k, v in os.environ.items() if not k.startswith("RX_VK_")} + with tempfile.TemporaryDirectory(prefix="order-wake-") as tmp: + src, exe = Path(tmp) / "wake.cpp", Path(tmp) / "wake.exe" + src.write_text(src_text, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Wno-unused-function", "-Wno-unused-parameter", str(src), "-o", str(exe)], check=True) + outs = {} + for name, env in (("unset", base_env), ("on", dict(base_env, RX_VK_ORDER_WAKE="1")), ("0", dict(base_env, RX_VK_ORDER_WAKE="0"))): + r = subprocess.run([str(exe), "on" if name == "on" else "off"], env=env, capture_output=True, text=True, timeout=120) + check(r.returncode == 0, "RX_VK_ORDER_WAKE %s:\n%s" % (name, r.stdout + r.stderr)) + outs[name] = r.stdout.splitlines() + print(outs[name][-1]) + check(outs["unset"][0] == outs["0"][0], "RX_VK_ORDER_WAKE=0 is not the same as unset") + check(outs["unset"][0] == outs["on"][0], "with the wake the order thread's looks are not the ones it made without: %s / %s" + % (outs["unset"][0], outs["on"][0])) + print("PASS: a CPU signal wakes the order thread while a queue is stopped, with no lock held; unset it sleeps as before, and " + "its looks are the same either way") + + +if __name__ == "__main__": + main() diff --git a/tests/test_os_yield.py b/tests/test_os_yield.py new file mode 100644 index 0000000..cbebf1d --- /dev/null +++ b/tests/test_os_yield.py @@ -0,0 +1,307 @@ +"""The game's YieldThread without the processor's yield, HOS_OS_YIELD=0 (host/hos_thread.c os_yield and the YieldThread +HLE; no game). + +(Triforce, 11-22, the performance audit's THREAD-41, after Source A's.) A yield of the game's is answered twice: the +virtual core is handed to a waiting game thread of equal or better priority, and then the processor is given up too +(SwitchToThread, a call into the kernel thousands of times a second). HOS_OS_YIELD=0 drops the second; off unless set. + +Two fixtures, both of code cut out of hos_thread.c unchanged: + +- os_yield, the HLE and sleep_away over counting stand-ins, a process a case (the switch is read once): not set, 1 and + any other number both answers, as before; 0 only the virtual core's; 0 with HOS_VCORES=0 both (the processor's yield + is then the only one); SleepThread(0) keeps both whatever the switch says. +- the wait a yield loop is: the HLE over the real virtual cores (vc_lock to hos_vcore_yield, with tests/ + vcore_spin_test.c's stand-ins for Windows' lock and condition variable) on real threads. A yielder holds the only + core and yields in a loop until a flag is set; the waker must get that core to set it. With HOS_OS_YIELD=0: + a waker as good gets the core from the yield itself and one that is better gets it at the yield after its request (no + wake-up lost: the loop ends, no escape, the processor's yield never called); a waker that is worse is not given the + core by a yield, with the switch or without it (the console's yield), and runs through the virtual cores' escape as + before: the switch changes nothing there. And the case the entry warns of, on Windows: the whole process held to one + logical processor, the waker on a core of its own, so that only the system's time slice can let it run while the + yielder spins: the loop still ends, with the switch at 0 and at 1, and the time it took is printed (not judged: it is + the stall to measure in the game). + + python -I -B tests/test_os_yield.py [--cc clang] +""" +import argparse +import os +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +RUN_ALL_ARGS = ["--cc", "clang"] + +ROOT = Path(__file__).resolve().parents[1] +HLE_MARK = "HLE(_ZN2nn2os11YieldThreadEv) {" + + +def fail(message): + raise SystemExit("FAIL: " + message) + + +def block(source, marker): + start = source.index(marker) + end = source.index("{", start) + 1 + depth = 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[start:end] + + +COUNTS = r''' +#include +#include +#include +typedef struct Ctx Ctx; +#define HLE(sym) void imp_##sym(Ctx *c) +static int vcore_yields, os_yields, vcores, aways, backs, slept; +static void hos_vcore_yield(void) { vcore_yields++; } +static int hos_vcores_on(void) { return vcores; } +static int hos_vcore_away(void) { aways++; return 1; } +static void hos_vcore_back(void) { backs++; } +static int SwitchToThread(void) { os_yields++; return 1; } +static void sleep_for(int64_t ns, int guest) { (void)ns, (void)guest; slept++; } +@OS_YIELD@ +@HLE@ +@SLEEP_AWAY@ +int main(int argc, char **argv) { + vcores = argc > 1 ? atoi(argv[1]) : 1; + for (int i = 0; i < 5; i++) imp__ZN2nn2os11YieldThreadEv(NULL); + printf("yield %d %d\n", vcore_yields, os_yields); + vcore_yields = os_yields = 0; + sleep_away(0, 1), sleep_away(0, 0); /* SleepThread(0), the game's and the host's */ + printf("sleep0 %d %d %d\n", vcore_yields, os_yields, slept + aways + backs); + sleep_away(20000, 1); + printf("sleep %d %d %d %d\n", slept, aways, backs, os_yields); + return 0; +} +''' + +# (HOS_OS_YIELD, virtual cores on) -> "the virtual core's yields, the processor's" of five YieldThread calls +COUNT_CASES = [ + (None, 1, "5 5"), # not set: both, as before + ("1", 1, "5 5"), + ("2", 1, "5 5"), + ("", 1, "5 5"), # (set to nothing: as not set) + ("0", 1, "5 0"), # the virtual core's only + ("0", 0, "5 5"), # no virtual cores: the processor's yield is the only one, kept + (None, 0, "5 5"), +] + +CASES = r''' +/* ---- (Triforce, THREAD-41) the yield loop over the real virtual cores ---- */ +static _Thread_local HosThread *cur; +static int os_yields; +static int SwitchToThread(void) { + __atomic_add_fetch(&os_yields, 1, __ATOMIC_RELAXED); + sched_yield(); + return 1; +} +#define HLE(sym) static void imp_##sym(Ctx *c) +@AWAY@ +@BACK@ +@VYIELD@ +@OS_YIELD@ +@HLE@ + +static HosThread yielder, waker; +static int ready, flag, got_core; +static long loops; +static int64_t ready_at, woke_ns; +static int64_t real_ns(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (int64_t)ts.tv_sec * 1000000000LL + ts.tv_nsec; +} +/* the waiter of the model: it holds a core and yields until the condition (flag) is true */ +static void *yield_loop(void *arg) { + (void)arg; + cur = &yielder; + hos_vcore_back(); + ready_at = real_ns(); + __atomic_store_n(&ready, 1, __ATOMIC_RELEASE); + while (!__atomic_load_n(&flag, __ATOMIC_ACQUIRE)) { + imp__ZN2nn2os11YieldThreadEv(NULL); + loops++; + } + hos_vcore_away(); + return NULL; +} +/* the waker: it needs a core to make the condition true */ +static void *wake(void *arg) { + (void)arg; + cur = &waker; + wait_for(&ready, 1); + hos_vcore_back(); + got_core = waker.vc_held; + __atomic_store_n(&flag, 1, __ATOMIC_RELEASE); + woke_ns = real_ns() - ready_at; /* (from the loop's start until its condition is true) */ + hos_vcore_away(); + return NULL; +} +static int64_t count(int64_t *field) { + AcquireSRWLockExclusive(&vc_lock); + const int64_t v = *field; + ReleaseSRWLockExclusive(&vc_lock); + return v; +} +/* waker_prio against the yielder's 20; waker_mask: 1 the yielder's core, 2 a core of its own */ +static void run(int waker_prio, uint64_t waker_mask) { + cores = 2; + thread_init(&yielder, "yielder", 20, 1); + thread_init(&waker, "waker", waker_prio, waker_mask); + pthread_t a, b; + pthread_create(&a, NULL, yield_loop, NULL); + pthread_create(&b, NULL, wake, NULL); + pthread_join(b, NULL); + pthread_join(a, NULL); /* (the loop ended: the wake-up was not lost) */ + CHECK(flag == 1 && loops > 0); + CHECK(!yielder.vc_held && !waker.vc_held && !yielder.vc_queued && !waker.vc_queued); +} +int main(int argc, char **argv) { + const char *c = argc > 1 ? argv[1] : ""; + const int os = getenv("HOS_OS_YIELD") ? atoi(getenv("HOS_OS_YIELD")) != 0 : 1; + { + pthread_t dog; + pthread_create(&dog, NULL, watchdog, NULL); + pthread_detach(dog); + } + if (!strcmp(c, "as-good") || !strcmp(c, "better")) { + run(!strcmp(c, "better") ? 10 : 20, 1); + CHECK(got_core == 1); /* the waker held the yielder's core when it set the flag */ + CHECK(count(&vc_st.yields) >= 1); /* handed over by a yield */ + CHECK(count(&vc_st.escapes) == 0); /* never by the escape */ + end_state(); + } else if (!strcmp(c, "worse")) { /* HOS_VCORE_ESCAPE_EQ_US small: a yield gives way to equal or better only */ + run(30, 1); + CHECK(got_core == 0); + CHECK(count(&vc_st.yields) == 0 && count(&vc_st.escapes) == 1); + } else if (!strcmp(c, "own-core")) { /* the waker needs no core of the yielder's, only a processor */ + run(20, 2); + CHECK(got_core == 2 && count(&vc_st.escapes) == 0); + end_state(); + printf("WOKE %.1f ms after the loop began, %ld yields\n", woke_ns / 1e6, loops); + } else { + FAIL("unknown case %s", c); + } + if (os) CHECK(__atomic_load_n(&os_yields, __ATOMIC_RELAXED) == loops); /* as before: the processor given up at every yield */ + else CHECK(__atomic_load_n(&os_yields, __ATOMIC_RELAXED) == 0); /* HOS_OS_YIELD=0: never */ + if (failures) return 1; + printf("PASS %s\n", c); + return 0; +} +''' + +FAR = "60000000" # (60 s: no escape, no time slice, unless a case sets one) +# case -> its switches (each also run with HOS_OS_YIELD=1, as before, where the same must hold) +THREAD_CASES = { + "as-good": {}, + "better": {}, + "worse": {"HOS_VCORE_ESCAPE_EQ_US": "30000"}, + "own-core": {}, +} + + +def read_source(thread, dev): + oy = block(thread, "static int os_yield(void) {") + if 'const char *e = getenv("HOS_OS_YIELD");' not in oy or "on = e && *e ? atoi(e) != 0 : 1;" not in oy or "static int on = -1;" not in oy: + fail("os_yield is not HOS_OS_YIELD read once, on unless it is 0") + hle = block(thread, HLE_MARK) + if "hos_vcore_yield();" not in hle or "if (os_yield() || !hos_vcores_on()) SwitchToThread();" not in hle or \ + hle.index("hos_vcore_yield();") > hle.index("SwitchToThread();"): + fail("YieldThread does not hand the virtual core over first, or its processor's yield is not behind the switch") + away = block(thread, "static void sleep_away(int64_t ns, int guest) {") + if "hos_vcore_yield();\n SwitchToThread();" not in away or "os_yield" in away: + fail("SleepThread(0) no longer gives the processor up: only YieldThread is behind the switch") + if "| `HOS_OS_YIELD=0` |" not in dev: + fail("docs/development.md: HOS_OS_YIELD is not in the switches table") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cc", default="clang") + args = parser.parse_args() + thread = (ROOT / "host/hos_thread.c").read_text(encoding="utf-8") + dev = (ROOT / "docs/development.md").read_text(encoding="utf-8") + try: + read_source(thread, dev) + pieces = {"@OS_YIELD@": block(thread, "static int os_yield(void) {"), "@HLE@": block(thread, HLE_MARK), + "@SLEEP_AWAY@": block(thread, "static void sleep_away(int64_t ns, int guest) {"), + "@AWAY@": block(thread, "int hos_vcore_away(void) {"), "@BACK@": block(thread, "void hos_vcore_back(void) {"), + "@VYIELD@": block(thread, "void hos_vcore_yield(void) {")} + cores = thread[thread.index("static SRWLOCK vc_lock = SRWLOCK_INIT;"):thread.index("int hos_vcore_away(void) {")] + spin = (ROOT / "tests/vcore_spin_test.c").read_text(encoding="utf-8") + stand_ins = spin[:spin.index("/* ---- storm: many hand-offs")] # (the lock, the condition variable, the thread, the helpers) + except ValueError as e: + fail("a text this test reads is not there (%s)" % e) + compiler = shutil.which(args.cc) + if not compiler: + raise SystemExit("C compiler not found: " + args.cc) + env = {k: v for k, v in os.environ.items() if not k.startswith("HOS_")} + with tempfile.TemporaryDirectory(prefix="os-yield-") as temporary: + folder = Path(temporary) + + def build(name, text, extra=()): + for mark, piece in pieces.items(): + text = text.replace(mark, piece) + (folder / (name + ".c")).write_text(text, encoding="utf-8") + exe = folder / (name + ".exe") + # (the stand-ins are the spin test's: what its own cases use and these do not is not an error here) + built = subprocess.run([compiler, "-std=gnu17", "-O2", "-Wall", "-Werror", "-Wno-unused-function", "-Wno-unused-variable", + "-Wno-unused-but-set-variable", "-Wno-unknown-warning-option", "-Wno-unused-but-set-global", + "-I" + str(folder), str(folder / (name + ".c")), "-o", str(exe), *extra], + capture_output=True, text=True, timeout=180) + if built.returncode: + print(built.stdout + built.stderr) + fail("the %s fixture did not build" % name) + return exe + + counts = build("counts", COUNTS) + for value, vcores, want in COUNT_CASES: + case_env = dict(env) if value is None else dict(env, HOS_OS_YIELD=value) + out = subprocess.run([str(counts), str(vcores)], env=case_env, capture_output=True, text=True, timeout=60).stdout.split("\n") + what = "HOS_OS_YIELD %s, virtual cores %s" % ("not set" if value is None else repr(value), "on" if vcores else "off") + if out[0] != "yield " + want: + fail("%s: five yields gave %r (the virtual core's, the processor's), not %r" % (what, out[0], want)) + if out[1] != "sleep0 2 2 0": + fail("%s: SleepThread(0) twice gave %r, not both yields each and no sleep" % (what, out[1])) + if out[2] != "sleep 1 1 1 2": + fail("%s: a 20 us sleep gave %r, not one sleep with the core given up and taken back" % (what, out[2])) + print("PASS: five yields, by the switch and the virtual cores: %s" % + ", ".join("%s/%s: %s" % ("unset" if v is None else repr(v), "vcores" if c else "no vcores", w) for v, c, w in COUNT_CASES)) + + (folder / "vcore_block.inc").write_text(cores, encoding="utf-8") + cases = build("cases", stand_ins + CASES, ["-pthread"]) + flags = subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0 + for case, extra in THREAD_CASES.items(): + for value in ("0", "1"): + case_env = dict(env, HOS_VCORE_ESCAPE_US=FAR, HOS_VCORE_ESCAPE_EQ_US=FAR, HOS_VCORE_SLICE_US=FAR, HOS_VCORE_SPIN_US="0", + HOS_OS_YIELD=value) + case_env.update(extra) + done = subprocess.run([str(cases), case], env=case_env, capture_output=True, text=True, timeout=120, creationflags=flags) + if done.returncode or "PASS " + case not in done.stdout: + print(done.stdout[-3000:]) + fail("the yield loop, a waker %s, HOS_OS_YIELD=%s" % (case, value)) + print("PASS: a yield loop and a waker %s: the loop ends with HOS_OS_YIELD=0 and =1" % case) + if os.name == "nt": + # the whole process on one logical processor: only the system's time slice lets the waker run while the yielder spins + for value in ("1", "0"): + case_env = dict(env, HOS_VCORE_ESCAPE_US=FAR, HOS_VCORE_ESCAPE_EQ_US=FAR, HOS_VCORE_SLICE_US=FAR, HOS_VCORE_SPIN_US="0", + HOS_OS_YIELD=value) + done = subprocess.run(["cmd", "/c", "start", "", "/b", "/wait", "/affinity", "1", str(cases), "own-core"], env=case_env, + capture_output=True, text=True, timeout=120, creationflags=flags) + woke = [line for line in done.stdout.splitlines() if line.startswith("WOKE ")] + if "PASS own-core" not in done.stdout or not woke: + print(done.stdout[-3000:] + done.stderr[-1000:]) + fail("on one logical processor the yield loop did not end, HOS_OS_YIELD=%s" % value) + print("one logical processor, HOS_OS_YIELD=%s: the waker set the flag %s" % (value, woke[0][5:])) + else: + print("SKIP: the one-processor case holds the process with Windows' start /affinity") + print("PASS: YieldThread keeps both yields unless HOS_OS_YIELD=0, and a yield loop's waker is never lost without the processor's") + + +if __name__ == "__main__": + main() diff --git a/tests/test_pace_steady.py b/tests/test_pace_steady.py index 10cf056..3aeed0d 100644 --- a/tests/test_pace_steady.py +++ b/tests/test_pace_steady.py @@ -523,7 +523,8 @@ def check_source(): check("if (held) __atomic_store_n(&frame_handed, 1, __ATOMIC_RELAXED);" in block(vi, "static int pace_handed(void) {"), "a frame another limiter held is not noted") presented = block(vi, "void hos_vi_native_presented(void) {") - order = [presented.index(s) for s in ("const int64_t period = steady_period(1000000000LL / fps_max() * interval, now);", + # ((Triforce, 11-22, PRESENT-30) the base is the pace's period: the cap's, since a movie's raise is not made with this row on) + order = [presented.index(s) for s in ("const int64_t period = steady_period(1000000000LL / pace_fps() * interval, now);", "pace_period_ns = period;", "realtime_step(now);", "shown_at[n & 15] = last + period > now")] check(order == sorted(order), "the presents are not paced by the steady period, or the steady step does not see it") api = (ROOT / "gpu/vk/vk_api.h").read_text(encoding="utf-8") diff --git a/tests/test_pace_wait.py b/tests/test_pace_wait.py index 84bc028..c83499d 100644 --- a/tests/test_pace_wait.py +++ b/tests/test_pace_wait.py @@ -123,7 +123,10 @@ def check_sources(): if "if (!SwitchToThread()) YieldProcessor();" not in block(thread, "static void yield_for_wait(void) {"): raise SystemExit("FAIL: host/hos_thread.c: the wait's end is not yielded as a short sleep yields") # (sleep_ns takes the timer from a millisecond on: the timer's part of the wait is never shorter) - if "if (ns < 1000000 && !short_timer) {" not in block(thread, "static void sleep_ns(int64_t ns) {"): + # ((11-21, OS-31) the body is sleep_for's and the rule short_sleep.h's: under PACE_WAIT_TIMER_MIN_NS it may yield) + if "if (short_sleep_yields(ns, how, guest)) {" not in block(thread, "static void sleep_for(int64_t ns, int guest) {") or \ + "return ns < 1000000 && how <" not in (ROOT / "host" / "short_sleep.h").read_text(encoding="utf-8") or \ + "static void sleep_ns(int64_t ns) { sleep_for(ns, 0); }" not in thread: raise SystemExit("FAIL: host/hos_thread.c sleep_ns no longer takes the timer from 1 ms on (PACE_WAIT_TIMER_MIN_NS)") if "void hos_sleep_until_ns(int64_t until, int64_t margin_ns);" not in hos: raise SystemExit("FAIL: host/hos.h does not declare hos_sleep_until_ns") diff --git a/tests/test_page_table_stream.py b/tests/test_page_table_stream.py new file mode 100644 index 0000000..39c6945 --- /dev/null +++ b/tests/test_page_table_stream.py @@ -0,0 +1,101 @@ +"""The guest page table's changes of a recording streamed into transfers of at most 64 KiB, without a pair for every page +and a sort (gpu/vk/table_stream.h, vk_memory.cpp memory_table_flush; no game, no GPU). + +(Triforce, 11-22, the performance audit's SYNC-04, after Source A's.) The flush made a (page, value) pair for every page +of every change of the recording, sorted them and kept the last of each page: 16 bytes a page and a sort on the recording +thread, for ranges of hundreds of MB when a pool is imported ahead or merged. + +tests/page_table_stream_test.cpp gets vk_memory.cpp's own table code (TableRun to memory_table_flush, the span +tests/test_parallel_fixes.py cuts) and runs the flush both ways on the same changes, the streamed one and the sort as it +was (table_flush_sorted, RX_VK_TABLE_STREAM=0's): + - 50 trials of 128 random runs that overlap in any order, 50 of runs in page order and apart (the path without the + sweep), the same with two swapped and one repeated, 256 single pages, the empty cases: the two write the SAME + transfers, entry for entry and in the same order; the table they leave is the last writer's of every page; no + transfer is over 64 KiB; pages come ascending, none twice; + - four overlapping ranges over 150,000 pages: the streamed flush's largest allocation is at most 128 KiB, the sorted + one's is over 2 MB (so the count means something), and with RX_VK_TABLE_STREAM=0 memory_table_flush is the sorted one. + +And reads the source: the switch, read once and on unless 0; the old flush kept whole; the switches table. + + python -I -B tests/test_page_table_stream.py [--cxx clang++] +""" +import argparse +import os +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + mem = (VK / "vk_memory.cpp").read_text(encoding="utf-8") + check((VK / "table_stream.h").exists(), "gpu/vk/table_stream.h is missing") + check('#include "table_stream.h"' in mem, "vk_memory.cpp does not include table_stream.h") + start = mem.index("struct TableRun {") + flush_at = mem.index("bool memory_table_flush() {", start) + table = mem[start:mem.index("\n}\n", flush_at) + 3] + flush = table[table.index("bool memory_table_flush() {"):] + + check("bool table_flush_sorted() {" in table and table.index("bool table_flush_sorted() {") < table.index("bool memory_table_flush() {"), + "the flush as it was is not kept, before memory_table_flush, in the span the tests cut") + order = [flush.find(s) for s in ( + "table_defer = false;", "if (table_pending.empty()) return false;", "if (!table_stream_on()) return table_flush_sorted();", + "VkCommandBuffer cmd = upload_command_buffer();", 'GprofSpan gp(cmd, "ptable update (guest page table)");', "gp.items = 0;", + "static thread_local table_stream::Scratch scratch;", + "table_stream::stream(table_pending.data(), table_pending.size(), scratch,", + "vkCmdUpdateBuffer(cmd, guest_pages_buffer, first * 8, n * 8, values), gp.items++, gp.bytes += n * 8;", + "table_pending.clear();", "if (ms > 1.0 &&", "return true;")] + check(min(order) >= 0 and order == sorted(order), "memory_table_flush: the switch, the span, the stream and its counts are not there in that order") + check("stable_sort" not in flush and "push_back" not in flush, "memory_table_flush still expands or sorts the pages itself") + old = table[table.index("bool table_flush_sorted() {"):table.index("bool memory_table_flush() {")] + for s in ("for (uint64_t i = 0; i < r.count; i++) pages.push_back({r.first + i, r.address ? r.address + (i << 12) : 0});", + "table_pending.clear();", "std::stable_sort(pages.begin(), pages.end(), [](const auto& a, const auto& b) { return a.first < b.first; });", + "if (i + 1 < pages.size() && pages[i + 1].first == pages[i].first) continue;", + "if (!run.empty() && (pages[i].first != run_first + run.size() || run.size() == 8192)) emit();"): + check(s in old, "the flush as it was lost %r" % s) + on = table[table.index("bool table_stream_on() {"):table.index("bool table_flush_sorted() {")] + check('getenv("RX_VK_TABLE_STREAM")' in on and "!(e && *e && atoi(e) == 0)" in on and "static const bool on" in on, + "RX_VK_TABLE_STREAM is not on unless set to 0, or is read more than once") + check("RX_VK_TABLE_NOW" in table, "RX_VK_TABLE_NOW is gone") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`RX_VK_TABLE_STREAM=0`" in dev, "docs/development.md: RX_VK_TABLE_STREAM=0 is not in the switches table") + + env = {k: v for k, v in os.environ.items() if not k.startswith("RX_VK_")} + with tempfile.TemporaryDirectory(prefix="page-table-stream-") as tmp: + src, exe = Path(tmp) / "stream.cpp", Path(tmp) / "stream.exe" + src.write_text((ROOT / "tests" / "page_table_stream_test.cpp").read_text(encoding="utf-8").replace("/* ACTUAL */", table), encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O2", "-Wall", "-Wextra", "-I" + str(VK), str(src), "-o", str(exe)], check=True) + + def case(name, extra=None): + r = subprocess.run([str(exe), name], env=dict(env, **(extra or {})), capture_output=True, text=True, timeout=300) + check(r.returncode == 0, "%s:\n%s" % (name, r.stdout + r.stderr)) + return r.stdout.strip().splitlines() + + print(case("random")[-1]) + largest = {} + for name, extra in (("four", None), ("four-sorted", None), ("four", {"RX_VK_TABLE_STREAM": "0"})): + line = [l for l in case(name, extra) if l.startswith("largest ")] + check(len(line) == 1, "%s did not say its largest allocation" % name) + largest[(name, bool(extra))] = int(line[0].split()[1]) + streamed, sorted_, off = largest[("four", False)], largest[("four-sorted", False)], largest[("four", True)] + print("four ranges over 150,000 pages: the largest allocation %d bytes streamed, %d sorted, %d with RX_VK_TABLE_STREAM=0" + % (streamed, sorted_, off)) + check(0 < streamed <= 128 << 10, "the streamed flush allocated %d bytes at once: over 128 KiB" % streamed) + check(sorted_ > 2 << 20, "the sorted flush's allocation was not counted (%d bytes): the limit above means nothing" % sorted_) + check(off > 2 << 20, "RX_VK_TABLE_STREAM=0 does not bring the sorted flush back") + print("PASS: the page table's changes are streamed into the transfers the sort made, in 64 KiB of scratch") + + +if __name__ == "__main__": + main() diff --git a/tests/test_parallel_fixes.py b/tests/test_parallel_fixes.py index 2cecfc1..7687b27 100644 --- a/tests/test_parallel_fixes.py +++ b/tests/test_parallel_fixes.py @@ -211,7 +211,7 @@ def main(): build_and_run(args.cxx, tmp, "dry_copies", fixture, ["t2t", "t2t-unmarked", "clear", "clear-unmarked", "clear-drawn", "b2t", "b2t-unmarked", "skips", "bc1", "b2t-cpu"]) fixture = (ROOT / "tests/parallel_table_order_test.cpp").read_text(encoding="utf-8").replace("/* ACTUAL */", table) - build_and_run(args.cxx, tmp, "table_order", fixture, ["segments", "segments-unreconciled", "serial", "direct"]) + build_and_run(args.cxx, tmp, "table_order", fixture, ["segments", "segments-unreconciled", "serial", "direct"], ("-I", str(VK))) lines = build_and_run(args.cxx, tmp, "touch_quick", (ROOT / "tests/parallel_touch_quick_test.cpp").read_text(encoding="utf-8"), [None], ("-Werror", "-I", str(VK))) for line in lines[:-1]: diff --git a/tests/test_pass_hazard.py b/tests/test_pass_hazard.py index 8ccdcd1..9d6c3e4 100644 --- a/tests/test_pass_hazard.py +++ b/tests/test_pass_hazard.py @@ -27,18 +27,18 @@ def main(): a = p.parse_args() source = (ROOT / "gpu" / "vk" / "vk_draw.cpp").read_text(encoding="utf-8") haz = block(source, "uint16_t draw_hazards(") - if "for (const Texture* t : pass().sampled)" in haz: + if "for (const Texture* t : pass().sampled)" in haz or "for (const Texture* t : ps.sampled)" in haz: raise SystemExit("a pass hazard walks every texture sampled so far") if "writes_sampled_target(" not in haz: raise SystemExit("a pass hazard ignores the targets earlier draws sampled") if haz.index("gfx_sampled") > haz.index("writes_sampled_target("): raise SystemExit("a draw's own samples are treated as an earlier draw's") begin = block(source, "void begin_pass(") - clear = begin.index("pass().sampled.clear()") + clear = begin.index("ps.sampled.clear()") if "sampled_targets = 0" not in begin[clear:clear + 80]: raise SystemExit("a new pass keeps the targets earlier draws sampled") - ins = source.index("pass().sampled.push_back(t);") - wrote = source.index("pass().written |= writes;", ins) + ins = source.index("ps.sampled.push_back(t);") + wrote = source.index("ps.written |= writes;", ins) if "sampled_targets" not in source[ins:wrote]: raise SystemExit("a sampled target is invisible to the next draw") fn = block(source, "bool writes_sampled_target(") diff --git a/tests/test_pcache_bound.py b/tests/test_pcache_bound.py index 17bbe68..082083e 100644 --- a/tests/test_pcache_bound.py +++ b/tests/test_pcache_bound.py @@ -95,7 +95,7 @@ def check_source(): draw = (ROOT / "gpu/vk/vk_draw.cpp").read_text(encoding="utf-8") init = between(draw, "void pcache_init() {", "\n}\n") setb = 'pcache_bound_bytes = pcache_bound::bytes(getenv("RX_VK_PCACHE_MAX_MB"), ram);' - readb = "pcache_bound::read(pcache_path.c_str(), 32, pcache_bound_bytes, data, &len)" + readb = "pcache_bound::read(pcache_path.c_str(), 32, pcache_bound_bytes, data, &len," # ((Triforce) and the header check: test_pcache_head_read.py) if setb not in init or readb not in init or init.index(setb) > init.index(readb): raise SystemExit("pcache_init does not read the file through the bound, or reads before the bound is set") if "fread(" in init[init.index(setb):]: diff --git a/tests/test_pcache_carry.py b/tests/test_pcache_carry.py index ccd49cb..54fb36f 100644 --- a/tests/test_pcache_carry.py +++ b/tests/test_pcache_carry.py @@ -321,9 +321,10 @@ def check_source(): "pcache_bound::carried(pcache_same,", "if (resumed && carry_refused) {", "} else if (!pcache_same && !pcache_carried && !carried) {", - "pcache_bound::read(pcache_path.c_str(), 32, pcache_bound_bytes, data, &len)", - # (the device's header is checked on a carried file as on any: another device's or driver's data is dropped) - "dev.props.pipelineCacheUUID", + "pcache_bound::read(pcache_path.c_str(), 32, pcache_bound_bytes, data, &len,", + # (the device's header is checked on a carried file as on any: another device's or driver's data is dropped; its + # header before the rest is read, test_pcache_head_read.py) + "!pcache_head_ok(data.data(), data.size())", "data.clear();", "vkCreatePipelineCache(dev.device, &ci, nullptr, &pcache)", "if (!with_data) pcache_carried = false;", @@ -462,7 +463,7 @@ def check_source(): if "seed_compute" in kept or "seed_program" in kept or "translate" in kept.replace("nothing translated", ""): raise SystemExit("warm_kept_computes translates again") build = between(draw, "bool prepare_compute_pipeline(Program& p) {", "\n}\n") - need("vk_draw.cpp prepare_compute_pipeline", build, ["pin_subgroup(ci.stage, p.workgroup[0]);", + need("vk_draw.cpp prepare_compute_pipeline", build, ["pin_compute(ci.stage, p);", "guarded_compute(pcache, &ci, &pl)", 'if (getenv("RX_VK_NOOPT")) ci.flags |= VK_PIPELINE_CREATE_DISABLE_OPTIMIZATION_BIT;']) rebuild = between(draw, "bool pcache_carry_rebuild() {", "\n}\n") diff --git a/tests/test_pcache_gate.py b/tests/test_pcache_gate.py index 2aea253..3251e30 100644 --- a/tests/test_pcache_gate.py +++ b/tests/test_pcache_gate.py @@ -123,8 +123,12 @@ def check_source(): # ((Triforce) cached_graphics: the guard's creation through the play cache, tests/test_pcache_play.py) if "const PcacheUse use(pcache_door, wanted);" not in build or build.count("cached_graphics(cache, &ci, &pl)") != 2: raise SystemExit("build_graphics does not hold the gate over its creation and its retry") - if "return vkCreateComputePipelines(dev.device, pcache_use().cache, 1, ci, nullptr, out);" not in \ - between(draw, "VkResult shared_cache_compute(", "\n}\n"): + # ((Triforce, 11-22, the performance audit's PIPE-95) through the play cache as a dispatch's, under the gate, held in a + # variable over the creation; the bare creation of RX_VK_PCACHE_OWN_PASSES=0 takes the same gate's cache + shared = between(draw, "VkResult shared_cache_compute(const VkComputePipelineCreateInfo* ci, VkPipeline* out) {", "\n}\n") + if "const auto use = pcache_use();" not in shared or "return cached_compute(use.cache, ci, out);" not in shared or \ + shared.index("const auto use = pcache_use();") > shared.index("vkCreateComputePipelines(") or \ + "vkCreateComputePipelines(dev.device, use.cache, 1, ci, nullptr, out);" not in shared or "pcache," in shared: raise SystemExit("the other files' compute pipelines are made past the gate") for name in ("vk_aa.cpp", "vk_fsr3.cpp"): text = (ROOT / "gpu/vk" / name).read_text(encoding="utf-8") diff --git a/tests/test_pcache_head_read.py b/tests/test_pcache_head_read.py new file mode 100644 index 0000000..02e14ab --- /dev/null +++ b/tests/test_pcache_head_read.py @@ -0,0 +1,173 @@ +"""The pipeline cache's files: the 32-byte header checked before the rest is read (gpu/vk/pcache_bound.h read's head_ok, +vk_draw.cpp pcache_head_ok, pcache_init and pcache_play_open; no game/GPU). + +The file's stamp names only the translated shaders, so after a driver update or with another GPU it still matched, and the +whole file (0.65-1.6 GB) was read into memory only for its header to be refused. pcache_bound.h is compiled as it is, with a +fread that counts the bytes it reads, and pcache_head_ok is cut out of vk_draw.cpp and compiled with a stand-in device. +Checked: another vendor, device or pipelineCacheUUID, another header version, a header size under 32 or past the file's end +give Foreign with nothing kept and 32 bytes read; a matching file comes back byte for byte, whole; short, missing and over-bound +files answer as before with nothing read; with no check the read is as before. Then the source: both of pcache_init's and +pcache_play_open's reads pass the check, and the data read is still checked before Vulkan sees it. + + python -I -B tests/test_pcache_head_read.py [--cxx clang++] +""" +import argparse +import os +from pathlib import Path +import subprocess +import tempfile + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def fail(message): + raise SystemExit("FAIL: " + message) + + +FIXTURE = r'''#include +#include +#include +#include +#include +static uint64_t read_bytes = 0; +static size_t counting_fread(void* p, size_t size, size_t n, FILE* f) { + const size_t got = std::fread(p, size, n, f); + read_bytes += got * size; + return got; +} +#define fread counting_fread +#include "pcache_bound.h" +#undef fread +constexpr uint32_t VK_PIPELINE_CACHE_HEADER_VERSION_ONE = 1; +constexpr uint32_t VK_UUID_SIZE = 16; +struct { struct { uint32_t vendorID = 0x10de, deviceID = 0x2b85; uint8_t pipelineCacheUUID[16]; } props; } dev; +@HEAD_OK@ +#define CHECK(x) do { if (!(x)) { std::printf("line %d: %s\n", __LINE__, #x); return 1; } } while (0) +using namespace pcache_bound; +static std::vector cache(size_t n, uint32_t head_size, uint32_t version, uint32_t vendor, uint32_t device, int uuid_off) { + std::vector d(n); + for (size_t i = 0; i < n; i++) d[i] = (uint8_t)(i * 13 + 5); + std::memcpy(d.data(), &head_size, 4); + std::memcpy(d.data() + 4, &version, 4); + std::memcpy(d.data() + 8, &vendor, 4); + std::memcpy(d.data() + 12, &device, 4); + std::memcpy(d.data() + 16, dev.props.pipelineCacheUUID, 16); + if (uuid_off >= 0) d[16 + uuid_off] ^= 0x5a; + return d; +} +static bool put(const std::string& path, const std::vector& d) { + FILE* f = std::fopen(path.c_str(), "wb"); + if (!f) return false; + const bool ok = std::fwrite(d.data(), 1, d.size(), f) == d.size(); + return std::fclose(f) == 0 && ok; +} +int main(int, char** argv) { + for (int i = 0; i < 16; i++) dev.props.pipelineCacheUUID[i] = (uint8_t)(0xa0 + i); + const std::string dir = argv[1]; + const size_t MB = 1u << 20; + const uint32_t V = dev.props.vendorID, D = dev.props.deviceID; + std::vector out{9, 9}; + uint64_t len = 0; + // (this GPU's and driver's: whole, byte for byte, read once) + const auto good = cache(MB + 77, 32, 1, V, D, -1); + CHECK(put(dir + "/good.bin", good)); + read_bytes = 0; + CHECK(read((dir + "/good.bin").c_str(), 32, 0, out, &len, pcache_head_ok) == Read::Ok); + CHECK(out == good && len == good.size() && read_bytes == good.size()); + // (with no check: as before) + read_bytes = 0; + CHECK(read((dir + "/good.bin").c_str(), 32, 0, out, &len) == Read::Ok && out == good && read_bytes == good.size()); + // (another GPU's or driver's: its header read, not a byte more, nothing kept) + const struct { const char* name; std::vector d; } foreign[] = { + {"vendor", cache(MB, 32, 1, 0x1002, D, -1)}, {"device", cache(MB, 32, 1, V, D + 1, -1)}, + {"uuid0", cache(MB, 32, 1, V, D, 0)}, {"uuid15", cache(MB, 32, 1, V, D, 15)}, + {"version", cache(MB, 32, 2, V, D, -1)}, {"head16", cache(MB, 16, 1, V, D, -1)}, + {"headpast", cache(MB, (uint32_t)MB + 1, 1, V, D, -1)}}; + for (const auto& c : foreign) { + CHECK(put(dir + "/" + c.name + ".bin", c.d)); + out = {1, 2, 3}; + read_bytes = 0; + const Read r = read((dir + "/" + c.name + ".bin").c_str(), 32, 0, out, &len, pcache_head_ok); + if (r != Read::Foreign || !out.empty() || read_bytes != 32 || len != c.d.size()) { + std::printf("%s: answer %d, %zu kept, %llu bytes read\n", c.name, (int)r, out.size(), (unsigned long long)read_bytes); + return 1; + } + // (and as before without the check: whole) + CHECK(read((dir + "/" + c.name + ".bin").c_str(), 32, 0, out, &len) == Read::Ok && out == c.d); + } + CHECK(std::string(why(Read::Foreign)).find("another GPU") != std::string::npos); + // (short, missing, over the bound: as before, nothing read) + CHECK(put(dir + "/short.bin", cache(32, 32, 1, V, D, -1))); + read_bytes = 0; + CHECK(read((dir + "/short.bin").c_str(), 32, 0, out, &len, pcache_head_ok) == Read::Short && out.empty() && !read_bytes); + CHECK(read((dir + "/none.bin").c_str(), 32, 0, out, &len, pcache_head_ok) == Read::None && out.empty() && len == 0); + read_bytes = 0; + CHECK(read((dir + "/good.bin").c_str(), 32, MB, out, &len, pcache_head_ok) == Read::OverBound && out.empty() && !read_bytes); + // (a header of 33 bytes and the rest: whole) + const auto small = cache(33, 32, 1, V, D, -1); + CHECK(put(dir + "/small.bin", small)); + CHECK(read((dir + "/small.bin").c_str(), 32, 0, out, &len, pcache_head_ok) == Read::Ok && out == small); + std::printf("OK\n"); + return 0; +} +''' + + +def check_source(draw): + init = block(draw, "void pcache_init() {") + play = block(draw, "void pcache_play_open(bool same) {") + for name, body in (("pcache_init", init), ("pcache_play_open", play)): + reads = body.count("pcache_bound::read(") + if reads != 1: + fail("%s reads its file %d times" % (name, reads)) + call = body[body.index("pcache_bound::read("):] + call = call[:call.index(";")] + if "pcache_head_ok" not in call: + fail("%s reads its file without checking the header first" % name) + if "!pcache_head_ok(data.data(), data.size())" not in body: + fail("%s no longer checks the data read before Vulkan sees it" % name) + if "pcache_bound::Read::Foreign" not in body: + fail("%s does not say when a file is another GPU's or driver's" % name) + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--cxx", default="g++") + a = p.parse_args() + vk = ROOT / "gpu" / "vk" + draw = (vk / "vk_draw.cpp").read_text(encoding="utf-8") + try: + check_source(draw) + text = FIXTURE.replace("@HEAD_OK@", block(draw, "bool pcache_head_ok(const uint8_t* head, uint64_t size) {")) + except ValueError as e: + fail("a text this test reads moved, or is not there (%s)" % e) + with tempfile.TemporaryDirectory(prefix="pcache-head-") as tmp: + cpp = Path(tmp) / "fixture.cpp" + exe = Path(tmp) / ("fixture" + (".exe" if os.name == "nt" else "")) + cpp.write_text(text, encoding="utf-8") + built = subprocess.run([a.cxx, "-std=c++20", "-O1", "-Wall", "-I", str(vk), str(cpp), "-o", str(exe)], + capture_output=True, text=True) + if built.returncode: + print(built.stdout + built.stderr) + fail("the fixture did not build") + done = subprocess.run([str(exe), tmp], capture_output=True, text=True, timeout=60) + if done.returncode or done.stdout.strip() != "OK": + fail("the reads: " + (done.stdout + done.stderr).strip()) + print("PASS: a pipeline cache of another GPU or driver refused after its 32-byte header, a matching one read whole as " + "before, both files' reads checked first") + + +if __name__ == "__main__": + main() diff --git a/tests/test_pcache_libs.py b/tests/test_pcache_libs.py new file mode 100644 index 0000000..a3be6c4 --- /dev/null +++ b/tests/test_pcache_libs.py @@ -0,0 +1,170 @@ +"""A pipeline cache file of stage libraries and compute pipelines alone, on NVIDIA PCs of little RAM (gpu/vk/pcache_libs.h; +vk_draw.cpp pcache_libs_decide, pcache_init, build_graphics, pcache_play_open; no game, no GPU). + +(Triforce, 11-22, the performance audit's PIPE-78, after Source A's and Source B's.) The whole pipeline cache file is read +and held all session on every PC. With RX_VK_PCACHE_LIBS_ONLY=1 (off by default), a fresh file on NVIDIA's own driver with +20 GB of RAM or less and 512 MB or more in the driver's own cache keeps the stage libraries (which a draw's link must find in +the cache it hands over) and the compute pipelines alone, whole graphics pipelines going to the driver's cache, and opens no +play file; recorded in .lean for that file's life. 2 forces it at a fresh file. + +The header is compiled and run: +- the switch (off unless set), and the rule over many made-up PCs: NVIDIA's own driver at 8, 16 and 20 GB with a 512 MB+ + cache lean; 24 and 32 GB (this PC), a small cache, no stage libraries, AMD's, Intel's, Mesa's, NVK, a start that only + prepares: full; 2 lean anywhere but a prepare-only start; 0 never; +- the driver's cache folder on Windows and Linux (its variable, XDG, HOME), and its size walked on a temporary tree; +- the record: written, read for its stamp only; and a start's verdict: fresh by the rule, a file read by its record, off + with the switch at 0 whatever was recorded. +And reads the source: pcache_init decides once the file read (or not) is known, through pcache_libs_decide, which records +or removes .lean only at a fresh file and decides nothing in a start that only prepares; build_graphics leaves whole +pipelines out of our cache when lean (libraries stay); pcache_play_open opens no play file when lean; the switches table has +the row. + + python -I -B tests/test_pcache_libs.py [--cxx clang++] +""" +import argparse +import os +import subprocess +import tempfile +from pathlib import Path + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] + +FIXTURE = r""" +#include +#include +#include "pcache_libs.h" +#define CHECK(x) do { if (!(x)) { std::printf("line %d: %s\n", __LINE__, #x); return 1; } } while (0) +using namespace pcache_libs; +struct Case { const char* what; bool nvidia; uint64_t ram_gb; uint64_t cache_mb; bool gpl; bool prepare_only; bool want1; }; +int main(int argc, char** argv) { + if (argc < 2) return 2; + const std::string dir = argv[1]; + CHECK(mode(nullptr) == 0 && mode("") == 0 && mode("0") == 0 && mode("1") == 1 && mode("2") == 2 && mode("3") == 2 && mode("-1") == 0); + const Case cases[] = { + {"RTX 3060, 16 GB, 1 GB cache", true, 16, 1024, true, false, true}, + {"RTX 3070 Ti, 16 GB, 600 MB cache", true, 16, 600, true, false, true}, + {"GTX 1660, 8 GB, 512 MB cache", true, 8, 512, true, false, true}, + {"RTX 4060, 20 GB, 2 GB cache", true, 20, 2048, true, false, true}, + {"RTX 4060, 24 GB", true, 24, 2048, true, false, false}, + {"this PC: RTX, 32 GB", true, 32, 4096, true, false, false}, + {"RTX 3060, 16 GB, 511 MB cache", true, 16, 511, true, false, false}, + {"RTX 3060, 16 GB, empty cache", true, 16, 0, true, false, false}, + {"RTX 3060, 16 GB, no stage libraries kept", true, 16, 1024, false, false, false}, + {"RTX 3060, 16 GB, a start that only prepares", true, 16, 1024, true, true, false}, + {"RTX 3060, RAM unknown", true, 0, 1024, true, false, false}, + {"RX 6600, 16 GB (AMD)", false, 16, 1024, true, false, false}, + {"Arc A750, 16 GB (Intel)", false, 16, 1024, true, false, false}, + {"Iris Xe, 8 GB (Mesa)", false, 8, 1024, true, false, false}, + {"RTX 3060, NVK, 16 GB", false, 16, 1024, true, false, false}, + }; + for (const Case& c : cases) { + Pc pc; + pc.nvidia_proprietary = c.nvidia, pc.ram = c.ram_gb << 30, pc.glcache = c.cache_mb << 20, pc.gpl = c.gpl, pc.prepare_only = c.prepare_only; + if (choose(1, pc) != c.want1) { std::printf("rule: %s\n", c.what); return 1; } + CHECK(!choose(0, pc)); + CHECK(choose(2, pc) == !c.prepare_only); + } + // the driver's folder + CHECK(nvidia_cache_dir(true, "C:\\Users\\a\\AppData\\Local", nullptr, nullptr, nullptr) == "C:\\Users\\a\\AppData\\Local\\NVIDIA\\GLCache"); + CHECK(nvidia_cache_dir(true, nullptr, "/x", "/y", "/z").empty()); + CHECK(nvidia_cache_dir(false, nullptr, "/cache/gl", "/xdg", "/home/a") == "/cache/gl"); + CHECK(nvidia_cache_dir(false, nullptr, "", "/xdg", "/home/a") == "/xdg/nvidia/GLCache"); + CHECK(nvidia_cache_dir(false, nullptr, nullptr, nullptr, "/home/a") == "/home/a/.cache/nvidia/GLCache"); + CHECK(nvidia_cache_dir(false, nullptr, nullptr, nullptr, nullptr).empty()); + // its size, on a tree + { + const std::string root = dir + "/GLCache"; + std::filesystem::create_directories(root + "/a/b"); + std::ofstream(root + "/one.bin", std::ios::binary) << std::string(1000, 'x'); + std::ofstream(root + "/a/two.bin", std::ios::binary) << std::string(2500, 'y'); + std::ofstream(root + "/a/b/three.bin", std::ios::binary) << std::string(7, 'z'); + CHECK(folder_bytes(root) == 3507 && folder_bytes(dir + "/none") == 0 && folder_bytes("") == 0); + CHECK(folder_bytes(root, 1) <= 3507); + } + // the record + const std::string lean = dir + "/mkw_vk_pipelines.bin.lean"; + const char* A = "0123456789abcdef"; + const char* B = "fedcba9876543210"; + CHECK(!lean_read(lean, A)); + CHECK(lean_write(lean, A) && lean_read(lean, A) && !lean_read(lean, B)); + // a start's verdict + CHECK(lean_now(1, true, true, false) && !lean_now(1, true, false, true)); // (fresh: the rule, whatever was recorded) + CHECK(lean_now(1, false, false, true) && !lean_now(1, false, true, false)); // (a file read: its record) + CHECK(!lean_now(0, true, true, true) && !lean_now(0, false, false, true)); // (the switch at 0: full) + CHECK(lean_now(2, false, false, true) && !lean_now(2, false, true, false)); + std::printf("ok\n"); + return 0; +} +""" + + +def block(text, marker): + start = text.index(marker) + at = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[at] == "{") - (text[at] == "}") + at += 1 + return text[start:at] + + +def need(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def ordered(text, parts, why): + last = 0 + for p in parts: + k = text.find(p, last) + need(k >= 0, why + " (missing, or out of order: " + p + ")") + last = k + len(p) + + +def check_source(): + draw = (ROOT / "gpu/vk/vk_draw.cpp").read_text(encoding="utf-8") + need('#include "pcache_libs.h"' in draw, "vk_draw.cpp does not include pcache_libs.h") + decide = block(draw, "bool pcache_libs_decide(bool fresh, const char* stamp, uint64_t ram) {") + ordered(decide, ['pcache_libs::mode(getenv("RX_VK_PCACHE_LIBS_ONLY"));', "if (!mode) return false;", 'pcache_path + ".lean"', + "if (!fresh || prepare_only_start()) {", "!fresh && pcache_libs::lean_read(lean, stamp);", + "return pcache_libs::lean_now(mode, fresh, false, recorded);", + "pc.nvidia_proprietary = dev.driver_id == VK_DRIVER_ID_NVIDIA_PROPRIETARY;", "pc.gpl = gpl_wanted() && gpl_libs_cached();", + "pcache_libs::folder_bytes(pcache_libs::nvidia_cache_dir(true, getenv(\"LOCALAPPDATA\")", + "const bool on = pcache_libs::choose(mode, pc);", "if (on) pcache_libs::lean_write(lean, stamp);", "else remove(lean.c_str());"], + "pcache_libs_decide does not decide only at a fresh file, by the rule, recording it") + need(decide.count("lean_write(") == 1 and decide.count("remove(") == 1, "the record is written or removed outside a fresh file") + init = block(draw, "void pcache_init() {") + ordered(init, ["bool with_data = !data.empty();", "pcache_side = pcache_carry::session(how, history);", + "pcache_libs_only = pcache_libs_decide(!with_data, pcache_for, ram);", "pcache_play_open(pcache_same);"], + "pcache_init does not decide before the play file is opened, once the file read is known") + play = block(draw, "void pcache_play_open(bool same) {") + need("if (!on || !pcache || !dev.vk13.pipelineCreationCacheControl || pcache_libs_only) {" in play, "a lean file still opens a play file") + build = block(draw, "VkPipeline build_graphics(const Program& p, const NvnState& s, const Targets& at, VkPrimitiveTopology topo, int part, bool* cold) {") + need("const VkPipelineCache wanted = lib_cached || (part < 0 && !driver_cache_only && !pcache_libs_only) ? pcache : VK_NULL_HANDLE;" in build, + "build_graphics does not leave whole pipelines out of our cache when lean, or the libraries are not kept in it") + docs = (ROOT / "docs/development.md").read_text(encoding="utf-8") + need("`RX_VK_PCACHE_LIBS_ONLY=1` or `2`" in docs, "docs/development.md: the switches table has no RX_VK_PCACHE_LIBS_ONLY row") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + with tempfile.TemporaryDirectory(prefix="pcache-libs-") as tmp: + src, exe = Path(tmp) / "fixture.cpp", Path(tmp) / "fixture.exe" + src.write_text(FIXTURE, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(ROOT / "gpu/vk"), str(src), "-o", str(exe)], + check=True) + work = Path(tmp) / "work" + work.mkdir() + r = subprocess.run([str(exe), str(work).replace("\\", "/")], capture_output=True, text=True, timeout=60) + need(r.returncode == 0 and r.stdout.strip() == "ok", "the libraries-only rule: " + (r.stdout.strip() or "exit %d" % r.returncode)) + check_source() + print("PASS: RX_VK_PCACHE_LIBS_ONLY=1 makes a fresh cache file of libraries alone on small-RAM NVIDIA PCs, recorded for its " + "life; unset, every pipeline in the file as before") + + +if __name__ == "__main__": + main() diff --git a/tests/test_pcache_own_passes.py b/tests/test_pcache_own_passes.py new file mode 100644 index 0000000..8dfd59d --- /dev/null +++ b/tests/test_pcache_own_passes.py @@ -0,0 +1,198 @@ +"""The back end's own compute passes made through the play cache (gpu/vk/pcache_play.h own_passes; vk_draw.cpp +shared_cache_compute; no game, no GPU). + +(Triforce, 11-22, the performance audit's PIPE-95, after Source A's.) FSR 3.1's dozen passes, the anti-aliasing's and the +texture converters' were made with a bare vkCreateComputePipelines into the main pipeline cache. That cache's file is +written in play only after 64 MB of growth, so they waited for the exit's save of the whole main file, and an exit that +cut that write short left them to be compiled again at the next start. They now go the way a dispatch's pipeline goes: +cached_compute, under the cache's gate: the main cache asked without compiling, a miss compiled into the play cache, a +driver fault caught. RX_VK_PCACHE_OWN_PASSES=0: the bare creation, as before. + +Compiles the switch's rule (on unless 0), then shared_cache_compute itself, cut out of vk_draw.cpp, over pcache_play.h's +real create() and a stand-in driver with a main cache and a play cache: + - a pass the main cache has comes from it, asked without compiling, and nothing is compiled; + - one it lacks is compiled into the play cache, with the creation's own flags, and found there the next time; + - the gate closed (the cache let go): made with no cache, nothing asked of either; + - no play cache: made into the main cache, as before; + - the gate is held for the whole creation; + - with the switch at 0: one bare creation into the main cache, the play cache never asked. +And reads the source: the callers (vk_fsr3.cpp, vk_aa.cpp, vk_texture.cpp) still make theirs through +shared_cache_compute, and the switches table has the row. + + python -I -B tests/test_pcache_own_passes.py [--cxx clang++] +""" +import argparse +import os +import subprocess +import tempfile +from pathlib import Path + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] + +FIXTURE = r""" +#include "pcache_play.h" +#include +#include +#include +#include +#include +#define CHECK(x) do { if (!(x)) { std::printf("line %d: %s\n", __LINE__, #x); return 1; } } while (0) + +// ---- a stand-in driver: two caches, each a set of the pipelines (by key) it holds +using VkResult = int; +constexpr VkResult VK_SUCCESS = 0, VK_PIPELINE_COMPILE_REQUIRED = 1000297000; +constexpr uint32_t VK_PIPELINE_CREATE_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT = 0x100; +struct Cache { std::set has; int asked = 0; }; +using VkPipelineCache = Cache*; +using VkPipeline = int; +#define VK_NULL_HANDLE 0 +struct VkComputePipelineCreateInfo { uint32_t flags; int key; }; +struct { int device = 0; } dev; +int compiles, bare_calls, guarded_calls, gate_depth, made_inside_gate; +uint32_t last_flags; +static VkResult driver_create(VkPipelineCache cache, const VkComputePipelineCreateInfo* ci, VkPipeline* out) { + made_inside_gate += gate_depth > 0; + last_flags = ci->flags; + if (cache) { + cache->asked++; + if (cache->has.count(ci->key)) return *out = ci->key, VK_SUCCESS; + if (ci->flags & VK_PIPELINE_CREATE_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT) return VK_PIPELINE_COMPILE_REQUIRED; + cache->has.insert(ci->key); + } + compiles++; + return *out = ci->key, VK_SUCCESS; +} +VkResult vkCreateComputePipelines(int, VkPipelineCache cache, uint32_t, const VkComputePipelineCreateInfo* ci, const void*, VkPipeline* out) { + bare_calls++; + return driver_create(cache, ci, out); +} +VkResult guarded_compute(VkPipelineCache cache, const VkComputePipelineCreateInfo* ci, VkPipeline* out) { + guarded_calls++; + return driver_create(cache, ci, out); +} +// ---- vk_draw.cpp's own, as it has them: the caches, the gate (open or let go), the creation through the play cache +Cache main_cache, play_cache; +VkPipelineCache pcache = &main_cache, pcache_play_cache = &play_cache; +std::atomic pcache_preparing{0}; +std::atomic pcache_main_built{0}, st_play_made{0}; +bool gate_open = true; +struct PcacheUse { + VkPipelineCache cache; + PcacheUse() : cache(gate_open ? pcache : nullptr) { gate_depth++; } + ~PcacheUse() { gate_depth--; } +}; +PcacheUse pcache_use() { return PcacheUse(); } +/* CREATE */ +VkResult cached_compute(VkPipelineCache cache, const VkComputePipelineCreateInfo* ci, VkPipeline* pl) { + return cached_create(cache, ci, pl, guarded_compute); +} +/* SHARED */ + +int main(int argc, char** argv) { + const bool off = argc > 1 && argv[1][0] == '0'; + main_cache.has = {1}; + VkPipeline pl = 0; + VkComputePipelineCreateInfo ci{0x40, 1}; + if (off) { // (RX_VK_PCACHE_OWN_PASSES=0: the bare creation into the main cache) + ci.key = 2; + CHECK(shared_cache_compute(&ci, &pl) == VK_SUCCESS && pl == 2 && bare_calls == 1 && guarded_calls == 0 && compiles == 1); + CHECK(main_cache.has.count(2) && play_cache.asked == 0 && last_flags == 0x40 && made_inside_gate == 1 && gate_depth == 0); + gate_open = false, ci.key = 3; + CHECK(shared_cache_compute(&ci, &pl) == VK_SUCCESS && pl == 3 && bare_calls == 2 && !main_cache.has.count(3) && compiles == 2); + std::printf("ok off\n"); + return 0; + } + // a pass the main cache has + CHECK(shared_cache_compute(&ci, &pl) == VK_SUCCESS && pl == 1 && compiles == 0 && bare_calls == 0 && guarded_calls == 1); + CHECK(main_cache.asked == 1 && play_cache.asked == 0 && (last_flags & VK_PIPELINE_CREATE_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT)); + CHECK(made_inside_gate == 1 && gate_depth == 0); + // one it lacks: into the play cache, with the creation's own flags + ci.key = 2; + CHECK(shared_cache_compute(&ci, &pl) == VK_SUCCESS && pl == 2 && compiles == 1 && play_cache.has.count(2) && !main_cache.has.count(2)); + CHECK(last_flags == 0x40 && guarded_calls == 3 && bare_calls == 0 && st_play_made == 1 && made_inside_gate == 3); + // and found there the next time: nothing compiled + CHECK(shared_cache_compute(&ci, &pl) == VK_SUCCESS && pl == 2 && compiles == 1 && play_cache.asked == 2); + // the gate closed: no cache at all + gate_open = false, ci.key = 5; + const int main_asked = main_cache.asked, play_asked = play_cache.asked; + CHECK(shared_cache_compute(&ci, &pl) == VK_SUCCESS && pl == 5 && compiles == 2); + CHECK(main_cache.asked == main_asked && play_cache.asked == play_asked && !main_cache.has.count(5) && !play_cache.has.count(5)); + gate_open = true; + // while a preparation builds: into the main cache + pcache_preparing = 1, ci.key = 6; + CHECK(shared_cache_compute(&ci, &pl) == VK_SUCCESS && main_cache.has.count(6) && !play_cache.has.count(6) && compiles == 3); + pcache_preparing = 0; + // no play cache: into the main cache, as before + pcache_play_cache = nullptr, ci.key = 7; + CHECK(shared_cache_compute(&ci, &pl) == VK_SUCCESS && pl == 7 && main_cache.has.count(7) && compiles == 4 && last_flags == 0x40); + CHECK(bare_calls == 0 && gate_depth == 0); + std::printf("ok on\n"); + return 0; +} +""" + +RULE = r""" +#include "pcache_play.h" +#include +int main() { + using pcache_play::own_passes; + const bool ok = own_passes(nullptr) && own_passes("1") && own_passes("2") && !own_passes("0") && !own_passes("00"); + std::printf(ok ? "ok rule\n" : "FAIL: RX_VK_PCACHE_OWN_PASSES is not on unless 0\n"); + return !ok; +} +""" + + +def block(text, marker): + start = text.index(marker) + at = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[at] == "{") - (text[at] == "}") + at += 1 + return text[start:at] + + +def need(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + draw = (ROOT / "gpu/vk/vk_draw.cpp").read_text(encoding="utf-8") + shared = block(draw, "VkResult shared_cache_compute(const VkComputePipelineCreateInfo* ci, VkPipeline* out) {") + for line in ('static const bool own_passes = pcache_play::own_passes(getenv("RX_VK_PCACHE_OWN_PASSES"));', "const auto use = pcache_use();", + "if (!own_passes) return vkCreateComputePipelines(dev.device, use.cache, 1, ci, nullptr, out);", + "return cached_compute(use.cache, ci, out);"): + need(line in shared, "shared_cache_compute lost: " + line) + need(shared.index("const auto use = pcache_use();") < shared.index("cached_compute("), "the gate is not taken before the creation") + create = block(draw, "VkResult cached_create(VkPipelineCache cache, const Info* ci, VkPipeline* pl, Guarded guarded) {") + for name, text in (("vk_fsr3.cpp", 3), ("vk_aa.cpp", 1), ("vk_texture.cpp", 1)): + src = (ROOT / "gpu/vk" / name).read_text(encoding="utf-8") + need(src.count("shared_cache_compute(&ci, ") >= text, name + " no longer makes its passes through shared_cache_compute") + docs = (ROOT / "docs/development.md").read_text(encoding="utf-8") + need("`RX_VK_PCACHE_OWN_PASSES=0`" in docs, "docs/development.md: the switches table has no RX_VK_PCACHE_OWN_PASSES row") + env = {k: v for k, v in os.environ.items() if not k.startswith("RX_VK_")} + with tempfile.TemporaryDirectory(prefix="pcache-own-passes-") as tmp: + tmp = Path(tmp) + fixture = FIXTURE.replace("/* CREATE */", "template \n" + create).replace("/* SHARED */", shared) + for name, text in (("rule", RULE), ("shared", fixture)): + src, exe = tmp / (name + ".cpp"), tmp / (name + ".exe") + src.write_text(text, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(ROOT / "gpu/vk"), str(src), "-o", str(exe)], + check=True) + for exe, arg, more, want in (("rule", "", {}, "ok rule"), ("shared", "on", {}, "ok on"), + ("shared", "0", {"RX_VK_PCACHE_OWN_PASSES": "0"}, "ok off")): + r = subprocess.run([str(tmp / (exe + ".exe")), arg], capture_output=True, text=True, timeout=60, env={**env, **more}) + need(r.returncode == 0 and r.stdout.strip() == want, "the own passes' creation (%s): %s" % (arg or "rule", r.stdout.strip() or r.returncode)) + print(r.stdout.strip()) + print("PASS: the back end's own compute passes go through the play cache under the gate; RX_VK_PCACHE_OWN_PASSES=0 as before") + + +if __name__ == "__main__": + main() diff --git a/tests/test_pcache_play.py b/tests/test_pcache_play.py index 0a5bd60..55fc456 100644 --- a/tests/test_pcache_play.py +++ b/tests/test_pcache_play.py @@ -218,7 +218,7 @@ def check_source(draw): opened = block(draw, "void pcache_play_open(bool same) {") need("vk_draw.cpp pcache_play_open", opened, ['getenv("RX_VK_PCACHE_PLAY_FILE")', "dev.vk13.pipelineCreationCacheControl", '(pcache_play_path + ".for")', - "program_cache_stamp()", "dev.props.pipelineCacheUUID", "pcache_bound::read(pcache_play_path.c_str()", + "program_cache_stamp()", "!pcache_head_ok(data.data(), data.size())", "pcache_bound::read(pcache_play_path.c_str()", "pcache_play::fold_due(held, fold)", "vkMergePipelineCaches(dev.device, pcache, 1, &made)", "pcache_fold = true;", 'getenv("RX_VK_PCACHE_FOLD_MB")', "pcache_play_cache = made;", "pcache_play_saved = held;"]) if opened.index("pcache_fold = true;") > opened.index("pcache_play_cache = made;"): @@ -253,6 +253,13 @@ def check_source(draw): need("vk_loader.h", loader, ["X(vkMergePipelineCaches)"]) dev = (ROOT / "docs/development.md").read_text(encoding="utf-8") need("docs/development.md", dev, ["RX_VK_PCACHE_PLAY_FILE=0", "RX_VK_PCACHE_FOLD_MB", "mkw_vk_pipelines_play.bin"]) + # ((Triforce, 11-22, the performance audit's PIPE-95) the back end's own compute passes through the play cache too, behind + # RX_VK_PCACHE_OWN_PASSES (tests/test_pcache_own_passes.py runs the function) + shared = block(draw, "VkResult shared_cache_compute(const VkComputePipelineCreateInfo* ci, VkPipeline* out) {") + need("vk_draw.cpp shared_cache_compute", shared, + ['static const bool own_passes = pcache_play::own_passes(getenv("RX_VK_PCACHE_OWN_PASSES"));', + "return cached_compute(use.cache, ci, out);"]) + need("docs/development.md", dev, ["RX_VK_PCACHE_OWN_PASSES=0"]) def main(): diff --git a/tests/test_perf_overlay.py b/tests/test_perf_overlay.py index 368b64d..c4a3322 100644 --- a/tests/test_perf_overlay.py +++ b/tests/test_perf_overlay.py @@ -231,8 +231,9 @@ def check_source(): for fn in ("void dry_mark_written(NvnState& s, bool compute, bool first) {", "bool dry_temporal_dispatch(NvnState& s) {"): check("note_wait(t0, st_shader_wait_us);" in block(draw_c, fn), "the dry pass's translation waits are not counted (%s)" % fn[:fn.index("(")]) # (every place that counts a wait counts it into one of the two: none adds to the waits by itself; every jobs_wait counted) + # ((Triforce, 11-22, PIPE-30) the fourth Shader wait: draw()'s bounded wait for a translating program, RX_VK_TRANSLATE_WAIT_MS) check(draw_c.count("st_wait_us.fetch_add") == 1 and draw_c.count("st_waits.fetch_add") == 1 and "st_waits++" not in draw_c and - draw_c.count("note_wait(t0, st_pipe_wait_us)") == 1 and draw_c.count("note_wait(t0, st_shader_wait_us)") == 3 and + draw_c.count("note_wait(t0, st_pipe_wait_us)") == 1 and draw_c.count("note_wait(t0, st_shader_wait_us)") == 4 and draw_c.count("note_wait(t0, into)") == 1, "a wait is counted outside note_wait") waits = re.findall(r"\n[^\n/]*\bjobs_wait\([^)]*\);[^\n]*\n([^\n]*)", draw_c) check(len(waits) == 4 and all("note_wait(t0, " in nxt for nxt in waits), "a jobs_wait in vk_draw.cpp is not timed into a wait counter: %r" % waits) diff --git a/tests/test_pipe_ahead.py b/tests/test_pipe_ahead.py new file mode 100644 index 0000000..32afb21 --- /dev/null +++ b/tests/test_pipe_ahead.py @@ -0,0 +1,106 @@ +"""Where the pipelines the draws first need stand in the order, against where the draws are: the count the audit asks for +before a builder of whole pipelines just ahead (gpu/vk/pipe_ahead.h; vk_draw.cpp order_met, graphics_pipeline, +draw_left_out_take; no game, no GPU). + +(Triforce, 11-22, the performance audit's PIPE-81, after Source B's: its first step.) In light mode every pipeline of a load +is new at its first draw. The entry would build whole pipelines just ahead of the draws from the order's next places, and +says to count first: each pipeline first drawn, its place ahead of where the draws are (or none), and whether the warm had +linked it; under a third within 1024 places, stop. This is the count alone: nothing is built or kept. + +The header is compiled and run: the window's switch (1024 unset, 0 off, clamped to 4096); where a place stands against the +front (none, within, further, behind; from the order's start before the draws are placed); the entry's rule (a third or +more within the window). And the source: order_met counts under the order's lock before the front moves, only with the +order used and the window on, and those within the window the warm had linked; graphics_pipeline tells it whether the +warm's link was taken; the line every 600 presents says whether a builder would be worth it; the switches table has the row. + + python -I -B tests/test_pipe_ahead.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] + +FIXTURE = r""" +#include +#include "pipe_ahead.h" +#define CHECK(x) do { if (!(x)) { std::printf("line %d: %s\n", __LINE__, #x); return 1; } } while (0) +using namespace pipe_ahead; +int main() { + CHECK(window(nullptr) == 1024 && window("") == 1024 && window("0") == 0 && window("-5") == 0 && window("256") == 256); + CHECK(window("100000") == 4096 && window("1") == 1); + CHECK(where(-1, 500, 1024) == Where::None && where(-1, -1, 1024) == Where::None); + CHECK(where(500, 500, 1024) == Where::Near && where(1524, 500, 1024) == Where::Near && where(1525, 500, 1024) == Where::Far); + CHECK(where(499, 500, 1024) == Where::Behind && where(0, 500, 1024) == Where::Behind); + CHECK(where(0, -1, 1024) == Where::Near && where(1024, -1, 1024) == Where::Near && where(1025, -1, 1024) == Where::Far); + CHECK(where(600, 500, 50) == Where::Far); + CHECK(worth(1, 3) && !worth(1, 4) && worth(34, 100) && !worth(33, 100) && !worth(0, 0) && worth(5, 5)); + std::printf("ok\n"); + return 0; +} +""" + + +def block(text, marker): + start = text.index(marker) + at = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[at] == "{") - (text[at] == "}") + at += 1 + return text[start:at] + + +def need(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def ordered(text, parts, why): + last = 0 + for p in parts: + k = text.find(p, last) + need(k >= 0, why + " (missing, or out of order: " + p + ")") + last = k + len(p) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + with tempfile.TemporaryDirectory(prefix="pipe-ahead-") as tmp: + src, exe = Path(tmp) / "fixture.cpp", Path(tmp) / "fixture.exe" + src.write_text(FIXTURE, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(ROOT / "gpu/vk"), str(src), "-o", str(exe)], + check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True, timeout=60) + need(r.returncode == 0 and r.stdout.strip() == "ok", "the count's rules: " + (r.stdout.strip() or "exit %d" % r.returncode)) + draw = (ROOT / "gpu/vk/vk_draw.cpp").read_text(encoding="utf-8") + need('#include "pipe_ahead.h"' in draw, "vk_draw.cpp does not include pipe_ahead.h") + need('static const int64_t w = pipe_ahead::window(getenv("RX_VK_PIPE_AHEAD"));' in block(draw, "int64_t pipe_ahead_window() {"), + "the window is not RX_VK_PIPE_AHEAD, read once") + om = block(draw, "void order_met(uint64_t key, bool warm_linked = false) {") + ordered(om, ["std::lock_guard lk(pipe_order_mu);", "pipe_order_rec.add(key);", + "if (pipe_order_used.load(std::memory_order_relaxed) && pipe_ahead_window()) {", + "pipe_ahead::where(pipe_order_index.at(key), pipe_order_front.furthest, pipe_ahead_window());", + "st_ahead[(int)w].fetch_add(1,", "if (w == pipe_ahead::Where::Near && warm_linked) st_ahead_warm.fetch_add(1,", + "!pipe_order_front.met(pipe_order_index.at(key))"], + "order_met does not count before the front moves, under the order's lock, with the order used") + gp = block(draw, "PipeEntry* graphics_pipeline(") + ordered(gp, ["VkPipeline linked = warm_link_take(key);", "warm_linked = linked != VK_NULL_HANDLE;", "order_met(key, warm_linked);"], + "graphics_pipeline does not tell the count whether the warm's link was taken") + take = block(draw, "int64_t draw_left_out_take() {") + ordered(take, ["if (pipe_ahead_window()) {", "if (calls >= a_told + 600 && none + placed != a_was) {", + "(RX_VK_PIPE_AHEAD)", "pipe_ahead::worth(within, placed + none)"], + "the count's line is not written every 600 presents while it grows") + need("vkCreateGraphicsPipelines" not in om and "build_graphics(" not in om, "the count builds something") + docs = (ROOT / "docs/development.md").read_text(encoding="utf-8") + need("`RX_VK_PIPE_AHEAD=`" in docs, "docs/development.md: the switches table has no RX_VK_PIPE_AHEAD row") + print("PASS: the pipelines first drawn are counted by their place against where the draws are (RX_VK_PIPE_AHEAD); nothing built") + + +if __name__ == "__main__": + main() diff --git a/tests/test_pipe_depth_budget.py b/tests/test_pipe_depth_budget.py new file mode 100644 index 0000000..d52aa51 --- /dev/null +++ b/tests/test_pipe_depth_budget.py @@ -0,0 +1,260 @@ +"""A frame budget, then a skip, for depth-tested draws into passes drawn once (gpu/vk/pipe_wait.h depth_budget, choose's +depth_new; vk_draw.cpp draw, pipe_depth_budget, pipe_depth_timed_out; no game, no GPU). + +(Triforce, 11-22, the performance audit's PIPE-29, after Source B's.) A draw into a pass drawn once waits for its pipeline +however long it takes; Source B saw an Arc B580 freeze 11-13 s twice on some fifty new pipelines of such passes. With +RX_VK_PIPE_DEPTH_BUDGET=1, a depth-tested draw into such a pass whose shaders write no memory, outside a samples-passed +count and not into the icon capture's kept targets, waits from the frame's pool, then is left out, and its pipeline is left +out at once until it is built. Off by default on every driver: the entry's default for Intel's Windows driver waits for the +owner's word. + +The header is compiled: the switch (off unless set, on any driver); choose() with depth_new waits from the pool and is left +out once it is spent; a writer, a kept pass and depth_new false choose as before, checked against the old choose() over a +grid of every input. Then the draw's miss block is cut out of draw() and run over stand-ins whose pipeline builds after a +set time: + - unset: a depth-tested draw into a new pass waits for its pipeline, as before; + - set: it waits from the pool and is drawn when the pipeline comes in time; when not, it is left out and its entry then + left out at once (no wait) until the set is emptied; a spent pool leaves the next out at once; + - set: a writer, a draw without depth (or its test off), one inside a samples-passed count, and one into targets kept + from an earlier frame wait (or are left out) as before; + - RX_VK_PIPE_SKIP=0 waits for all. +And reads the source for the hooks and the switches table's row. + + python -I -B tests/test_pipe_depth_budget.py [--cxx clang++] +""" +import argparse +import os +import subprocess +import tempfile +from pathlib import Path + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] + +RULE = r""" +#include +#include "pipe_wait.h" +#define CHECK(x) do { if (!(x)) { std::printf("line %d: %s\n", __LINE__, #x); return 1; } } while (0) +using namespace pipe_wait; +// choose() as it was before depth_new +static Choice old_choose(const Rules& r, bool kept, bool writes, bool full, bool held, int64_t pool_ns, int64_t last_ns) { + if (r.always_wait) return {Act::WaitAll, 0, K_NONE}; + if (!kept || writes) { + if (!r.oneshot_budget) return {Act::WaitAll, 0, K_NONE}; + if (pool_ns >= kMinWaitNs) return {Act::Wait, pool_ns, K_ONESHOT}; + return {Act::Skip, 0, K_NONE}; + } + if (!full || held || !r.full_wait) return {Act::Skip, 0, K_NONE}; + if (pool_ns >= kMinWaitNs) return {Act::Wait, pool_ns, K_FULL}; + if (last_ns >= kMinWaitNs) return {Act::Wait, last_ns, K_LAST}; + return {Act::Skip, 0, K_NONE}; +} +static bool same(Choice a, Choice b) { return a.act == b.act && a.ns == b.ns && a.kind == b.kind; } +int main() { + CHECK(!depth_budget(nullptr) && !depth_budget("") && !depth_budget("0") && depth_budget("1") && depth_budget("2")); + const Rules plain; + CHECK(same(choose(plain, false, false, false, false, 50 * kMs, 40 * kMs, true), Choice{Act::Wait, 50 * kMs, K_DEPTH})); + CHECK(same(choose(plain, false, false, false, false, kMs - 1, 40 * kMs, true), Choice{Act::Skip, 0, K_NONE})); // (spent) + // as before wherever depth_new does not apply, over every input + const int64_t pools[] = {0, kMs - 1, kMs, 50 * kMs}; + for (int always = 0; always < 2; always++) + for (int oneshot = 0; oneshot < 2; oneshot++) + for (int fw = 0; fw < 2; fw++) + for (int kept = 0; kept < 2; kept++) + for (int writes = 0; writes < 2; writes++) + for (int full = 0; full < 2; full++) + for (int held = 0; held < 2; held++) + for (int64_t pool : pools) + for (int64_t last : pools) + for (int dn = 0; dn < 2; dn++) { + Rules r; + r.always_wait = always, r.oneshot_budget = oneshot, r.full_wait = fw; + const Choice now = choose(r, kept, writes, full, held, pool, last, dn); + const Choice was = old_choose(r, kept, writes, full, held, pool, last); + const bool applies = dn && !always && !kept && !writes; + if (!applies) CHECK(same(now, was)); + else CHECK(now.kind == K_DEPTH ? pool >= kMs : now.act == Act::Skip); + } + std::printf("ok\n"); + return 0; +} +""" + +FIXTURE = r""" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "pipe_wait.h" +#define CHECK(x) do { if (!(x)) { std::printf("line %d: %s\n", __LINE__, #x); return 1; } } while (0) +#define PROF_EXACT() ((void)0) +struct Program { uint64_t key = 0x1234; bool writes = false; }; +struct PipeEntry { std::atomic ready{false}; }; +struct Targets { bool kept = false; void* depth = nullptr; }; +struct NvnState { struct { bool dt = false; } ds; }; +struct DrawArgs {}; +uint64_t frame_now = 1; +uint64_t texture_frame_no() { return frame_now; } +bool pass_kept(const Targets& at, bool) { return at.kept; } +bool targets_persistent(const Targets& at) { return at.kept; } +bool pipe_skip_new() { return true; } +bool writes_memory(const Program& p) { return p.writes; } +bool full_screen_draw(const NvnState&, const Targets&, const DrawArgs&) { return false; } +bool hold_takes_frame() { return false; } +bool counting = false; +bool occ_counting_now() { return counting; } +const pipe_wait::Rules& pipe_rules() { + static const pipe_wait::Rules r = [] { pipe_wait::Rules x; x.always_wait = getenv("RX_VK_PIPE_SKIP") && atoi(getenv("RX_VK_PIPE_SKIP")) == 0; return x; }(); + return r; +} +pipe_wait::Pool& pipe_pool() { static pipe_wait::Pool p(50 * pipe_wait::kMs); return p; } +pipe_wait::Pool& pipe_last() { static pipe_wait::Pool p(40 * pipe_wait::kMs); return p; } +int64_t st_pending_skips, left; +int64_t st_pipe_new_waits, st_pipe_full_waits, st_pipe_full_held, st_pipe_full_left; +void left_out_pending() { left++; } +void logf(const char*, ...) {} +// (the pipeline: built after finish_ms; a bounded wait hurries it, a full wait runs it out) +int finish_ms = 0, bounded = 0, full_waits = 0; +bool pipe_wait_for(const std::atomic& ready, int64_t ns) { + bounded++; + const int ms = (int)(ns / pipe_wait::kMs); + std::this_thread::sleep_for(std::chrono::milliseconds(std::min(ms, finish_ms))); + if (finish_ms <= ms) const_cast&>(ready).store(true); + return ready.load(); +} +/* SWITCH */ +static std::string run(PipeEntry* pe, Program& p, const Targets& at, const NvnState& s) { + static const bool always_wait = getenv("RX_VK_PIPE_SKIP") && atoi(getenv("RX_VK_PIPE_SKIP")) == 0; + std::atomic st_pipe_wait_us{0}; + DrawArgs d; + auto wait_for = [&](const std::atomic& ready, std::atomic&) { + full_waits++; + const_cast&>(ready).store(true); + }; + const int64_t l0 = left; + [&] { + /* MISS */ + }(); + return left != l0 ? "left" : full_waits ? "waited" : "drawn"; +} +int main(int argc, char** argv) { + const std::string arm = argc > 1 ? argv[1] : ""; + int depth_target = 0; + Targets fresh_depth{false, &depth_target}, kept_depth{true, &depth_target}, no_depth{false, nullptr}; + NvnState dt, no_dt; + dt.ds.dt = true; + Program clean, writer; + writer.writes = true; + const auto go = [&](PipeEntry& e, Program& p, const Targets& at, const NvnState& s) { + e.ready = false, full_waits = 0, bounded = 0; + return run(&e, p, at, s); + }; + PipeEntry a, b, c; + if (arm == "off" || arm == "always") { + finish_ms = 500; + CHECK(go(a, clean, fresh_depth, dt) == "waited" && bounded == 0); + CHECK(go(a, writer, fresh_depth, dt) == "waited" && bounded == 0); + CHECK(st_pipe_depth_left == 0 && st_pipe_depth_waits == 0); + std::printf("ok\n"); + return 0; + } + // set: in time, drawn from the pool + finish_ms = 5; + CHECK(go(a, clean, fresh_depth, dt) == "drawn" && bounded == 1 && st_pipe_depth_waits == 1); + // not in time: left out, then its entry left out at once + finish_ms = 500; + CHECK(go(b, clean, fresh_depth, dt) == "left" && bounded == 1 && st_pipe_depth_left == 1); + frame_now++; + CHECK(go(b, clean, fresh_depth, dt) == "left" && bounded == 0 && st_pipe_depth_left == 2); + // a spent pool leaves another entry out at once + CHECK(go(c, clean, fresh_depth, dt) == "left"); // (this one spends the new frame's 50 ms) + PipeEntry e2; + CHECK(go(e2, clean, fresh_depth, dt) == "left" && bounded == 0); + // emptied: waits again + { + std::lock_guard lk(pipe_depth_mu); + pipe_depth_out.clear(); + } + frame_now++, finish_ms = 2; + CHECK(go(b, clean, fresh_depth, dt) == "drawn" && bounded == 1); + // as before: a writer, no depth target, its test off, a samples-passed count open: waited for + finish_ms = 500; + PipeEntry f; + CHECK(go(f, writer, fresh_depth, dt) == "waited" && bounded == 0); + CHECK(go(f, clean, no_depth, dt) == "waited" && bounded == 0); + CHECK(go(f, clean, fresh_depth, no_dt) == "waited" && bounded == 0); + counting = true; + CHECK(go(f, clean, fresh_depth, dt) == "waited" && bounded == 0); + counting = false; + // kept targets: left out as before (no full-screen draw), no budget spent + CHECK(go(f, clean, kept_depth, dt) == "left" && bounded == 0); + std::printf("ok\n"); + return 0; +} +""" + + +def block(text, marker): + start = text.index(marker) + at = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[at] == "{") - (text[at] == "}") + at += 1 + return text[start:at] + + +def need(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + draw = (ROOT / "gpu/vk/vk_draw.cpp").read_text(encoding="utf-8") + d = block(draw, "void draw(VkCommandBuffer cmd, NvnState& s, const DrawArgs& d) {") + miss = d[d.index("if (!pe->ready.load(std::memory_order_acquire)) {"):d.index("const VkPipeline pl = pe->pipeline;")] + for line in ("const bool depth_new = !always_wait && !kept && pipe_depth_budget() && at.depth && s.ds.dt && !targets_persistent(at) &&", + "!writes_memory(p) && !occ_counting_now();", "const bool depth_out = depth_new && pipe_depth_timed_out(pe);", + ": depth_out ? pipe_wait::Choice{pipe_wait::Act::Skip}", "pipe_last().left(frame), depth_new);", + "if (depth_new) {", "pipe_depth_time_out(pe);"): + need(line in miss, "draw()'s miss block lost: " + line) + switch = "\n".join(block(draw, m) for m in ("bool pipe_depth_budget() {", "bool pipe_depth_timed_out(const void* pe) {", + "void pipe_depth_time_out(const void* pe) {")) + switch = ("std::mutex pipe_depth_mu;\nstd::unordered_set pipe_depth_out;\n" + "std::atomic st_pipe_depth_waits{0}, st_pipe_depth_left{0};\n" + switch) + need("std::unordered_set pipe_depth_out;" in draw and "pipe_depth_out.clear();" in block(draw, "int64_t draw_left_out_take() {"), + "the timed-out entries are not let go at the presents") + need("bool occ_counting_now() { return occ_live && occ_ctr().counting(" in (ROOT / "gpu/vk/vk_replay.cpp").read_text(encoding="utf-8"), + "vk_replay.cpp does not say whether a samples-passed count is open") + docs = (ROOT / "docs/development.md").read_text(encoding="utf-8") + need("`RX_VK_PIPE_DEPTH_BUDGET=1`" in docs, "docs/development.md: the switches table has no RX_VK_PIPE_DEPTH_BUDGET row") + base = {k: v for k, v in os.environ.items() if not k.startswith("RX_VK_")} + with tempfile.TemporaryDirectory(prefix="pipe-depth-") as tmp: + tmp = Path(tmp) + fixture = FIXTURE.replace("/* SWITCH */", switch).replace("/* MISS */", miss) + for name, text, runs in (("rule", RULE, [("", {})]), + ("miss", fixture, [("on", {"RX_VK_PIPE_DEPTH_BUDGET": "1"}), ("off", {}), + ("always", {"RX_VK_PIPE_DEPTH_BUDGET": "1", "RX_VK_PIPE_SKIP": "0"})])): + src, exe = tmp / (name + ".cpp"), tmp / (name + ".exe") + src.write_text(text, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-Wno-unused-parameter", "-Wno-unused-variable", + "-I" + str(ROOT / "gpu/vk"), str(src), "-o", str(exe)], check=True) + for arg, env in runs: + r = subprocess.run([str(exe)] + ([arg] if arg else []), capture_output=True, text=True, timeout=60, env={**base, **env}) + need(r.returncode == 0 and r.stdout.strip() == "ok", "%s %s: %s" % (name, arg, r.stdout.strip() or r.returncode)) + print("PASS: RX_VK_PIPE_DEPTH_BUDGET=1 bounds a depth-tested draw into a new pass by the frame's pool, then leaves it out; " + "unset (every driver), as before") + + +if __name__ == "__main__": + main() diff --git a/tests/test_pipe_order.cpp b/tests/test_pipe_order.cpp index a808d67..41a4f58 100644 --- a/tests/test_pipe_order.cpp +++ b/tests/test_pipe_order.cpp @@ -350,11 +350,53 @@ static void elsewhere() { CHECK(out.size() == 301); // (300 programs' and the shared one, each once) } +// ((Triforce, 11-21) the pipelines' order keeps more than the programs': a file of 4096 < n <= 16384 reads back, the programs' +// reader still refuses it, a pipeline Record keeps up to its own cap and a merge goes up to it +void cap() { + std::vector keys; + for (uint64_t k = 1; k <= 10000; k++) keys.push_back(k); + const auto bytes = po::write(keys); + CHECK(po::read(bytes.data(), bytes.size()) == keys); + CHECK(program_order::read(bytes.data(), bytes.size(), po::kMagic).empty()); + po::Record r; + for (uint64_t k = 1; k <= po::kKeysMax + 10; k++) r.add(k); + CHECK(r.keys.size() == po::kKeysMax); + program_order::Record pr; + for (uint64_t k = 1; k <= po::kKeysMax; k++) pr.add(k); + CHECK(pr.keys.size() == program_order::kKeysMax); + std::vector before; + for (uint64_t k = 100000; k < 100000 + 9000; k++) before.push_back(k); + CHECK(po::merge(keys, before).size() == po::kKeysMax && po::merge(keys, before)[0] == 1); +} + +// ((Triforce, 11-21) soft behind: behind() lets nothing go; a link behind goes first when a new one needs the room +void soft() { + po::Held h(4); + h.soft(true); + std::vector::One> gone; + const uint64_t libs[1] = {7}; + for (int64_t p = 0; p < 4; p++) h.put(100 + (uint64_t)p, p, (int)p, libs, 1, gone); + h.behind(2, gone); + CHECK(gone.empty() && h.size() == 4); + CHECK(h.room(50)); // (full, but 0 and 1 are behind: room) + h.put(200, 50, 50, libs, 1, gone); + CHECK(gone.size() == 1 && gone[0].key == 100 && h.size() == 4); // (the farthest behind went, not the farthest ahead) + po::Held::One o; + CHECK(h.take(101, o) && o.place == 1); // (one behind still taken by a draw that came back) + po::Held hard(4); + std::vector::One> g2; + for (int64_t p = 0; p < 4; p++) hard.put(100 + (uint64_t)p, p, (int)p, libs, 1, g2); + hard.behind(2, g2); + CHECK(g2.size() == 2 && hard.size() == 2); // (not soft: as before) +} + int main() { CHECK(po::on(nullptr) && po::on("1") && !po::on("0") && po::on("00")); - CHECK(po::kMagic != program_order::kMagic && po::kKeysMax == program_order::kKeysMax); + CHECK(po::kMagic != program_order::kMagic && po::kKeysMax == 16384 && program_order::kKeysMax == 4096); CHECK(po::kLinksMax >= 2 * pipe_warm::kLinksMax && po::kLibsMax >= 2 * po::kLinksMax && po::kPerVariant > 0); file(); + cap(); + soft(); indexed(); front(); held(); diff --git a/tests/test_pipe_order.py b/tests/test_pipe_order.py index acc717c..6a30bec 100644 --- a/tests/test_pipe_order.py +++ b/tests/test_pipe_order.py @@ -42,11 +42,11 @@ def check_source(): if "!found->looked.exchange(true)) st_pipe_looked++, order_met(key);" not in found: raise SystemExit("vk_draw.cpp graphics_pipeline: a draw's first need of a pipeline the precompile made is not in the order") take = gp.index("VkPipeline linked = warm_link_take(key);") - met = gp.index("order_met(key); // ((Triforce, wp18)") + met = gp.index("order_met(key, warm_linked); // ((Triforce, wp18)") # ((Triforce, 11-22, PIPE-81) and whether the warm linked it) if not take < met < gp.index("auto job = std::make_shared();"): raise SystemExit("vk_draw.cpp graphics_pipeline: a new pipeline goes into the order before the warm's link for it is " "taken (the draws' move could let it go), or not at all") - om = function(draw, "void order_met(uint64_t key) {") + om = function(draw, "void order_met(uint64_t key, bool warm_linked = false) {") for need in ("if (!pipe_order_on()) return;", "pipe_order_rec.add(key);", "!pipe_order_front.met(pipe_order_index.at(key))", "pipe_order_links.behind(from, gone);", "order_gone_locked(gone, true);"): diff --git a/tests/test_pipe_warm.py b/tests/test_pipe_warm.py index 40e11f6..f8c5b87 100644 --- a/tests/test_pipe_warm.py +++ b/tests/test_pipe_warm.py @@ -56,6 +56,7 @@ FIXTURE = r""" #include #include "pipe_warm.h" #include "pipe_order.h" +#include "pipe_ahead.h" // ((Triforce, 11-22, PIPE-81) order_met counts where first-drawn pipelines stand) namespace pipe_order = mkwvk::pipe_order; // (vk_draw.cpp's code is inside mkwvk) #define CHECK(x) do { if (!(x)) { std::printf("line %d: %s\n", __LINE__, #x); return 1; } } while (0) using VkPipeline = uint64_t; @@ -625,7 +626,8 @@ def check_source(draw, shader): or "readahead_wait_stats(&raw_n, &raw_ns);" not in replay: raise SystemExit("the stall line does not split the dispatches' time, or lacks the waits for the read-ahead") disp_pick = disp.index("const Program* cp = program_for(s, ProgramKind::Compute, types);") - if not disp.index("const auto t_pick = std::chrono::steady_clock::now();") < disp_pick < disp.index("st_disp_pick_ns +=") or \ + # ((11-22, DESC-30) the pick's time is the change of prof_t.exact across it: a clock pair only under prof_full) + if not disp.index("const int64_t x_pick = prof_t.exact;") < disp_pick < disp.index("st_disp_pick_ns +=") < disp.index("prof_t.exact - x_pick") or \ not disp.index("pipe_clock{") < disp.index("ensure_layout(p);") < disp.index("pipe_clock.stop();") < disp.index("end_pass(cmd);"): raise SystemExit("vk_draw.cpp dispatch: the program pick or the first layout and pipeline are not what is timed") for need in ("`RX_VK_PIPELINE_WARM=0` or `1`", "`RX_VK_PIPELINE_WARM_FOLLOW=1`", "`RX_VK_COMPUTE_WARM=1`"): diff --git a/tests/test_pipeline_contrib.py b/tests/test_pipeline_contrib.py new file mode 100644 index 0000000..50c0698 --- /dev/null +++ b/tests/test_pipeline_contrib.py @@ -0,0 +1,148 @@ +"""The pipeline state lists' checked merge and export (nx2/pipeline_states.py --check, --contrib, --export; no game, made-up +lists only). + +(Triforce, 11-22, the performance audit's PIPE-11, after Source A's.) The shipped list is to grow from more than one PC's +play, so a list handed in is checked before it is merged, the merge says what each source added, and a player's list is +packed with a report of what it holds. Not ported: the in-game tour and the menu's export row. + +Checks, each through the tool as a process: + - --check: a sound list passes; a list without the head, of another record size, cut short, or with a record whose counts + or topology are out of range, or with no first stage, is refused with why; point-list records are refused unless + --keep-points; + - --contrib: the positional lists, then contrib/'s *.bin.xz by name, each checked (one not sound is left out whole), each + pipeline once in the order first met, point-list records left out (kept with --keep-points), each source's line with + its records, what it added, its repeats and its note; the same output whatever order the files were written in; + - --export: the list as triforce_pipeline_states.bin.xz (the same bytes) and report.txt with its counts; a list that is + not sound makes no file; + - the plain merge keeps every record, point lists too, as before. + + python -I -B tests/test_pipeline_contrib.py +""" +import importlib.util +import lzma +import os +import struct +import subprocess +import sys +import tempfile +import zipfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +TOOL = ROOT / "nx2" / "pipeline_states.py" + + +def load(): + spec = importlib.util.spec_from_file_location("pipeline_states", TOOL) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +tool = load() + + +def rec(vs, fs=0, topo=3, n=1, samples=1, nvattr=2, nanswers=0, mesh=0): + r = bytearray(tool.SIZE) + struct.pack_into(" +#include +#include + +using namespace mkwvk::precompile_save; +static int checks, fails; +static void want(const char* what, bool ok) { + checks++; + if (!ok) fails++, printf("FAIL %s\n", what); +} + +int main() { + want("30 s by default", after_s(nullptr) == 30 && after_s("") == 30 && kAfterS == 30); + want("0 is never", after_s("0") == 0 && !on(after_s("0"))); + want("a negative number is never", after_s("-5") == 0); + want("a number is taken", after_s("45") == 45 && after_s("1") == 1); + want("clamped to a day", after_s("999999") == 86400 && after_s("86401") == 86400 && after_s("86400") == 86400); + want("on by default", on(after_s(nullptr))); + want("a long build from the seconds on", long_build(30.0, 30) && long_build(1454.0, 30) && !long_build(29.9, 30)); + want("never long with the switch at 0", !long_build(100000.0, 0)); + want("due at 30 s", due(30.0, 30, false, false, false)); + want("not due before", !due(29.9, 30, false, false, false)); + want("not due when the game went on", !due(500.0, 30, true, false, false)); + want("not due after a stop", !due(500.0, 30, false, true, false)); + want("not due in a start that only prepares", !due(500.0, 30, false, false, true)); + want("not due with the switch at 0", !due(500.0, 0, false, false, false)); + want("why: stopped first", why_not(true, true, true, true) == Outcome::Stopped); + want("why: gone on", why_not(true, false, true, true) == Outcome::GoneOn); + want("why: prepare only", why_not(false, false, true, true) == Outcome::PrepareOnly); + want("why: no cache", why_not(false, false, false, false) == Outcome::NoCache && why_not(true, true, true, false) == Outcome::NoCache); + want("the saved line's words", !strcmp(said(Outcome::Saved), "the pipeline cache saved before the title")); + for (Outcome o : {Outcome::Saved, Outcome::Nothing, Outcome::GoneOn, Outcome::Stopped, Outcome::PrepareOnly, Outcome::NoCache}) + want("every outcome says something", said(o)[0] != 0); + want("only the saved one says saved", !strstr(said(Outcome::GoneOn), "saved before the title (") && + strstr(said(Outcome::GoneOn), "not saved") && strstr(said(Outcome::Stopped), "not saved") && + strstr(said(Outcome::PrepareOnly), "not saved")); + want("a preparation without libraries saves its computes", prepare_saves(30, false, false, 12)); + want("not one with libraries (their own save)", !prepare_saves(30, true, false, 12)); + want("not behind the game", !prepare_saves(30, false, true, 12)); + want("not with nothing built", !prepare_saves(30, false, false, 0)); + want("not with the switch at 0", !prepare_saves(0, false, false, 12)); + if (fails) printf("precompile save rule: FAILED (%d of %d checks)\n", fails, checks); + else printf("precompile save rule: all %d checks passed\n", checks); + return fails != 0; +} +''' + +HOOK = r''' +#include "precompile_save.h" +#include +#include +#include +#include +#include +#include +#include + +namespace precompile_save = mkwvk::precompile_save; +using VkPipelineCache = void*; +#define VK_NULL_HANDLE nullptr +static int one; +VkPipelineCache pcache = &one; +std::atomic pcache_saved_size{0}, pcache_play_saved{0}; +thread_local bool pcache_play_only_now = false; +bool stop = false; +bool prepare_stopped() { return stop; } +bool only_prepares = false; +bool prepare_only_start() { return only_prepares; } // (vk_main.cpp) +std::string line; +int lines = 0; +void logf(const char* f, ...) { + char b[1024]; + va_list ap; + va_start(ap, f); + vsnprintf(b, sizeof b, f, ap); + va_end(ap); + line = b, lines++; +} +int saves = 0; +bool forced = false, marked = false; +size_t next_play = 0, next_main = 0; +void pipeline_cache_save(bool force) { // (what the real one would write: the play file, the main file) + saves++, forced = force, marked = pcache_play_only_now; + if (next_play) pcache_play_saved = next_play; + if (next_main) pcache_saved_size = next_main; +} +/* CUT */ + +static int checks, fails; +static void want(const char* what, bool ok) { + checks++; + if (!ok) fails++, printf("FAIL %s (line: %s)\n", what, line.c_str()); +} +static bool has(const char* s) { return line.find(s) != std::string::npos; } + +int main(int argc, char** argv) { + const std::string mode = argc > 1 ? argv[1] : ""; + if (mode == "off") { // (RX_VK_PRECOMPILE_SAVE_S=0) + precompile_end_save(5000.0, false); + want("with the switch at 0 nothing is saved or said", saves == 0 && lines == 0 && precompile_save_s() == 0); + } else if (mode == "prepare-only") { // (a start that only prepares the shaders) + only_prepares = true; + precompile_end_save(500.0, false); + want("a start that only prepares saves at its own end", saves == 0 && lines == 1 && has("only prepares")); + } else { + want("30 s by default", precompile_save_s() == 30); + precompile_end_save(29.0, false); + want("a short build saves nothing and says nothing", saves == 0 && lines == 0); + precompile_end_save(31.0, true); + want("the game went on: not saved, and said", saves == 0 && lines == 1 && has("31 s of building") && has("went on")); + stop = true; + precompile_end_save(31.0, false); + want("stopped: not saved, and said", saves == 0 && lines == 2 && has("the stop's own save")); + stop = false; + next_play = 5u << 20; + precompile_end_save(1454.0, false); + want("a long build saves once, forced", saves == 1 && forced); + want("under the play-only mark, cleared after", marked && !pcache_play_only_now); + want("and says what it wrote", lines == 3 && has("vk: precompile: 1454 s of building: the pipeline cache saved before the title (5120 KB in ")); + precompile_end_save(40.0, false); + want("nothing new: saved again, nothing written, said so", saves == 2 && lines == 4 && has("nothing new")); + next_main = 7u << 20, next_play = 6u << 20; + precompile_end_save(40.0, false); + want("both files' bytes are counted", saves == 3 && has("(13312 KB in ")); + pcache = nullptr; + precompile_end_save(40.0, false); + want("no cache: not saved, and said", saves == 3 && lines == 6 && has("no pipeline cache")); + } + if (fails) printf("precompile save hook (%s): FAILED (%d of %d checks)\n", mode.c_str(), fails, checks); + else printf("precompile save hook (%s): all %d checks passed\n", mode.c_str(), checks); + return fails != 0; +} +''' + + +def block(text, marker): + start = text.index(marker) + at = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[at] == "{") - (text[at] == "}") + at += 1 + return text[start:at] + + +def need(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def ordered(text, parts, why): + at, last = [], 0 + for p in parts: + k = text.find(p, last) + need(k >= 0, why + " (missing, or out of order: " + p + ")") + at.append(k) + last = k + return at + + +def check_source(draw): + need('#include "precompile_save.h"' in draw, "vk_draw.cpp does not include precompile_save.h") + need('static const int s = precompile_save::after_s(getenv("RX_VK_PRECOMPILE_SAVE_S"));' in block(draw, "int precompile_save_s() {"), + "the switch is not RX_VK_PRECOMPILE_SAVE_S, read once") + end = block(draw, "void precompile_end_save(double build_s, bool gone_on) {") + ordered(end, ["if (!precompile_save::long_build(build_s, save_s)) return;", + "if (!pcache || !precompile_save::due(build_s, save_s, gone_on, stopped, prepare_only)) {", "return;", + "Now() { pcache_play_only_now = true; }", "~Now() { pcache_play_only_now = false; }", "pipeline_cache_save(true);", + "precompile_save::said(precompile_save::Outcome::Saved)"], + "precompile_end_save does not save under the play-only mark only when due") + need("prepare_only = prepare_only_start();" in end and "prepare_stopped()" in end, + "precompile_end_save does not ask whether the start was stopped or only prepares") + run = block(draw, "void precompile_run(void (*progress)(size_t done, size_t total)) {") + ordered(run, ["const auto build_from = std::chrono::steady_clock::now();", "threads.emplace_back(worker);", "checkpoints.end();", + "for (std::thread& t : threads) t.join();", "if (light && !prepare_stopped() && (!building.empty() || !computes_built)) {", + "pipeline_cache_save(true);", "built_write(built_stamp, built);", + "if (!light) precompile_end_save(std::chrono::duration(std::chrono::steady_clock::now() - build_from).count(), gone_on);", + 'logf("vk: precompile: %zu programs, '], + "precompile_run's keep-all end does not save after a long build, or the light branch lost its own save") + save = block(draw, "void pipeline_cache_save(bool force) {") + ordered(save, ["const bool play_only = force && pcache_play_only_now;", "const file_writers::Writing writing;", + "const int64_t main_built = pcache_main_built.load();", "if (play_only && pcache_play_cache) {", "pcache_play_save(0);", + "return;", "if (!pcache_play_save(main_built) && pcache_preparing.load() == 0) return;", + "MoveFileExA(tmp.c_str(), pcache_path.c_str(),"], + "pipeline_cache_save does not write the play file alone under the play-only mark, before the main file's turn") + need("thread_local bool pcache_play_only_now = false;" in draw, "the play-only mark is not a thread's own") + compute = block(draw, "bool prepare_compute_pipeline(Program& p) {") + ordered(compute, ["built = guarded_compute(pcache, &ci, &pl) == VK_SUCCESS && pl;", "if (built) {", "pcache_built_since_save++;", + "if (precompile_save::on(precompile_save_s())) pcache_main_built++;"], + "prepare_compute_pipeline does not count its builds for the main file's save, behind the switch") + for name in ("bool fallback_compute_pipelines(Program& p) {", "VkResult shared_cache_compute("): + if name in draw: + need("pcache_main_built++" not in block(draw, name), "the mesh fallback's or the shared passes' pipelines are counted for the main file here") + prep = block(draw, "bool prepare_run(void (*progress)(size_t done, size_t total), bool background) {") + tail = prep[prep.rindex("if (!background) warm_end(warming);"):] + ordered(tail, ["if (precompile_save::prepare_saves(precompile_save_s(), warm, background, libs.load())) pcache_saved_now();", + "prepared_for.warmed = warmed;", "prepared_write(path, prepared_for);", "return true;"], + "prepare_run's end without stage libraries does not save the cache before its record") + main_cpp = (ROOT / "gpu/vk/vk_main.cpp").read_text(encoding="utf-8") + need("bool prepare_only_start() { return prepare_only(); }" in main_cpp and + "bool prepare_only_start();" in (ROOT / "gpu/vk/vk_internal.h").read_text(encoding="utf-8"), + "vk_main.cpp does not tell vk_draw.cpp whether the start only prepares") + docs = (ROOT / "docs/development.md").read_text(encoding="utf-8") + need("`RX_VK_PRECOMPILE_SAVE_S=`" in docs and "`gpu/vk/precompile_save.h`" in docs, + "docs/development.md: the switches table has no RX_VK_PRECOMPILE_SAVE_S row") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + draw = (ROOT / "gpu/vk/vk_draw.cpp").read_text(encoding="utf-8") + env = {k: v for k, v in os.environ.items() if not k.startswith("RX_VK_")} + with tempfile.TemporaryDirectory(prefix="precompile-save-") as tmp: + def build(name, text): + cpp, exe = Path(tmp) / (name + ".cpp"), Path(tmp) / (name + ".exe") + cpp.write_text(text, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-Wall", "-Wextra", "-Werror", "-I" + str(ROOT / "gpu/vk"), str(cpp), "-o", str(exe)], + check=True) + return exe + + def run(exe, mode, more): + r = subprocess.run([str(exe), mode], capture_output=True, text=True, env={**env, **more}) + print(r.stdout.rstrip()) + if r.returncode: + raise SystemExit("FAIL: the precompile save's checks failed (%s)" % (mode or "rule")) + + run(build("rule", RULE), "", {}) + need("int precompile_save_s() {" in draw and "void precompile_end_save(double build_s, bool gone_on) {" in draw, + "vk_draw.cpp has no precompile_save_s and precompile_end_save") + cut = block(draw, "int precompile_save_s() {") + "\n" + block(draw, "void precompile_end_save(double build_s, bool gone_on) {") + hook = build("hook", HOOK.replace("/* CUT */", cut)) + run(hook, "default", {}) + run(hook, "off", {"RX_VK_PRECOMPILE_SAVE_S": "0"}) + run(hook, "prepare-only", {}) + check_source(draw) + print("PASS: a long precompile's end saves the pipeline cache before the title, the play file alone; a preparation without " + "stage libraries counts and saves its compute pipelines; RX_VK_PRECOMPILE_SAVE_S=0 as before") + + +if __name__ == "__main__": + main() diff --git a/tests/test_predict_check.py b/tests/test_predict_check.py new file mode 100644 index 0000000..e4c799f --- /dev/null +++ b/tests/test_predict_check.py @@ -0,0 +1,286 @@ +"""Whether the driver still has the predicted pipelines, at a start whose record says they were all built +(gpu/vk/predict_check.h; vk_draw.cpp predict_check_cold, predict_background, predicted_play_build; no game, no GPU). + +(Triforce, 11-22, the performance audit's PIPE-68, after Source B's and Source A's.) The predicted pipelines are built once +for the driver's own cache and a record says which are done; nothing looked at whether the driver kept them (a size cap, +a cleared folder, a driver that keeps a third). The check samples up to 12 of them at a start whose record says all, +times a build against a second one, and on a cold majority removes the record so that they are built again; cold again +after that, it stops for that driver. + +The header is compiled and run as it is: +- RX_VK_PREDICT_CHECK: 1 unset, 0 off, 2 every start; +- a pipeline is cold by its first build against its own second (4 times and 3 ms more), never by a fixed time, and one + the driver refused does not count; +- a sample is cold when more than half of it is, says nothing under half a sample, and is decided as soon as what is + still to come cannot change it; +- the places sampled are spread over the whole plan, distinct, the same for one seed and other ones for another; +- find(): the records of the wanted notes in an authored notes file, the first of a key met twice, none for another + build's head, and the walk ends at the last wanted key; +- the record of the strikes: written whole, read back, a broken one or another stamp's is no record; +- starts in a row on a driver that loses them every time: the record removed once (Again), then Stop, then not checked + at all (but with 2); on one that lost them once: Again, then warm forgets the strike, and a later loss is Again anew; + another stamp begins anew; +- the counter: over 20 ms, a line at most every 10 s, the plan's programs sorted without repeats. +Then the source: predict_background asks the check only where the record says all and builds everything again when it +says cold; the check builds for the driver's cache alone, twice, only at light frames, not in a start whose predict_run +built them, and removes the record under its lock; build_graphics counts a play build of a predicted program; the +switches table has the row. + + python -I -B tests/test_predict_check.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] + +FIXTURE = r""" +#include +#include +#include "predict_check.h" +#define CHECK(x) do { if (!(x)) { std::printf("line %d: %s\n", __LINE__, #x); return 1; } } while (0) +using namespace predict_check; +static Probe timed(double first, double again, bool made = true) { + Probe p; + p.made = made, p.first_ms = first, p.again_ms = again; + return p; +} +static std::vector sample(int cold_n, int warm_n, int refused = 0) { + std::vector v; + for (int i = 0; i < cold_n; i++) v.push_back(timed(80, 0.6)); + for (int i = 0; i < warm_n; i++) v.push_back(timed(1.1, 0.6)); + for (int i = 0; i < refused; i++) v.push_back(timed(500, 0, false)); + return v; +} +struct Plan { uint64_t prog, variant, vertex_note, output_note; }; +// A start as predict_background makes it: whether it checks, the verdict of the driver now, what it does, the record +static Do start(const std::string& rec, uint64_t stamp, Verdict v, int mode = 1, bool* checked = nullptr) { + const State had = state_read(rec); + if (checked) *checked = runs(mode, had, stamp); + if (!runs(mode, had, stamp)) return Do::Nothing; + int next = -1; + const Do d = decide(v, had, stamp, &next); + if (v == Verdict::Warm && had.found) remove(rec.c_str()); + if (d != Do::Nothing && !state_write(rec, stamp, next)) return Do::Nothing; + return d; +} + +int main(int argc, char** argv) { + if (argc < 2) return 2; + const std::string dir = argv[1]; + // the switch + CHECK(mode(nullptr) == 1 && mode("") == 1 && mode("1") == 1 && mode("0") == 0 && mode("2") == 2 && mode("7") == 2 && mode("-3") == 0); + // one pipeline: cold by its own second build + CHECK(cold(timed(80, 0.6)) && cold(timed(5.5, 0.6)) && !cold(timed(5.3, 0.6))); // (4 x 0.6 + 3 = 5.4) + CHECK(!cold(timed(1.1, 0.6)) && !cold(timed(2.9, 0.0)) && cold(timed(3.1, 0.0))); + CHECK(!cold(timed(40, 12)) && cold(timed(52, 12))); // (a slow driver's warm time: 51) + CHECK(!cold(timed(500, 0, false))); // (refused: says nothing) + // a sample + CHECK(verdict(tally(sample(7, 5))) == Verdict::Cold && verdict(tally(sample(6, 6))) == Verdict::Warm); + CHECK(verdict(tally(sample(0, 12))) == Verdict::Warm && verdict(tally(sample(12, 0))) == Verdict::Cold); + CHECK(verdict(tally(sample(5, 0))) == Verdict::Unknown && verdict(tally(sample(3, 3))) == Verdict::Warm); + CHECK(verdict(tally(sample(4, 2))) == Verdict::Cold && verdict(tally(sample(0, 5, 7))) == Verdict::Unknown); + CHECK(verdict(tally(sample(0, 0))) == Verdict::Unknown); + { + const Tally t = tally(sample(7, 5, 3)); + CHECK(t.got == 12 && t.cold == 7 && t.first_median == 80 && t.again_median == 0.6); + } + // decided: full, or the rest cannot change it; and what it says then is what the full sample would + CHECK(!decided(tally(sample(6, 5))) && decided(tally(sample(7, 0))) && decided(tally(sample(0, 6))) && decided(tally(sample(6, 6)))); + CHECK(!decided(tally(sample(0, 0))) && !decided(tally(sample(6, 0))) && !decided(tally(sample(3, 5))) && !decided(tally(sample(0, 5, 20)))); + for (int c = 0; c <= kSample; c++) + for (int w = 0; c + w <= kSample; w++) { + if (!decided(tally(sample(c, w)))) continue; + const Verdict now = verdict(tally(sample(c, w))); + CHECK(now != Verdict::Unknown); + for (int more = 0; c + w + more <= kSample; more++) // (whatever comes after, cold or warm) + CHECK(verdict(tally(sample(c + more, w))) == now && verdict(tally(sample(c, w + more))) == now); + } + // the places + { + const std::vector a = spots(28000, kLook, 12345), b = spots(28000, kLook, 12345), c = spots(28000, kLook, 12346); + CHECK(a == b && a != c && a.size() == (size_t)kLook); + std::set distinct(a.begin(), a.end()); + CHECK(distinct.size() == a.size() && a.front() < 28000 / kLook && a.back() >= 28000 - 2 * (28000 / kLook) && a.back() < 28000); + CHECK(spots(0, kLook, 5).empty() && spots(7, kLook, 5).size() == 7 && spots(1, kLook, 99) == std::vector{0}); + } + // find(): an authored notes file, records of 64 bytes whose first 8 are the key + { + const std::string notes = dir + "/notes.bin"; + const uint64_t stamp = 0x1122334455667788ull, rec_bytes = 64; + const uint64_t keys[] = {100, 200, 300, 200, 400, 500, 600}; // (200 twice: its first record, 1) + FILE* f = fopen(notes.c_str(), "wb"); + CHECK(f != nullptr); + const uint64_t head[2] = {stamp, rec_bytes}; + fwrite(head, 8, 2, f); + for (const uint64_t k : keys) { + unsigned char rec[64] = {}; + memcpy(rec, &k, 8); + memset(rec + 8, 0xEE, 56); + fwrite(rec, 1, 64, f); + } + fclose(f); + f = fopen(notes.c_str(), "rb"); + auto at = find(f, stamp, rec_bytes, {300, 200, 999, 500}); + CHECK(at.size() == 3 && at[300] == 2 && at[200] == 1 && at[500] == 5 && !at.count(999)); + fclose(f); + f = fopen(notes.c_str(), "rb"); + at = find(f, stamp, rec_bytes, {100, 200}); + CHECK(at.size() == 2 && at[100] == 0 && at[200] == 1 && ftell(f) == 16 + 2 * 64); // (it stopped at the last wanted key) + fclose(f); + f = fopen(notes.c_str(), "rb"); + CHECK(find(f, stamp + 1, rec_bytes, {100}).empty()); // (another build's file) + fclose(f); + f = fopen(notes.c_str(), "rb"); + CHECK(find(f, stamp, 128, {100}).empty() && find(nullptr, stamp, rec_bytes, {100}).empty()); + fclose(f); + f = fopen(notes.c_str(), "rb"); + CHECK(find(f, stamp, rec_bytes, {}).empty() && find(f, stamp, 4, {100}).empty()); + fclose(f); + } + // the record + const std::string rec = dir + "/notes.predicted.check"; + CHECK(!state_read(rec).found); + CHECK(state_write(rec, 0xABCDEF0123456789ull, 1)); + { + const State s = state_read(rec); + CHECK(s.found && s.stamp == 0xABCDEF0123456789ull && s.strikes == 1); + } + { + FILE* f = fopen(rec.c_str(), "wb"); + fputs("lib-check 0000000000000001 0\n", f); // (the libraries' record is not this one) + fclose(f); + CHECK(!state_read(rec).found); + f = fopen(rec.c_str(), "wb"); + fputs("predict-check zz", f); + fclose(f); + CHECK(!state_read(rec).found); + remove(rec.c_str()); + } + // a driver that loses them every time + const uint64_t A = 0x1111, B = 0x2222; + bool checked = false; + CHECK(start(rec, A, Verdict::Cold) == Do::Again && state_read(rec).strikes == 1); + CHECK(start(rec, A, Verdict::Cold, 1, &checked) == Do::Stop && checked && state_read(rec).strikes == 2); + CHECK(stopped(state_read(rec), A) && !stopped(state_read(rec), B)); + CHECK(start(rec, A, Verdict::Cold, 1, &checked) == Do::Nothing && !checked); // (not checked again) + CHECK(start(rec, A, Verdict::Cold, 2, &checked) == Do::Stop && checked); // (but with 2, and still no rebuild) + CHECK(start(rec, A, Verdict::Cold, 0, &checked) == Do::Nothing && !checked); + // another stamp (a new driver, new shaders) begins anew + CHECK(start(rec, B, Verdict::Cold, 1, &checked) == Do::Again && checked && state_read(rec).stamp == B && state_read(rec).strikes == 1); + // a driver that lost them once: warm forgets the strike, and a later loss is built again + CHECK(start(rec, B, Verdict::Warm) == Do::Nothing && !state_read(rec).found); + CHECK(start(rec, B, Verdict::Cold) == Do::Again && state_read(rec).strikes == 1); + CHECK(start(rec, B, Verdict::Unknown) == Do::Nothing && state_read(rec).strikes == 1); // (nothing said: nothing changed) + CHECK(start(rec, B, Verdict::Warm) == Do::Nothing && !state_read(rec).found); + // the counter + CHECK(!slow(20000) && slow(20001) && !slow(0) && slow(3000000)); + CHECK(say(5, 0) && !say(9999, 1) && say(10001, 1) && !say(15000, 10000) && say(20000, 10000)); + { + const std::vector plan = {{30, 1, 0, 0}, {10, 1, 0, 0}, {30, 2, 0, 0}, {20, 1, 0, 0}, {10, 3, 0, 0}}; + const std::vector p = programs(plan); + CHECK(p == (std::vector{10, 20, 30}) && has(p, 20) && has(p, 10) && has(p, 30) && !has(p, 25) && !has({}, 10)); + } + std::printf("ok\n"); + return 0; +} +""" + + +def block(text, marker): + start = text.index(marker) + at = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[at] == "{") - (text[at] == "}") + at += 1 + return text[start:at] + + +def need(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def ordered(text, parts, why): + last = 0 + for p in parts: + k = text.find(p, last) + need(k >= 0, why + " (missing, or out of order: " + p + ")") + last = k + len(p) + + +def check_source(): + draw = (ROOT / "gpu/vk/vk_draw.cpp").read_text(encoding="utf-8") + need('#include "predict_check.h"' in draw, "vk_draw.cpp does not include predict_check.h") + need('static const int mode = predict_check::mode(getenv("RX_VK_PREDICT_CHECK"));' in block(draw, "int predict_check_mode() {"), + "the switch is not RX_VK_PREDICT_CHECK, read once") + bg = block(draw, "void predict_background() {") + ordered(bg, ["std::unordered_set had = predicted_read(pstamp);", "if (had.count(kPredictedAll)) {", + "if (!predict_check_cold(pstamp)) return;", "had.clear();", "predict_plan_keys_publish(plan);", + "groups = predict_stream::group(plan, at, had);", "driver_cache_only = true;", "predicted_done(pstamp, keys[i]);"], + "predict_background does not check where the record says all, or does not build everything again when cold") + need(bg.count("predict_check_cold(") == 1 and bg.index("Sleep(100); // (past the start's own loading)") < bg.index("predict_check_cold("), + "the check does not run on the builder's thread after its delay") + check = block(draw, "bool predict_check_cold(uint64_t pstamp) {") + ordered(check, ["if (!mode) return false;", "if (!plan_read(&plan) || plan.empty()) return false;", "predict_plan_keys_publish(plan);", + 'predicted_path() + ".check"', "if (!predict_check::runs(mode, had, pstamp)) return false;", + "std::lock_guard lk(predicted_mu);", "if (predicted_file) return false;", + "predict_check::spots(plan.size(), (size_t)predict_check::kLook,", + "predict_check::find(f, program_cache_stamp(), sizeof(PipeNote), want);", + "if (predict_check::decided(predict_check::tally(probes))) break;", + "while (!prepare_stopped() && bg_game_in_play()) Sleep(50);", "if (prepare_stopped()) return false;", + "predicted_into(*n, *v, *o, e.prog, e.variant);", "program_cache_borrow(e.prog,", "ensure_layout(p);", + "driver_cache_only = true;", "const VkPipeline first = build_graphics(p, n->s, note_targets(*n), (VkPrimitiveTopology)n->topo);", + "const VkPipeline again = first ? build_graphics(p, n->s, note_targets(*n), (VkPrimitiveTopology)n->topo) : VK_NULL_HANDLE;", + "driver_cache_only = false;", "probe.made = first && again,", + "predict_check::decide(verdict, had, pstamp, &next);", "if (had.found) remove(state_path.c_str());", + "predict_check::state_write(state_path, pstamp, next);", "if (what == predict_check::Do::Stop) {", "return false;", + "std::lock_guard lk(predicted_mu);", "if (predicted_file) fclose(predicted_file), predicted_file = nullptr;", + "remove(predicted_path().c_str());", "return true;"], + "predict_check_cold does not sample for the driver's cache alone at light frames, or does not remove the record only when cold") + need("FAIL_ON_PIPELINE_COMPILE_REQUIRED" not in check and check.count("vkDestroyPipeline(") == 2, + "the check asks with the compile-required flag (a driver answers that only from a cache it is handed), or keeps its pipelines") + need(check.count("return true;") == 1, "the check says cold in more than one place") + build = block(draw, "VkPipeline build_graphics(const Program& p, const NvnState& s, const Targets& at, VkPrimitiveTopology topo, int part, bool* cold) {") + ordered(build, ["st_pipelines++;", "const int64_t build_us =", "st_pipeline_us += build_us;", + "if (!driver_cache_only && pl && predict_check_mode()) predicted_play_build(p.key, build_us);", "return pl;"], + "build_graphics does not count a play build of a predicted program") + count = block(draw, "void predicted_play_build(uint64_t prog, int64_t us) {") + ordered(count, ["if (!predict_check::slow(us)) return;", "predict_plan_keys.load(std::memory_order_acquire);", + "if (!keys || !predict_check::has(*keys, prog)) return;", "st_predicted_slow.fetch_add(1,", + "if (predict_check::say(now, said) && said_ms.compare_exchange_strong(said, now))", 'logf("vk: predicted pipelines: %lld builds in play'], + "predicted_play_build does not count only slow builds of the plan's programs, or logs at every one") + publish = block(draw, "void predict_plan_keys_publish(const std::vector& plan) {") + need("if (!predict_check_mode() || predict_plan_keys.load(std::memory_order_acquire)) return;" in publish and + "new std::vector(predict_check::programs(plan))" in publish and "compare_exchange_strong(none, keys)" in publish, + "the plan's programs are not published once for the counter") + docs = (ROOT / "docs/development.md").read_text(encoding="utf-8") + need("`RX_VK_PREDICT_CHECK=0`, `=2`" in docs and "`gpu/vk/predict_check.h`" in docs, + "docs/development.md: the switches table has no RX_VK_PREDICT_CHECK row") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + with tempfile.TemporaryDirectory(prefix="predict-check-") as tmp: + cpp, exe = Path(tmp) / "fixture.cpp", Path(tmp) / "fixture.exe" + cpp.write_text(FIXTURE, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(ROOT / "gpu/vk"), str(cpp), "-o", str(exe)], + check=True) + work = Path(tmp) / "work" + work.mkdir() + r = subprocess.run([str(exe), str(work).replace("\\", "/")], capture_output=True, text=True, timeout=60) + if r.returncode or r.stdout.strip() != "ok": + raise SystemExit("FAIL: the predicted pipelines' check: " + (r.stdout.strip() or "exit %d" % r.returncode)) + check_source() + print("PASS: the predicted pipelines' check samples the driver's cache, builds again once when cold, then stops; " + "RX_VK_PREDICT_CHECK=0 as before") + + +if __name__ == "__main__": + main() diff --git a/tests/test_predict_stream.py b/tests/test_predict_stream.py new file mode 100644 index 0000000..8122e2c --- /dev/null +++ b/tests/test_predict_stream.py @@ -0,0 +1,147 @@ +"""The predicted builder keeps its plan as record numbers and reads its notes when it builds (gpu/vk/predict_stream.h, +vk_draw.cpp predict_background, vk_predict.cpp; no game, no GPU). + +(Triforce, 11-21, the performance audit's PIPE-83, after Source B's.) predict_background builds the predicted pipelines +the start did not get to, behind the game, resting while it is in play. It held every note of the notes file (~27 KB +each) and a full note of every predicted pipeline until it ended: about 1 GB for much of a session. + +This compiles predict_stream.h alone and, on a notes file of its own: each key's record, a record that is not sane +skipped and still counted, a key twice keeping its first record, another build's head giving nothing; the plan as +programs in the order it first names them, the ones built by earlier starts left out, a pipeline whose note is gone +dropped and a program left with none not listed. + +And reads the source: predict_background reads every note and rebuilds the plan only under the switch, a worker opens +its own handle on the notes file and reads the two notes by their records, predict_rebuild's note is made by +predicted_into (one result), and the switch is in the switches table. + + python -I -B tests/test_predict_stream.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +PROGRAM = r""" +#include +#include +#include +#include "predict_stream.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +struct Rec { uint64_t key; uint32_t ok, pad[5]; }; // (a stand-in note: 32 bytes, its key first) +struct Plan { uint64_t prog, variant, vertex_note, output_note; }; +int main(int argc, char** argv) { + if (argc < 2) return 2; + const uint64_t stamp = 0x1234; + { // records 0..5: keys 10, 11 (not sane), 12, 10 again, 13, 14 + FILE* f = std::fopen(argv[1], "wb"); + const uint64_t head[2] = {stamp, sizeof(Rec)}; + std::fwrite(head, 8, 2, f); + const Rec recs[] = {{10, 1, {}}, {11, 0, {}}, {12, 1, {}}, {10, 1, {}}, {13, 1, {}}, {14, 1, {}}}; + std::fwrite(recs, sizeof(Rec), 6, f); + std::fwrite("half", 1, 4, f); // (a record cut short at the end: not one) + std::fclose(f); + } + Rec rec{}; + FILE* f = std::fopen(argv[1], "rb"); + const auto at = predict_stream::index(f, stamp, sizeof(Rec), &rec, [&] { return rec.ok != 0; }); + std::fclose(f); + CHECK(at.size() == 4); + CHECK(at.count(10) && at.at(10) == 0); // (the first of the two) + CHECK(!at.count(11)); // (not sane: not used) + CHECK(at.count(12) && at.at(12) == 2); // (and still counted: 12 is record 2) + CHECK(at.count(13) && at.at(13) == 4 && at.count(14) && at.at(14) == 5); + f = std::fopen(argv[1], "rb"); + CHECK(predict_stream::index(f, stamp + 1, sizeof(Rec), &rec, [] { return true; }).empty()); // (another build's) + std::fclose(f); + f = std::fopen(argv[1], "rb"); + CHECK(predict_stream::index(f, stamp, sizeof(Rec) + 8, &rec, [] { return true; }).empty()); // (another note's size) + std::fclose(f); + CHECK(predict_stream::index((FILE*)nullptr, stamp, sizeof(Rec), &rec, [] { return true; }).empty()); + + // the plan: programs 7, 5, 9, 7 again, 6 (built by an earlier start), 8 (its only pipeline's note gone) + const std::vector plan = { + {7, 1, 10, 12}, {5, 0, 12, 13}, {9, 2, 14, 10}, {7, 3, 13, 14}, {7, 1, 11, 12}, // (the last: note 11 is gone) + {6, 0, 10, 12}, {8, 0, 10, 99}, {5, 4, 99, 10}, {5, 4, 10, 10}}; + const auto g = predict_stream::group(plan, at, {6}); + CHECK(g.size() == 3); + CHECK(g[0].prog == 7 && g[1].prog == 5 && g[2].prog == 9); // (the plan's order) + CHECK(g[0].entries.size() == 2); + CHECK(g[0].entries[0].variant == 1 && g[0].entries[0].vertex_rec == 0 && g[0].entries[0].output_rec == 2); + CHECK(g[0].entries[1].variant == 3 && g[0].entries[1].vertex_rec == 4 && g[0].entries[1].output_rec == 5); + CHECK(g[1].entries.size() == 2); + CHECK(g[1].entries[0].variant == 0 && g[1].entries[0].vertex_rec == 2 && g[1].entries[0].output_rec == 4); + CHECK(g[1].entries[1].variant == 4 && g[1].entries[1].vertex_rec == 0 && g[1].entries[1].output_rec == 0); + CHECK(g[2].entries.size() == 1 && g[2].entries[0].vertex_rec == 5 && g[2].entries[0].output_rec == 0); + CHECK(predict_stream::group(plan, at, {5, 6, 7, 9}).empty()); // (all built: nothing to do) + CHECK(predict_stream::group(plan, {}, {}).empty()); + CHECK(sizeof(predict_stream::Entry) == 16); + return fails ? 1 : 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + draw = (VK / "vk_draw.cpp").read_text(encoding="utf-8") + start = draw.index("void predict_background() {") + bg = draw[start:draw.index("\n}\n", start)] + check('#include "predict_stream.h"' in draw, "vk_draw.cpp does not use predict_stream.h") + check('static const bool stream = !(getenv("RX_VK_PREDICT_BG_STREAM") && !atoi(getenv("RX_VK_PREDICT_BG_STREAM")));' in bg, + "predict_background has no RX_VK_PREDICT_BG_STREAM switch, on unless it is 0") + # (every note read and the plan made into full notes only with the switch at 0) + check(bg.count("pipe_notes_read()") == 1 and bg.count("predict_rebuild(") == 1, "predict_background reads all the notes more than once") + other = bg.index("} else {", bg.index("if (stream) {")) + check(other < bg.index("notes = pipe_notes_read();") < bg.index("predicted = predict_rebuild(notes, plan);") < bg.index("if (keys.empty()) {"), + "predict_background reads every note or rebuilds the plan with the switch on") + first = bg[bg.index("if (stream) {"):other] + check("predict_stream::index(f, program_cache_stamp(), sizeof(PipeNote), rec.get(), [&] { return note_sane(*rec); });" in first and + "groups = predict_stream::group(plan, at, had);" in first and "if (!plan_read(&plan)) return no_plan();" in first, + "the records' index or the plan's groups are not made as predict_stream.h makes them") + check("std::unordered_map at;" in first and "std::vector plan;" in first, + "the index and the plan outlive the block that makes the groups: held while the builder rests") + worker = bg[bg.index("const auto worker = [&](unsigned id) {"):] + check('in.f = fopen(pipe_notes_path().c_str(), "rb");' in worker and "~Notes() { if (f) fclose(f); }" in worker, + "a worker does not open (and close) its own handle on the notes file") + at = [worker.index(p) for p in ("for (const predict_stream::Entry& e : groups[i].entries) {", + "if (e.variant != p.variant || !in.f) continue;", + "if (!pipe_note_at(in.f, e.vertex_rec, *in.v) || !pipe_note_at(in.f, e.output_rec, *in.o)) continue;", + "predicted_into(*in.n, *in.v, *in.o, keys[i], e.variant);", "build(in.n.get());")] + check(at == sorted(at), "a worker does not read a pipeline's two notes by their records and build of them") + check("by_program[keys[i]]" not in worker, "a worker writes into by_program (operator[]) from its thread") + check("predicted_done(pstamp, keys[i]);" in worker, "a program built is no longer recorded") + pred = (VK / "vk_predict.cpp").read_text(encoding="utf-8") + note = pred[pred.index("std::unique_ptr predicted_note("):pred.index("predict_rebuild(")] + check("predicted_into(*n, V, O, prog, variant);" in note, "predict_rebuild's note is not made by predicted_into: two results") + into = pred[pred.index("void predicted_into(PipeNote& n, const PipeNote& V, const PipeNote& O, uint64_t prog, uint64_t variant) {"):pred.index("std::unique_ptr predicted_note(")] + for line in ("n = O;", "std::memcpy(n.s.vattr, V.s.vattr, sizeof n.s.vattr);", "n.s.nvattr = V.s.nvattr;", + "std::memcpy(n.s.vstream, V.s.vstream, sizeof n.s.vstream);", "n.topo = V.topo;", "n.prog = prog, n.variant = variant;", "n.key = 0;"): + check(line in into, "predicted_into no longer does: " + line) + check("void predicted_into(PipeNote& n, const PipeNote& V, const PipeNote& O, uint64_t prog, uint64_t variant);" in + (VK / "vk_draw.h").read_text(encoding="utf-8"), "vk_draw.h does not declare predicted_into") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`RX_VK_PREDICT_BG_STREAM=0`" in dev, "docs/development.md: RX_VK_PREDICT_BG_STREAM is not in the switches table") + + with tempfile.TemporaryDirectory(prefix="predict-stream-") as tmp: + c, exe = Path(tmp) / "stream.cpp", Path(tmp) / "stream.exe" + c.write_text(PROGRAM, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(VK), str(c), "-o", str(exe)], check=True) + r = subprocess.run([str(exe), str(Path(tmp) / "notes.bin")], capture_output=True, text=True) + check(r.returncode == 0, "the plan as records:\n" + r.stdout) + print("PASS: the predicted builder keeps record numbers and reads its notes when it builds") + + +if __name__ == "__main__": + main() diff --git a/tests/test_predict_types.py b/tests/test_predict_types.py new file mode 100644 index 0000000..7f2124b --- /dev/null +++ b/tests/test_predict_types.py @@ -0,0 +1,294 @@ +"""The prediction's checks: a predicted vertex layout against the program's vertex inputs' types, and each predicted +pipeline built once (gpu/vk/predict_types.h; vk_predict.cpp candidates, layout_kinds, predict_pipelines; vk_draw.cpp +predict_first, predict_run, predict_background; no game, no GPU). + +(Triforce, 11-22, the performance audit's PIPE-44, after Source A's.) A program's key already says which vertex inputs it +reads as integers, so a predicted layout borrowed from other programs can hand an integer input a float attribute, or the +other signedness: an invalid pipeline. With RX_VK_PREDICT_TYPES=1 the plan drops such a layout (read from the vertex +module's SPIR-V; scaled formats count as integers). And a program's predictions could give the same pipeline twice; unset, +each is built once by its key as play keys it: the same pipelines. 0: as before. + +The header is compiled against hand-made SPIR-V: the inputs with a Location and their base types through pointers, +vectors and arrays, built-ins and outputs not counted; a uint input against a float attribute does not fit, float against +float and sint against a signed integer do, an unset attribute or an unread location says nothing; the switch. +Then cut out of vk_predict.cpp and run: candidates() with every class fitting gives what it gave before (an exact module's +classes, else the signature's most common, the earliest of a tie); with the check, a class that does not fit is dropped and +counted, the signature's next most common taken, an exact module whose classes all differ falls back to the signature; +layout_kinds reads unset attributes as unknown and scaled formats as integers. +Then predict_first, cut out of vk_draw.cpp, in the builder's loop over a stand-in driver: the same pipelines are built as +with every prediction built, each once, and the repeats counted; with the switch at 0, every one built, as before. +And reads the source for the hooks and the switches table's row. + + python -I -B tests/test_predict_types.py [--cxx clang++] +""" +import argparse +import os +import subprocess +import tempfile +from pathlib import Path + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] + +HEADER = r""" +#include +#include "predict_types.h" +#define CHECK(x) do { if (!(x)) { std::printf("line %d: %s\n", __LINE__, #x); return 1; } } while (0) +using namespace predict_types; +static void op(std::vector& w, uint32_t code, std::initializer_list args) { + w.push_back(((uint32_t)args.size() + 1) << 16 | code); + w.insert(w.end(), args); +} +int main() { + CHECK(mode(nullptr) == 1 && mode("") == 1 && mode("0") == 0 && mode("1") == 2 && mode("2") == 1); + CHECK(!once(0) && once(1) && once(2) && !typed(0) && !typed(1) && typed(2)); + std::vector w = {0x07230203, 0x10000, 0, 40, 0}; + op(w, 22, {1, 32}); // %1 float + op(w, 21, {2, 32, 0}); // %2 uint + op(w, 21, {3, 32, 1}); // %3 int + op(w, 23, {4, 1, 4}); // %4 vec4 + op(w, 23, {5, 2, 4}); // %5 uvec4 + op(w, 23, {6, 3, 2}); // %6 ivec2 + op(w, 28, {14, 5, 1}); // %14 uvec4[1] (an array of vectors) + op(w, 32, {7, 1, 4}); // %7 Input vec4* + op(w, 32, {8, 1, 5}); // %8 Input uvec4* + op(w, 32, {9, 1, 6}); // %9 Input ivec2* + op(w, 32, {15, 1, 14}); // %15 Input uvec4[1]* + op(w, 32, {16, 1, 3}); // %16 Input int* (the built-in's) + op(w, 32, {17, 3, 4}); // %17 Output vec4* + op(w, 71, {10, 30, 0}); // %10 Location 0 + op(w, 71, {11, 30, 1}); // %11 Location 1 + op(w, 71, {12, 30, 3}); // %12 Location 3 + op(w, 71, {18, 30, 5}); // %18 Location 5 + op(w, 71, {13, 11, 42}); // %13 BuiltIn VertexIndex (no Location) + op(w, 71, {19, 30, 2}); // %19 Location 2, an output + op(w, 59, {7, 10, 1}); // OpVariable %10 Input + op(w, 59, {8, 11, 1}); + op(w, 59, {9, 12, 1}); + op(w, 59, {15, 18, 1}); + op(w, 59, {16, 13, 1}); + op(w, 59, {17, 19, 3}); + const Locations in = locations(w); + CHECK(in.mask == 0x2B && in.ints == 0x2A && in.sign == 0x08); + Kind k[16] = {}; + k[0] = Kind::Float, k[1] = Kind::Uint, k[3] = Kind::Sint, k[5] = Kind::Uint; + CHECK(fits(in, k)); + k[2] = Kind::Sint; // (a location the module does not read) + CHECK(fits(in, k)); + k[1] = Kind::Float; // (a uint input against a float attribute) + CHECK(!fits(in, k)); + k[1] = Kind::Sint; // (the other signedness) + CHECK(!fits(in, k)); + k[1] = Kind::Unknown; // (unset: reads zeros, as in play) + CHECK(fits(in, k)); + k[0] = Kind::Uint; // (a float input against an integer attribute) + CHECK(!fits(in, k)); + k[0] = Kind::Float, k[3] = Kind::Uint; + CHECK(!fits(in, k)); + CHECK(locations({}).mask == 0 && locations({0x07230203, 0, 0, 0, 0, 0xFFFF0047}).mask == 0); // (empty, broken) + std::printf("ok\n"); + return 0; +} +""" + +PREDICT = r""" +#include +#include +#include +#include +#include +#include +#include +#include "predict_types.h" +#define CHECK(x) do { if (!(x)) { std::printf("line %d: %s\n", __LINE__, #x); return 1; } } while (0) +namespace Shader { enum class AttributeType { Float, SignedInt, UnsignedInt, SignedScaled, UnsignedScaled, Disabled }; } +Shader::AttributeType attribute_type_of(uint32_t vattr) { return (Shader::AttributeType)(vattr & 7); } +struct NvnState { int nvattr; uint32_t vattr[16]; struct { uint32_t stride, divisor; } vstream[16]; }; +struct PipeNote { uint64_t key; uint32_t topo, samples, n, color_format[8], depth_format; bool has_depth; NvnState s; }; +namespace t { +/* CUT */ +} +using namespace t; +int main() { + size_t mis = 0; + const auto all = [](const Class&) { return true; }; + Seen exact, sig; + const Class a{1, 2}, b{3, 4}, c{5, 6}; + exact.add(a, {1, 0}), exact.add(b, {2, 0}); + sig.add(a, {1, 0}), sig.add(b, {2, 0}), sig.add(b, {3, 0}), sig.add(c, {4, 0}), sig.add(c, {5, 0}); + // every class fitting: what it gave before + CHECK(candidates(&exact, &sig, all, &mis) == (std::vector{a, b}) && mis == 0); + CHECK(candidates(nullptr, &sig, all, &mis) == (std::vector{b}) && mis == 0); // (b and c tie: the earliest) + CHECK(candidates(nullptr, nullptr, all, &mis).empty() && candidates(nullptr, &exact, all, &mis) == (std::vector{a})); + // the check: a class that does not fit is dropped and counted + const auto no_b = [&](const Class& x) { return x != b; }; + CHECK(candidates(&exact, &sig, no_b, &mis) == (std::vector{a}) && mis == 1); + mis = 0; + CHECK(candidates(nullptr, &sig, no_b, &mis) == (std::vector{c}) && mis == 1); // (the next most common) + mis = 0; + const auto only_c = [&](const Class& x) { return x == c; }; + CHECK(candidates(&exact, &sig, only_c, &mis) == (std::vector{c})); // (the exact all differ: the signature) + const auto none = [](const Class&) { return false; }; + CHECK(candidates(&exact, &sig, none, &mis).empty()); + // layout_kinds: unset (past nvattr, or no stream) unknown, scaled as integers + PipeNote n{}; + n.s.nvattr = 6; + n.s.vattr[0] = 0; // float + n.s.vattr[1] = 2; // unsigned int + n.s.vattr[2] = 1; // signed int + n.s.vattr[3] = 3; // signed scaled + n.s.vattr[4] = 4 | (15u << 27); // unsigned scaled, but no stream: unset + n.s.vattr[5] = 4; // unsigned scaled + n.s.vattr[6] = 2; // past nvattr: unset + predict_types::Kind k[16]; + layout_kinds(n, k); + using K = predict_types::Kind; + CHECK(k[0] == K::Float && k[1] == K::Uint && k[2] == K::Sint && k[3] == K::Sint && k[4] == K::Unknown && k[5] == K::Uint && k[6] == K::Unknown); + std::printf("ok\n"); + return 0; +} +""" + +ONCE = r""" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "predict_types.h" +#define CHECK(x) do { if (!(x)) { std::printf("line %d: %s\n", __LINE__, #x); return 1; } } while (0) +using VkPrimitiveTopology = int; +struct NvnState { uint32_t stride, blend; }; +struct PipeNote { uint64_t prog, variant; uint32_t topo; NvnState s; }; +struct Targets { int n; }; +struct Program { uint64_t key, variant; }; +Targets note_targets(const PipeNote&) { return {1}; } +// (as play keys a pipeline: the stride is dynamic and not in it) +uint64_t pipeline_key(const Program& p, const NvnState& s, const Targets&, VkPrimitiveTopology topo) { + return p.key * 1000003u ^ p.variant * 7919u ^ (uint64_t)topo * 131u ^ s.blend; +} +std::multiset driver; // (every pipeline the stand-in driver was asked to build, by key) +uint64_t build_graphics(const Program& p, const NvnState& s, const Targets& at, VkPrimitiveTopology topo) { + driver.insert(pipeline_key(p, s, at, topo)); + return 1; +} +std::atomic st_predict_twice{0}; +/* CUT */ +int main(int argc, char** argv) { + const bool as_before = argc > 1 && argv[1][0] == '0'; + // two programs' predictions: layouts that differ by stride alone give the same pipeline + std::map> by_program; + for (uint64_t prog : {10ull, 20ull}) + for (uint64_t variant : {0ull, 1ull}) + for (uint32_t stride : {12u, 16u, 24u}) + for (uint32_t blend : {0u, 5u}) by_program[prog].push_back(PipeNote{prog, variant, 3, NvnState{stride, blend}}); + std::set wanted; + size_t asked = 0; + for (const auto& kv : by_program) + for (const PipeNote& n : kv.second) wanted.insert(pipeline_key(Program{n.prog, n.variant}, n.s, note_targets(n), (int)n.topo)), asked++; + // the builder's loop: a program borrowed, each of its variants handed over, its notes of that variant built + for (const auto& kv : by_program) { + std::unordered_set seen; + for (uint64_t variant : {0ull, 1ull}) { + Program p{kv.first, variant}; + for (const PipeNote& n : kv.second) + if (n.variant == p.variant && predict_first(seen, p, n)) build_graphics(p, n.s, note_targets(n), (int)n.topo); + } + } + const std::set got(driver.begin(), driver.end()); + CHECK(got == wanted); // (the same pipelines either way) + if (as_before) { + CHECK(driver.size() == asked && st_predict_twice == 0); + } else { + CHECK(driver.size() == wanted.size() && wanted.size() == 8 && st_predict_twice == (int64_t)(asked - wanted.size())); + } + std::printf("ok\n"); + return 0; +} +""" + + +def block(text, marker): + start = text.index(marker) + at = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[at] == "{") - (text[at] == "}") + at += 1 + return text[start:at] + + +def need(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def ordered(text, parts, why): + last = 0 + for p in parts: + k = text.find(p, last) + need(k >= 0, why + " (missing, or out of order: " + p + ")") + last = k + len(p) + + +def compile_run(cxx, tmp, name, text, runs): + src, exe = tmp / (name + ".cpp"), tmp / (name + ".exe") + src.write_text(text, encoding="utf-8") + subprocess.run([cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-Wno-unused-function", "-I" + str(ROOT / "gpu/vk"), + str(src), "-o", str(exe)], check=True) + base = {k: v for k, v in os.environ.items() if not k.startswith("RX_VK_")} + for arg, env in runs: + r = subprocess.run([str(exe)] + ([arg] if arg else []), capture_output=True, text=True, timeout=60, env={**base, **env}) + need(r.returncode == 0 and r.stdout.strip() == "ok", "%s %s: %s" % (name, arg, r.stdout.strip() or r.returncode)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + pred = (ROOT / "gpu/vk/vk_predict.cpp").read_text(encoding="utf-8") + draw = (ROOT / "gpu/vk/vk_draw.cpp").read_text(encoding="utf-8") + need('#include "predict_types.h"' in pred and '#include "predict_types.h"' in draw, "predict_types.h is not included") + head = pred.index("uint64_t fnv(") + cut = pred[head:pred.index("} // namespace", head)] + need("void layout_kinds(" in cut and "std::vector candidates(const Seen* exact, const Seen* sig, Fits fits, size_t* misfits)" in cut, + "vk_predict.cpp: candidates or layout_kinds are not where the test cuts them") + once = block(draw, "bool predict_first(std::unordered_set& seen, const Program& p, const PipeNote& n) {") + with tempfile.TemporaryDirectory(prefix="predict-types-") as tmp: + tmp = Path(tmp) + compile_run(args.cxx, tmp, "header", HEADER, [("", {})]) + compile_run(args.cxx, tmp, "predict", PREDICT.replace("/* CUT */", cut), [("", {})]) + compile_run(args.cxx, tmp, "once", ONCE.replace("/* CUT */", once), + [("", {}), ("1", {"RX_VK_PREDICT_TYPES": "1"}), ("0", {"RX_VK_PREDICT_TYPES": "0"})]) + pp = block(pred, "std::vector> predict_pipelines(") + ordered(pp, ['static const bool typed = predict_types::typed(predict_types::mode(getenv("RX_VK_PREDICT_TYPES")));', + "if (typed) f.vs_in = predict_types::locations(spirv);", "tally.no_face++;", + "if (!typed || !f.vs_in.mask) return true;", "layout_kinds(*vdonor.at(c), kinds);", "return predict_types::fits(f.vs_in, kinds);", + "candidates(find(vh, f.vs), find(vsig, f.vs_locs), vertex_fits, &tally.misfits);", + "candidates(find(fh, f.fs), find(fsig, f.fs_locs), any, &tally.misfits);", "if (counts) *counts = tally;"], + "predict_pipelines does not check the layouts' types only under the switch, or does not count") + need('static const bool once = predict_types::once(predict_types::mode(getenv("RX_VK_PREDICT_TYPES")));' in once, + "predict_first does not read the switch once") + run = block(draw, "void predict_run(void (*progress)(size_t done, size_t total)) {") + ordered(run, ["predict_pipelines(notes, &programs, &plan, &left);", "std::unordered_set seen;", "program_cache_borrow(keys[i],", + "lent = true;", "if (n->variant == p.variant && predict_first(seen, p, *n))", "build_graphics(p, n->s,", + "if (!lent) st_predict_unlent.fetch_add(1", "left.no_face, left.misfits);"], + "predict_run does not build each pipeline once, or does not count") + bg = block(draw, "void predict_background() {") + ordered(bg, ["std::unordered_set seen;", "program_cache_borrow(keys[i],", "lent = true;", "const auto build = [&](const PipeNote* n) {", + "if (!predict_first(seen, p, *n)) return;", "build_graphics(p, n->s,", "if (!lent) st_predict_unlent.fetch_add(1", + "predicted_done(pstamp, keys[i]);"], + "predict_background does not build each pipeline once, or does not count") + docs = (ROOT / "docs/development.md").read_text(encoding="utf-8") + need("`RX_VK_PREDICT_TYPES=1` or `0`" in docs, "docs/development.md: the switches table has no RX_VK_PREDICT_TYPES row") + print("PASS: each predicted pipeline built once (the same pipelines); RX_VK_PREDICT_TYPES=1 drops layouts whose types differ; " + "0 as before") + + +if __name__ == "__main__": + main() diff --git a/tests/test_prepare_again.py b/tests/test_prepare_again.py new file mode 100644 index 0000000..402507a --- /dev/null +++ b/tests/test_prepare_again.py @@ -0,0 +1,140 @@ +"""The shaders prepared again when the program cache was made again for another GPU, and the mark of the driver's part +without stage libraries by what it was made for (gpu/vk/prepare_again.h; vk_draw.cpp prepare_run; no game, no GPU). + +After Casual Gamer's #dev-fixes post. triforce_prepared.bin said the shaders were translated for this stamp while the +program cache had been made again since for another one (a start on another GPU or with other settings that did not +prepare): prepare_check said nothing to do and every program was translated in play. And the empty mark of a start that +died in the driver's part without stage libraries left that part out for good, on any GPU and driver after it. + +The header is compiled and run as it is, on files in a temporary folder: +- a program cache holds none of the shaders when there is none, when it is shorter than its head or is its head alone, + when its head is no program cache's, or names another stamp; one of this stamp with records holds them, and so does one + carried over an update (another build's word, the same stamp); +- the mark: none; an empty one (an older build's) is this driver's; one of this stamp is, one of another is not, unless + RX_VK_PREPARE_RECHECK=0 (any mark is then, as before); written with the stamp, or empty with the switch off. +Then the source: prepare_run asks the program cache only where the record says nothing to do, before the game (not in the +background) and with the program cache on, removes the record before it prepares again, and reads and writes the mark through +the header in the branch without libraries; the switch is in docs/development.md. + + python -I -B tests/test_prepare_again.py --cxx clang++ +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] + +FIXTURE = r""" +#include +#include +#include "prepare_again.h" +#define CHECK(x) do { if (!(x)) { std::printf("line %d: %s\n", __LINE__, #x); return 1; } } while (0) +using namespace prepare_again; +static std::string dir; +static void put(const std::string& name, const void* b, size_t n) { + FILE* f = std::fopen((dir + name).c_str(), "wb"); + if (n) std::fwrite(b, 1, n, f); + std::fclose(f); +} +static void programs(const std::string& name, uint32_t magic, uint32_t writer, uint64_t stamp, size_t records) { + std::string b(16 + records, 'r'); + std::memcpy(&b[0], &magic, 4), std::memcpy(&b[4], &writer, 4), std::memcpy(&b[8], &stamp, 8); + put(name, b.data(), b.size()); +} +int main(int argc, char** argv) { + if (argc < 2) return 2; + dir = std::string(argv[1]) + "/"; + const uint64_t A = 0x1234567890ABCDEFull, B = 0x0FEDCBA987654321ull; + uint64_t size = 1; + CHECK(on(nullptr) && on("1") && on("") && !on("0") && on("00")); + CHECK(programs_none_at(dir + "missing.bin", A, &size) && size == 0); + programs("kept.bin", kProgramsMagic, 7, A, 4096); + CHECK(!programs_none_at(dir + "kept.bin", A, &size) && size == 16 + 4096); + programs("carried.bin", kProgramsMagic, 8, A, 20); // (another build's word, the same stamp: carried over an update) + CHECK(!programs_none_at(dir + "carried.bin", A)); + CHECK(programs_none_at(dir + "kept.bin", B)); // (made again for another stamp) + programs("head.bin", kProgramsMagic, 7, A, 0); // (its head alone: made again, nothing in it yet) + CHECK(programs_none_at(dir + "head.bin", A, &size) && size == 16); + programs("magic.bin", 0x12345678, 7, A, 4096); + CHECK(programs_none_at(dir + "magic.bin", A)); + put("short.bin", "HSP1", 4); + CHECK(programs_none_at(dir + "short.bin", A)); + { // (the head's bytes alone) + uint8_t h[16]{}; + const uint32_t m = kProgramsMagic; + std::memcpy(h, &m, 4), std::memcpy(h + 8, &A, 8); + CHECK(!programs_none(h, 16, 17, A) && programs_none(h, 16, 16, A) && programs_none(h, 15, 4096, A) && programs_none(h, 16, 4096, B)); + } + const std::string mark = dir + "triforce_prepared.bin.warming"; + CHECK(mark_left(mark, A, true) == Mark::None && mark_left(mark, A, false) == Mark::None); + put("triforce_prepared.bin.warming", "", 0); // (an older build's empty mark: this driver's, as before) + CHECK(mark_left(mark, A, true) == Mark::Ours && mark_left(mark, A, false) == Mark::Ours); + CHECK(mark_write(mark, A, true)); + CHECK(mark_left(mark, A, true) == Mark::Ours && mark_left(mark, B, true) == Mark::Other && mark_left(mark, B, false) == Mark::Ours); + CHECK(mark_write(mark, B, false)); // (the switch off: empty, as before) + CHECK(mark_left(mark, A, true) == Mark::Ours); + { + FILE* f = std::fopen(mark.c_str(), "rb"); + std::fseek(f, 0, SEEK_END); + const long n = std::ftell(f); + std::fclose(f); + CHECK(n == 0); + } + std::printf("prepare_again.h: ok\n"); + return 0; +} +""" + + +def between(text, start, end): + a = text.index(start) + return text[a:text.index(end, a)] + + +def check_source(): + draw = (ROOT / "gpu/vk/vk_draw.cpp").read_text(encoding="utf-8") + run = between(draw, "bool prepare_run(void (*progress)(size_t done, size_t total), bool background) {", "\n}\n") + ask = "if (why == PrepareWhy::None && !background && recheck && !getenv(\"RX_VK_NO_PROGRAM_CACHE\")) {" + if ask not in run or run.index(ask) > run.index("if (why == PrepareWhy::None && !getenv(\"RX_VK_PREPARE_AGAIN\")) {"): + raise SystemExit("prepare_run does not ask the program cache before it takes the record's word that nothing is to do") + again = between(run, ask, "\n }\n") + if "prepare_again::programs_none_at(programs, program_cache_stamp(), &size)" not in again or \ + "triforce_programs.bin" not in again or "why = PrepareWhy::First;" not in again or \ + again.index("remove(path.c_str());") > again.index("why = PrepareWhy::First;"): + raise SystemExit("prepare_run does not prepare again, with the stale record removed first, for a program cache of none " + "of these shaders") + if 'prepare_again::on(getenv("RX_VK_PREPARE_RECHECK"))' not in run: + raise SystemExit("RX_VK_PREPARE_RECHECK is not read") + if "prepare_again::mark_left(warming, warmed, recheck)" not in run or \ + "prepare_again::mark_write(warming, warmed, recheck)" not in run or 'fopen(warming.c_str(), "wb")' in run: + raise SystemExit("the mark of the driver's part without libraries is not read and written by what it is made for") + dev_md = (ROOT / "docs/development.md").read_text(encoding="utf-8") + if "`RX_VK_PREPARE_RECHECK=0`" not in dev_md: + raise SystemExit("RX_VK_PREPARE_RECHECK is not in docs/development.md's switches") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="g++") + args = parser.parse_args() + check_source() + with tempfile.TemporaryDirectory(prefix="prepare-again-") as tmp: + tmp = Path(tmp) + (tmp / "t.cpp").write_text(FIXTURE, encoding="utf-8") + exe = tmp / "prepare-again.exe" + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(ROOT / "gpu/vk"), + str(tmp / "t.cpp"), "-o", str(exe)], check=True) + files = tmp / "files" + files.mkdir() + r = subprocess.run([str(exe), str(files)], capture_output=True, text=True, timeout=120) + if r.returncode: + raise SystemExit("prepare_again.h: " + (r.stdout.strip() or str(r.returncode))) + print("PASS: a program cache of none of the prepared shaders prepares them again, and the mark of the driver's part " + "without libraries is this driver's only by its stamp") + + +if __name__ == "__main__": + main() diff --git a/tests/test_prepare_threads.cpp b/tests/test_prepare_threads.cpp index 454eea5..56b41c3 100644 --- a/tests/test_prepare_threads.cpp +++ b/tests/test_prepare_threads.cpp @@ -4,6 +4,7 @@ #include #include +#include static int failures = 0; static void expect(bool ok, const char* what) { @@ -55,6 +56,11 @@ int main() { expect(translate(count(4, 64 * GB, false, 0), true, false) == 4 && translate(2, true, false) == 2, "AMD's older driver, few processors: count's"); expect(translate(count(24, 26 * GB, false, 20), true, true) == 20, "AMD's older driver: the switch still decides"); expect(translate(count(24, 26 * GB, false, 0), false, false) == 24, "other drivers: count's"); + // ((Triforce) the process heap's signature: the segment heap's, the NT heap's, anything else not known) + expect(prepare_threads::segment_heap(0xDDEEDDEEu) && !prepare_threads::segment_heap(0xFFEEFFEEu) && !prepare_threads::segment_heap(0), + "segment_heap: the segment heap's signature only"); + expect(std::string(prepare_threads::heap_name(0xDDEEDDEEu)) == "segment" && std::string(prepare_threads::heap_name(0xFFEEFFEEu)) == "NT" && + std::string(prepare_threads::heap_name(0x12345678u)) == "not known", "heap_name"); if (failures) return EXIT_FAILURE; std::printf("prepare threads: all checks passed\n"); return EXIT_SUCCESS; diff --git a/tests/test_prepare_threads.py b/tests/test_prepare_threads.py index 2ca67c1..cb39817 100644 --- a/tests/test_prepare_threads.py +++ b/tests/test_prepare_threads.py @@ -46,7 +46,10 @@ def check_source(): need("vk_draw.cpp: prepare_workers()", workers, ['getenv("RX_VK_PREPARE_THREADS")', "GlobalMemoryStatusEx(&ms)", "ram_free = ms.ullAvailPhys;", "GetPriorityClass(GetCurrentProcess())", "cls == ABOVE_NORMAL_PRIORITY_CLASS || cls == HIGH_PRIORITY_CLASS", - "prepare_threads::count(cpus, ram_free, above_normal, e ? atoi(e) : 0)", "MemAvailable"]) + "prepare_threads::count(cpus, ram_free, above_normal, e ? atoi(e) : 0)", "MemAvailable", + # ((Triforce) which heap the process is on, said in the same line: nx2/win_manifest.py's HOOKSHOT_SEGMENT_HEAP) + "memcpy(&heap_signature, (const char*)heap + 16, sizeof heap_signature);", + "prepare_threads::heap_name(heap_signature)", '", the process heap: "']) # ((Triforce) the translation pass's workers: fewer on AMD's driver for RDNA 1 and 2, told apart as vk_vendor.h tells it) translate = block(draw, "unsigned prepare_translate_workers() {") need("vk_draw.cpp: prepare_translate_workers()", translate, @@ -62,7 +65,7 @@ def check_source(): need("vk_draw.cpp: %s()" % name, body, ["const unsigned workers = prepare_workers();", "for (unsigned i = 0; i < workers; i++) threads.emplace_back(", "SetThreadPriority(GetCurrentThread(), prepare_priority());"]) - if "cpus / 2" in body or "hardware_concurrency" in body: + if "cpus / 2" in body or "hardware_concurrency" in body or "cpu_count()" in body: raise SystemExit("vk_draw.cpp: %s() counts its workers itself again" % name) prepare = block(draw, "bool prepare_run(void (*progress)(size_t done, size_t total), bool background) {") need("vk_draw.cpp: prepare_run()", prepare, diff --git a/tests/test_prepare_timing.py b/tests/test_prepare_timing.py index da97495..2d3f65e 100644 --- a/tests/test_prepare_timing.py +++ b/tests/test_prepare_timing.py @@ -10,7 +10,9 @@ Only clocks are read around the builds: each build is the call it was. The workers of the preparation before the title (prepare_run, warm_libraries, state_list_prepare, predict_run) ran below normal priority; in a start that only prepares (HOOKSHOT_PREPARE_ONLY: Setup's build and update) they run at normal, but only in a process known to be below normal already (Windows: the below normal or idle class; elsewhere a nice value above 0), so -that they stay below every program of normal priority. The game's own starts keep below normal, the preparation behind the +that they stay below every program of normal priority. (Since the performance audit's OS-57 the Windows builders start the +build tools below normal only with HOOKSHOT_LOW_PRIORITY=1; in their default, normal-class start these workers stay below +normal, which is still under every program of normal priority: tests/test_build_priority.py.) The game's own starts keep below normal, the preparation behind the game idle, and precompile_run's workers the priority they had. The timing line every two minutes is written only after progress, so that a warm stuck in one build still goes silent for Setup's silence watch. diff --git a/tests/test_prepared_stamps.py b/tests/test_prepared_stamps.py index 169f216..f777fc6 100644 --- a/tests/test_prepared_stamps.py +++ b/tests/test_prepared_stamps.py @@ -161,11 +161,15 @@ int main(int argc, char** argv) { stamp.index("add(v, sizeof v);") > stamp.index("if (patches) {"): raise SystemExit("warmed_stamp_of: the module patches' and the rate's words are not added after the others, and only " "where they apply") - if "return warmed_stamp_of(libraries, module_patches(), false);" not in block(draw, "uint64_t warmed_stamp(") or \ - "return warmed_stamp_of(libraries, module_patches(), libraries && dev.foliage_rate);" not in \ + # ((Triforce, 11-21) and kMeshOptRecipe in both, where RADV builds its native task and mesh pipelines with the optimizer + # (mesh_opt_recipe, tests/test_radv_mesh_optimizer.py) + if "if (mesh_opt) add(&kMeshOptRecipe, sizeof kMeshOptRecipe);" not in stamp: + raise SystemExit("warmed_stamp_of: the mesh optimizer's word is not added only where it applies") + if "return warmed_stamp_of(libraries, module_patches(), false, mesh_opt_recipe());" not in block(draw, "uint64_t warmed_stamp(") or \ + "return warmed_stamp_of(libraries, module_patches(), libraries && dev.foliage_rate, mesh_opt_recipe());" not in \ block(draw, "uint64_t libraries_stamp("): - raise SystemExit("warmed_stamp / libraries_stamp: wanted the modules' patches in both and the rate's libraries in the " - "driver's part's alone") + raise SystemExit("warmed_stamp / libraries_stamp: wanted the modules' patches and the mesh optimizer's word in both and the " + "rate's libraries in the driver's part's alone") # ((Triforce, 11-19) and bit 2, the water's simplification (shader_water_mode, the RX 470 .. 590 port that came without a # new stamp): those cards warm their driver's part again once) if "return (dev.fixes.nan_bits && nan_selects ? 1u : 0u) | (shader_water_mode() ? 2u : 0u);" not in block(draw, "uint32_t module_patches("): diff --git a/tests/test_present_device_loss.py b/tests/test_present_device_loss.py index 9bc3cb3..9a150e1 100644 --- a/tests/test_present_device_loss.py +++ b/tests/test_present_device_loss.py @@ -54,6 +54,7 @@ struct { int device=1, queue=2, present_queue=3, swapchain=4; } dev; struct { int done=5, acquired=6, pool=7, cmd=8; } f; int bi=0, si=0, pi=0, index=0; uint64_t acquire_timeout=UINT64_MAX; +bool fenced=false; int fence=0; // ((Triforce) RX_VK_ACQUIRE_FENCE: the acquire's fence or semaphore) int result=VK_SUCCESS, calls=0, reports=0, continued=0; struct Lost {}; [[noreturn]] void device_lost() { ++reports; throw Lost{}; } diff --git a/tests/test_present_image_free.py b/tests/test_present_image_free.py new file mode 100644 index 0000000..7233543 --- /dev/null +++ b/tests/test_present_image_free.py @@ -0,0 +1,307 @@ +"""The swapchain's image waited for on the CPU through the acquire's fence, not on the replay's queue (gpu/vk/vk_replay.cpp +present_image, acquire_fence, image_free, held_image_drop, RX_VK_ACQUIRE_FENCE). + +AMD's Windows driver returns from vkAcquireNextImageKHR at once, the image not yet free: with the window covered the blit's +submit waited on the queue for the acquire's semaphore, every later submit of the replay queued behind it, and the game froze +for as long as the cover. present_image's acquire, its hold and its submit's waits are cut out unchanged and run against a model +presentation engine that hands out an image before it is free (its fence signaled only once it is). Checked: + +- the image free at once: one acquire a present, with the fence and no semaphore; the submit waits for nothing; +- the window covered (images back after 1 s): the first present waits 250 ms and holds the image, no frame is submitted, later + presents acquire nothing more and do not wait, and the first present after the image is back submits into it, waiting for + nothing; the next acquire then gets an unsignaled fence; +- a swapchain made again during a hold: the hold let go, its fence abandoned (never reset or waited for again), the next + acquire with a new, unsignaled fence; +- device loss in the fence's wait reported; +- RX_VK_ACQUIRE_FENCE=0, and DLSS frame generation running: the acquire signals the frame's semaphore and the submit waits for it, + as before, and nothing is held. + + python -I -B tests/test_present_image_free.py [--cxx clang++] +""" +import argparse +import os +from pathlib import Path +import subprocess +import tempfile + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def fail(message): + raise SystemExit("FAIL: " + message) + + +HEAD = r'''#include +#include +#include +#include +#include +#include +#include +#include +#include +using VkResult = int; +using VkFence = uint64_t; +using VkSemaphore = uint64_t; +using VkCommandBuffer = uint64_t; +using VkSwapchainKHR = uint64_t; +constexpr uint64_t VK_NULL_HANDLE = 0; +constexpr VkResult VK_SUCCESS = 0, VK_NOT_READY = 1, VK_TIMEOUT = 2, VK_ERROR_DEVICE_LOST = -4, + VK_SUBOPTIMAL_KHR = 1000001003, VK_ERROR_OUT_OF_DATE_KHR = -1000001004; +constexpr int VK_TRUE = 1, VK_STRUCTURE_TYPE_FENCE_CREATE_INFO = 8, VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO = 1000314005, + VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO = 1000314006, VK_STRUCTURE_TYPE_SUBMIT_INFO_2 = 1000314004; +constexpr uint64_t VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT = 0x10000; +struct VkFenceCreateInfo { int sType; }; +struct VkSemaphoreSubmitInfo { int sType; const void* pNext; VkSemaphore semaphore; uint64_t value; uint64_t stageMask; uint32_t deviceIndex; }; +struct VkCommandBufferSubmitInfo { int sType; const void* pNext; VkCommandBuffer commandBuffer; uint32_t deviceMask; }; +struct VkSubmitInfo2 { + int sType; const void* pNext; uint32_t flags; uint32_t waitSemaphoreInfoCount; const VkSemaphoreSubmitInfo* pWaitSemaphoreInfos; + uint32_t commandBufferInfoCount; const VkCommandBufferSubmitInfo* pCommandBufferInfos; uint32_t signalSemaphoreInfoCount; + const VkSemaphoreSubmitInfo* pSignalSemaphoreInfos; +}; +struct Extent { uint32_t width = 1280, height = 720; }; +struct { + int device = 1; + VkSwapchainKHR swapchain = 7; + Extent swap_extent; + std::vector swap_rendered{0x51, 0x52, 0x53}; + std::mutex present_mu; + uint64_t swap_generation = 1; + std::atomic swap_resize{0}; +} dev; +namespace dlssg { bool on = false; bool active() { return on; } } +// (PRESENT-45: a lost surface's handling at the acquire; none is lost here, tests/test_present_surface_lost.py runs it) +bool surface_recover(VkResult) { return false; } +void invalidate_lost_surface() {} +struct { std::atomic paced_ns{0}; } present_fig; +struct Lost {}; +int lost_reports = 0; +[[noreturn]] void device_lost() { ++lost_reports; throw Lost{}; } +void logf(const char*, ...) {} +#define CHECK(c) do { if (!(c)) { std::printf("CHECK failed line %d: %s\n", __LINE__, #c); std::exit(7); } } while (0) + +// ---- the model presentation engine: it hands out an image at once, before it is free; the acquire's fence is signaled +// when the image is free. One clock, moved by the presents and by the fence waits +int64_t now_ms = 0; +struct Image { bool app = false; int64_t free_at = 0; }; +std::vector images(3); +uint32_t next_image = 0; +struct FenceState { bool signaled = false, pending = false; int64_t at = 0; int resets = 0, waits = 0; }; +std::map fences; +VkFence next_fence = 0xf00; +int acquires = 0, acquire_with_semaphore = 0, remakes = 0; +bool wait_lost = false, bad_reset = false, bad_fence = false; +int64_t return_after_ms = 16; // (an image presented is free again this long after) +VkResult vkCreateFence(int, const VkFenceCreateInfo* fi, const void*, VkFence* out) { + CHECK(fi && fi->sType == VK_STRUCTURE_TYPE_FENCE_CREATE_INFO); + *out = ++next_fence; + fences[*out] = {}; + return VK_SUCCESS; +} +VkResult vkAcquireNextImageKHR(int, VkSwapchainKHR sc, uint64_t, VkSemaphore semaphore, VkFence fence, uint32_t* index) { + CHECK(sc == dev.swapchain && (semaphore || fence) && !(semaphore && fence)); + acquires++; + uint32_t i = next_image, tried = 0; + while (images[i].app && tried++ < images.size()) i = (i + 1) % images.size(); + CHECK(!images[i].app); // (the app never asks for more images than it may hold) + next_image = (i + 1) % images.size(); + images[i].app = true; + if (fence) { + FenceState& s = fences.at(fence); + if (s.signaled || s.pending) bad_fence = true; // (a fence given to an acquire must be unsignaled) + s.pending = true, s.at = images[i].free_at; + } else { + acquire_with_semaphore++; + } + *index = i; + return VK_SUCCESS; +} +VkResult vkWaitForFences(int, uint32_t n, const VkFence* f, int all, uint64_t timeout_ns) { + CHECK(n == 1 && all == VK_TRUE); + FenceState& s = fences.at(*f); + s.waits++; + if (wait_lost) return VK_ERROR_DEVICE_LOST; + const int64_t timeout_ms = (int64_t)(timeout_ns / 1000000); + if (s.pending && s.at <= now_ms + timeout_ms) { + if (s.at > now_ms) now_ms = s.at; + s.pending = false, s.signaled = true; + } + if (s.signaled) return VK_SUCCESS; + now_ms += timeout_ms; + return VK_TIMEOUT; +} +VkResult vkResetFences(int, uint32_t n, const VkFence* f) { + CHECK(n == 1); + FenceState& s = fences.at(*f); + if (s.pending) bad_reset = true; + s.signaled = false, s.resets++; + return VK_SUCCESS; +} +bool create_swapchain(uint32_t, uint32_t) { return ++remakes, true; } +void remake(int64_t free_at) { // (a swapchain made again: its images new, the generation one more) + for (auto& im : images) im = {false, free_at}; + next_image = 0; + dev.swap_generation++; +} +struct Frame { VkSemaphore acquired = 0x5e0; VkCommandBuffer cmd = 0xc0; } frame_slot; +int submits = 0; +uint32_t submit_waits = 99, submitted_index = 99; +VkSemaphore submit_wait_semaphore = 0; +''' + +STEP = r''' +/// One present, as present_image's (its image presented at the submit, back return_after_ms later) +void present_body() { + Frame& f = frame_slot; +@ACQUIRE@ +@SUBMIT@ + submits++; + submit_waits = si.waitSemaphoreInfoCount; + submit_wait_semaphore = si.waitSemaphoreInfoCount ? si.pWaitSemaphoreInfos[0].semaphore : 0; + submitted_index = index; + CHECK(si.signalSemaphoreInfoCount == 1 && si.pSignalSemaphoreInfos[0].semaphore == dev.swap_rendered[index]); + images[index].app = false, images[index].free_at = now_ms + return_after_ms; +} +/// true when the present got to the submit +bool present_step() { + const int before = submits; + present_body(); + return submits > before; +} +int main(int argc, char** argv) { + CHECK(argc == 2); + const std::string k = argv[1]; + const bool off = getenv("RX_VK_ACQUIRE_FENCE") && !atoi(getenv("RX_VK_ACQUIRE_FENCE")); + dlssg::on = getenv("TEST_DLSSG") != nullptr; + const bool semaphore_way = off || dlssg::on; + if (k == "free") { + for (int i = 0; i < 30; i++, now_ms += 16) { + CHECK(present_step()); + CHECK(acquires == i + 1); + if (semaphore_way) CHECK(submit_waits == 1 && submit_wait_semaphore == frame_slot.acquired); + else CHECK(submit_waits == 0 && acquire_with_semaphore == 0); + } + CHECK(!bad_fence && !bad_reset); + if (!semaphore_way) CHECK(fences.size() == 1); // (one fence, reset after each wait) + } else if (k == "covered") { + for (auto& im : images) im.free_at = 1000; // (the window covered until 1 s) + int shown = 0, first_shown_at = -1; + for (int i = 0; i < 100; i++) { + const int64_t before = now_ms; + const int acquired_before = acquires; + const bool went = present_step(); + if (semaphore_way) { + CHECK(went && submit_waits == 1 && submit_wait_semaphore == frame_slot.acquired); // (the queue waits, as before) + } else { + if (i == 0) CHECK(!went && now_ms - before == 250 && acquires == 1); // (250 ms, then held) + if (i > 0 && !went) CHECK(acquires == acquired_before && now_ms == before); // (no second acquire, no wait) + if (went) { + CHECK(submit_waits == 0); + if (first_shown_at < 0) { + first_shown_at = (int)now_ms; + CHECK(submitted_index == 0 && acquires == 1); // (the held image, back) + } + } + } + shown += went; + now_ms += 16; + } + if (!semaphore_way) { + CHECK(first_shown_at >= 1000 && first_shown_at < 1016 + 16); + CHECK(shown > 0 && !bad_fence && !bad_reset); + } + } else if (k == "remake") { + if (semaphore_way) { std::printf("OK %s\n", k.c_str()); return 0; } + for (auto& im : images) im.free_at = 100000; + CHECK(!present_step()); // (held) + const VkFence old = fences.begin()->first; + const int old_resets = fences.at(old).resets, old_waits = fences.at(old).waits; + remake(now_ms); // (made again, its images free) + now_ms += 16; + CHECK(present_step()); + CHECK(acquires == 2 && fences.size() == 2 && !bad_fence && !bad_reset); + CHECK(fences.at(old).resets == old_resets && fences.at(old).waits == old_waits); // (abandoned) + now_ms += 16; + CHECK(present_step() && acquires == 3 && fences.size() == 2 && submit_waits == 0); + } else if (k == "lost") { + if (semaphore_way) { std::printf("OK %s\n", k.c_str()); return 0; } + wait_lost = true; + try { present_step(); CHECK(!"went on past a device loss"); } catch (const Lost&) {} + CHECK(lost_reports == 1 && submits == 0); + } else { + CHECK(!"no such case"); + } + std::printf("OK %s\n", k.c_str()); + return 0; +} +''' + + +def fixture(replay): + present = block(replay, "void present_image(VkImage src, uint32_t sw, uint32_t sh, const int crop[4], VkImageView view) {") + a = present.index(" uint32_t index = 0;\n") + acquire = present[a:present.index(" if (vkResetFences(dev.device, 1, &f.done)", a)] + s = present.index(" VkSemaphoreSubmitInfo waits[1] = {{VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO}};") + e = present.index(" si.pSignalSemaphoreInfos = &signal;", s) + submit = present[s:e] + " si.pSignalSemaphoreInfos = &signal;\n" + h = replay.index("bool acquire_fence_on() {") + helpers = replay[h:replay.index(block(replay, "VkResult image_free(uint64_t timeout_ns) {"), h)] + \ + block(replay, "VkResult image_free(uint64_t timeout_ns) {") + return HEAD + helpers + STEP.replace("@ACQUIRE@", acquire).replace("@SUBMIT@", submit) + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--cxx", default="g++") + a = p.parse_args() + replay = (ROOT / "gpu" / "vk" / "vk_replay.cpp").read_text(encoding="utf-8") + vk_main = (ROOT / "gpu" / "vk" / "vk_main.cpp").read_text(encoding="utf-8") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + try: + text = fixture(replay) + made = vk_main[vk_main.index(" dev.swapchain = sc;"):] + if "dev.swap_generation++;" not in made[:made.index("swap_views_make(")]: + fail("create_swapchain does not count the swapchains it makes (dev.swap_generation)") + except ValueError as e: + fail("a text this test reads moved, or is not there (%s)" % e) + if "`RX_VK_ACQUIRE_FENCE=0`" not in dev: + fail("docs/development.md has no RX_VK_ACQUIRE_FENCE row") + problems = [] + with tempfile.TemporaryDirectory(prefix="present-image-free-") as tmp: + cpp = Path(tmp) / "fixture.cpp" + exe = Path(tmp) / ("fixture" + (".exe" if os.name == "nt" else "")) + cpp.write_text(text, encoding="utf-8") + built = subprocess.run([a.cxx, "-std=c++20", "-O1", "-Wall", "-Wno-unused-function", "-Wno-unused-variable", str(cpp), + "-o", str(exe)], capture_output=True, text=True) + if built.returncode: + print(built.stdout + built.stderr) + fail("the fixture did not build") + base = {k: v for k, v in os.environ.items() if k not in ("RX_VK_ACQUIRE_FENCE", "TEST_DLSSG")} + n = 0 + for label, extra in (("default", {}), ("=1", {"RX_VK_ACQUIRE_FENCE": "1"}), ("=0", {"RX_VK_ACQUIRE_FENCE": "0"}), + ("DLSS frame generation", {"TEST_DLSSG": "1"})): + for case in ("free", "covered", "remake", "lost"): + n += 1 + done = subprocess.run([str(exe), case], env={**base, **extra}, capture_output=True, text=True, timeout=30) + if done.returncode or done.stdout.strip() != "OK " + case: + problems.append("%s (%s): %s" % (case, label, (done.stdout + done.stderr).strip())) + if problems: + fail("\n ".join([""] + problems)) + print("PASS: the swapchain's image waited for on the CPU, held while covered with no second acquire, the blit's submit " + "waiting for nothing; =0 and DLSS frame generation keep the semaphore (%d cases)" % n) + + +if __name__ == "__main__": + main() diff --git a/tests/test_present_period_line.py b/tests/test_present_period_line.py index 0bdc656..bd1a5f0 100644 --- a/tests/test_present_period_line.py +++ b/tests/test_present_period_line.py @@ -63,6 +63,9 @@ static void hos_log(const char *fmt, ...) { static uint64_t hos_ticks(void) { return 0; } static int fps_max(void) { return 240; } static int64_t steady_period(int64_t base, int64_t now) { (void)now; return base; } /* ((Triforce) the Steady frame rate row off) */ +static void movie_pace_decide(void) {} /* ((Triforce, 11-22, PRESENT-30) no movie plays here: paced by the cap) */ +static int pace_fps(void) { return fps_max(); } +void hos_bench_present(int64_t now) { (void)now; } /* (benchmark mode's note of each present: not on here) */ ''' SUFFIX = r''' @@ -81,7 +84,7 @@ int main(void) { clock_now += d; hos_vi_native_presented(); } - if (total < 5000000000LL || nlines != 3) return 90; /* (one window: its three lines) */ + if (total < 5000000000LL || nlines != 4) return 90; /* (one window: its three lines, and its 1% low) */ if (pace_period_ns != P) return 91; /* (the period it counts against is the one it paces by) */ for (int i = 0; i < nlines; i++) puts(lines[i]); printf("expected %.3f\n", off / 1e6 / 1091); @@ -98,7 +101,8 @@ def main(): actual = block(source, "void hos_vi_native_presented(") with tempfile.TemporaryDirectory(prefix="present-period-") as tmp: cpp, exe = Path(tmp) / "fixture.cpp", Path(tmp) / "fixture.exe" - cpp.write_text(PREFIX + actual + "\n" + SUFFIX, encoding="utf-8") + # (with the 1% low's sort, which the cut-out present calls) + cpp.write_text(PREFIX + block(source, "static int cmp_i64_desc(") + "\n" + actual + "\n" + SUFFIX, encoding="utf-8") subprocess.run([args.cxx, "-std=c++17", "-Wall", "-Wextra", "-Werror", str(cpp), "-o", str(exe)], check=True) out = subprocess.run([str(exe)], capture_output=True, text=True, check=True).stdout period = re.search(r"vi: frame intervals against the paced period of ([\d.]+) ms: under 0.75 of it (\d+), 0.75-0.95 (\d+), " diff --git a/tests/test_present_rate.py b/tests/test_present_rate.py index 6b13587..31824e1 100644 --- a/tests/test_present_rate.py +++ b/tests/test_present_rate.py @@ -58,8 +58,10 @@ def check_source(): need("vk_texture.cpp: texture_sweep()", sweep, ["const auto age60 = [](uint64_t presents) { return present_rate::at60(presents, tex_rate.rate); };", # (the eviction's own wait, in the hold's count of the dead targets and in the pass) - "if (age60(frame_no - last) < need_unused && !shifted) continue;", - "if (age60(frame_no - last) < need_unused && !shifted && !gone_early) continue;", + # ((Triforce, 11-21, TEX-76) a depth target another size replaced goes sooner: slot_retire.h) + "if (age60(frame_no - last) < need_unused && !shifted && !slot_retired(tex)) continue;", + "const bool young = age60(frame_no - last) < need_unused;", + "if (young && !shifted && !gone_early && !retired) continue;", # (what the game let go: in the mass that turns the rule on, and for each texture; the longest gap before a # take-back, keep taken's, too) "return age60(frame_no - tex->let_go) >= tex_pressure::let_go_wait(let_go_after, age60(tex->back_gap), keep_taken);", diff --git a/tests/test_present_semaphores.py b/tests/test_present_semaphores.py index d37075d..911a051 100644 --- a/tests/test_present_semaphores.py +++ b/tests/test_present_semaphores.py @@ -25,7 +25,9 @@ assert 'uint32_t index = o.index;' in out and 'const PresentOut out{index, smoot signal = re.search(r' signal\.semaphore = [^;]+;', present)[0] wait = re.search(r' pi\.pWaitSemaphores = [^;]+;', out)[0] acquire_wait = re.search(r' waits\[0\]\.semaphore = [^;]+;', present)[0] -assert 'acquire_timeout, f.acquired, VK_NULL_HANDLE, &index)' in present +# ((Triforce) RX_VK_ACQUIRE_FENCE: the acquire signals the frame's semaphore only when it has no fence; tests/test_present_image_free.py +# runs the fence's way) +assert 'acquire_timeout, fenced ? VK_NULL_HANDLE : f.acquired, fence, &index)' in present # Include the actual creation attempt: passing oldSwapchain retires it even on failure. start = main.index(' ci.oldSwapchain = dev.swapchain;') end = main.index(' logf("vk: swapchain', start) @@ -99,6 +101,11 @@ void vkDeviceWaitIdle(int) {idle=true;} // (vk_replay.cpp's helper: held submits out, then the device idle; the real one also takes the present queue's own lock) void device_wait_idle_locked() {queue_flush_locked();vkDeviceWaitIdle(dev.device);} void swap_views_drop() {} // (hookshot: the overlays' views, vk_main.cpp) +// ((Triforce, 11-22, the performance audit's PRESENT-45) a lost surface's handling: none is lost here, and +// tests/test_present_surface_lost.py runs it against a driver that loses one) +std::atomic surface_lost_noted{false}; +bool surface_recover(int) {return false;} +void invalidate_lost_surface() {} bool recreate() { uint32_t n=0,width=640,height=480; struct {int format=1;} fmt; CreateInfo ci;int mode=0;VkResult result; diff --git a/tests/test_present_surface_lost.py b/tests/test_present_surface_lost.py new file mode 100644 index 0000000..907f633 --- /dev/null +++ b/tests/test_present_surface_lost.py @@ -0,0 +1,632 @@ +"""Presentation made again after the window's surface is lost (gpu/vk/surface_lost.h, vk_main.cpp create_swapchain, +invalidate_lost_surface and open_window_surface, vk_replay.cpp present_image and present_out, RX_VK_SURFACE_RECOVER; no +game, no GPU). + +(Triforce, 11-22, the performance audit's PRESENT-45, after Source A's.) VK_ERROR_SURFACE_LOST_KHR from an acquire, a +present or a swapchain's creation was not handled: every later acquire failed without a word and the picture stayed +frozen until a restart. Now what hangs on the surface is let go and the next swapchain's creation makes a surface for +the window again. + +The scripted runs have no window, so this is the proof there is. It compiles surface_lost.h (the rule) and the real +code cut out of the two files: invalidate_lost_surface and open_window_surface whole (the Windows branch and the other +one, each compiled here as code), create_swapchain's first step (the missing surface and the capabilities' query) and its +creation step, present_image's first step and its acquire, and present_out's last step; against a model of a driver that +can lose its surface at each of those calls and counts every handle. It runs: + +- the loss at the acquire: the present thread settled first, then the device idle, the views, the image semaphores, the + swapchain and the surface destroyed in that order, each once; nothing recorded or submitted; the next present makes a + surface (the startup's calls, the presents' queue chosen again) and a swapchain, and shows a frame; +- the loss at the present, on the presenting thread (let go there) and on the present thread (only noted: no Vulkan + call from that thread; the next present_image lets go and makes again); +- the loss at the creation and at the capabilities' query: the surface goes too, nothing is left published; +- a window with no size (minimized): no surface is made and none is asked of the driver until it has one again; +- a surface that cannot be made, or that the replay's queue cannot present to: tried again no sooner than 250 ms later, + the first 8 failures said, nothing leaked; +- many losses in a row: every handle made is destroyed once (a steady handle count), never one in use; +- RX_VK_SURFACE_RECOVER=0 and NVIDIA DLSS frame generation: nothing is let go, as before; +- a held image (RX_VK_ACQUIRE_FENCE) of a swapchain that went with a lost surface is left behind, not looked at again. + +Then the source: every new statement of the default path is behind the lost result or the missing surface; start() is +untouched (it still makes the first surface itself); the switch is read once; and the switch's row is in the guide. + + python -I -B tests/test_present_surface_lost.py [--cxx clang++] +""" +import argparse +import os +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def between(text, first, last, inclusive=True): + start = text.index(first) + end = text.index(last, start) + return text[start:end + (len(last) if inclusive else 0)] + + +FIXTURE = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "surface_lost.h" + +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) + +// ---- Vulkan's names, as far as the cut code uses them +using VkFence = uint64_t; +using VkSemaphore = uint64_t; +using VkSwapchainKHR = uint64_t; +using VkSurfaceKHR = uint64_t; +using VkImage = uint64_t; +using VkImageView = uint64_t; +using VkResult = int; +using VkBool32 = uint32_t; +constexpr uint64_t VK_NULL_HANDLE = 0; +enum { VK_SUCCESS = 0, VK_NOT_READY = 1, VK_TIMEOUT = 2, VK_SUBOPTIMAL_KHR = 1000001003, VK_ERROR_OUT_OF_DATE_KHR = -1000001004, + VK_ERROR_SURFACE_LOST_KHR = -1000000000, VK_ERROR_DEVICE_LOST = -4, VK_TRUE = 1, VK_FALSE = 0 }; +enum { VK_STRUCTURE_TYPE_FENCE_CREATE_INFO = 8, VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO = 9, + VK_STRUCTURE_TYPE_WIN32_SURFACE_CREATE_INFO_KHR = 1000009000, VK_STRUCTURE_TYPE_IMAGE_FORMAT_LIST_CREATE_INFO = 1000147000 }; +enum { VK_PRESENT_MODE_MAILBOX_KHR = 1, VK_PRESENT_MODE_IMMEDIATE_KHR = 0, VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT = 16, + VK_SWAPCHAIN_CREATE_MUTABLE_FORMAT_BIT_KHR = 4 }; +using VkFormat = int; +struct VkFenceCreateInfo { int sType; const void* pNext = nullptr; unsigned flags = 0; }; +struct VkSemaphoreCreateInfo { int sType; }; +struct VkExtent2D { uint32_t width = 0, height = 0; }; +struct VkSurfaceCapabilitiesKHR { VkExtent2D currentExtent; }; +struct VkImageFormatListCreateInfo { int sType; const void* pNext = nullptr; uint32_t viewFormatCount = 0; const VkFormat* pViewFormats = nullptr; }; +struct VkSwapchainCreateInfoKHR { VkSurfaceKHR surface = 0; VkSwapchainKHR oldSwapchain = 0; unsigned imageUsage = 1, minImageCount = 4, flags = 0; + const void* pNext = nullptr; }; +// (the window's side: Windows' names for the Windows branch, the SDL bridge's for the other) +struct RECT { long left = 0, top = 0, right = 0, bottom = 0; }; +using HWND = void*; +struct VkWin32SurfaceCreateInfoKHR { int sType; void* hinstance = nullptr; HWND hwnd = nullptr; }; +struct MkwVkWindow { + uint32_t version; + int (*create_surface)(void* window, void* instance, void* surface_out); + void (*drawable_size)(void* window, int* width, int* height); + void* window; +}; + +// ---- the renderer's names +struct Device { + uint64_t instance = 7, physical = 8; + int device = 1, queue = 2; + uint32_t queue_family = 0, present_family = 3; + uint64_t present_queue = 55; // (the presents' own queue, asked for at the start) + VkSurfaceKHR surface = 0; + VkSwapchainKHR swapchain = 0; + uint64_t swap_generation = 0; + int swap_format = 0; + VkExtent2D swap_extent{640, 480}; + bool swap_mutable = false; + std::vector swap_images; + std::vector swap_views; + std::vector swap_rendered; + std::atomic swap_resize{0}; + std::mutex present_mu; +} dev; +struct { int hdr = 0, vsync = 1; } opts; +struct Frame { VkFence done; VkSemaphore acquired; int cmd; }; +Frame frames[3] = {{900, 800, 1}, {901, 801, 2}, {902, 802, 3}}; +uint32_t frame_index = 0; +struct Lost {}; +[[noreturn]] void device_lost() { throw Lost{}; } +int log_lines = 0; +std::string last_log; +template void logf(const char* f, A...) { ++log_lines, last_log = f; } +bool vk_ft_on() { return false; } +template void vk_ft(const char*, A...) {} +bool dlssg_active = false; +namespace dlssg { bool active() { return dlssg_active; } } +namespace pace { +int64_t steady_floor_ns() { return 0; } +void steady_held(int64_t) {} +} +struct Host { int64_t (*now_ns)(); }; +int64_t clock_ns = 1000000000; +Host host_model{[]() -> int64_t { return clock_ns; }}; +Host* host = &host_model; +VkFormat unorm_of(VkFormat f) { return f; } +void reflex_swapchain_info(VkSwapchainCreateInfoKHR&) {} +bool reflex_swapchain_stale() { return false; } + +// ---- the model: a driver that counts its handles, and loses its surface when told +struct Model { + uint64_t serial = 1000; + std::set surfaces, swapchains, semaphores, views, fences; + std::vector order; // (what was destroyed or waited for, in order) + int bad = 0; + bool idle = false; // (the device was waited for since the last work) + bool in_flight = false; // (a frame names the swapchain's images) + // what the next call of each kind answers + VkResult acquire_result = VK_SUCCESS, present_result = VK_SUCCESS, create_result = VK_SUCCESS, caps_result = VK_SUCCESS, + surface_result = VK_SUCCESS; + VkBool32 can_present = VK_TRUE, own_queue_can_present = VK_TRUE; + long window_w = 640, window_h = 480; + int surfaces_made = 0, swapchains_made = 0, surface_tries = 0, support_asked = 0, acquires = 0, presents = 0, submits = 0, settles = 0; + int queue_chosen = 0, invalidated = 0, reflex_off = 0; + bool on_present_thread = false; + int thread_calls = 0; // (Vulkan calls made while the model is on the present thread, the present itself apart) +} m; +void fail(const char* why) { std::printf("MODEL: %s\n", why); m.bad++; } +void touch() { if (m.on_present_thread) m.thread_calls++; } +void queue_flush_locked() {} +void device_wait_idle_locked() { touch(); m.idle = true, m.in_flight = false; m.order.push_back("idle"); } +void reflex_swapchain_destroyed() { m.reflex_off++; m.order.push_back("reflex"); } +void swap_views_drop() { + touch(); + for (auto v : dev.swap_views) { + if (!m.idle) fail("a view destroyed with the device busy"); + if (!m.views.erase(v)) fail("a view destroyed twice"); + m.order.push_back("view"); + } + dev.swap_views.clear(); +} +void swap_views_make(unsigned, VkFormat) { + swap_views_drop(); + for (size_t i = 0; i < dev.swap_images.size(); i++) dev.swap_views.push_back(++m.serial), m.views.insert(m.serial); +} +void vkDestroySemaphore(int, VkSemaphore s, void*) { + touch(); + if (!m.idle) fail("a semaphore destroyed with the device busy"); + if (!m.semaphores.erase(s)) fail("a semaphore destroyed twice, or never made"); + m.order.push_back("semaphore"); +} +VkResult vkCreateSemaphore(int, const VkSemaphoreCreateInfo*, void*, VkSemaphore* out) { + *out = ++m.serial, m.semaphores.insert(*out); + return VK_SUCCESS; +} +void vkDestroySwapchainKHR(int, VkSwapchainKHR sc, void*) { + touch(); + if (!m.idle) fail("a swapchain destroyed with the device busy"); + if (!m.views.empty() && sc == dev.swapchain) fail("the swapchain destroyed before its views"); + if (!m.swapchains.erase(sc)) fail("a swapchain destroyed twice, or never made"); + m.order.push_back("swapchain"); +} +void vkDestroySurfaceKHR(uint64_t, VkSurfaceKHR s, void*) { + touch(); + if (!m.swapchains.empty()) fail("the surface destroyed before its swapchain"); + if (!m.surfaces.erase(s)) fail("a surface destroyed twice, or never made"); + m.order.push_back("surface"); +} +VkResult vkCreateSwapchainKHR(int, const VkSwapchainCreateInfoKHR* ci, void*, VkSwapchainKHR* out) { + if (!ci->surface || !m.surfaces.count(ci->surface)) fail("a swapchain asked for a surface that is not there"); + if (ci->oldSwapchain && !m.swapchains.count(ci->oldSwapchain)) fail("an old swapchain that is not there"); + if (m.create_result != VK_SUCCESS) return m.create_result; + *out = ++m.serial, m.swapchains.insert(*out), m.swapchains_made++; + m.idle = false; + return VK_SUCCESS; +} +VkResult vkGetSwapchainImagesKHR(int, VkSwapchainKHR, uint32_t* n, VkImage* out) { + *n = 4; + if (out) for (uint32_t i = 0; i < 4; i++) out[i] = 5000 + i; + return VK_SUCCESS; +} +VkResult vkGetPhysicalDeviceSurfaceCapabilitiesKHR(uint64_t, VkSurfaceKHR s, VkSurfaceCapabilitiesKHR* caps) { + if (!s || !m.surfaces.count(s)) fail("the capabilities asked of a surface that is not there"); + if (m.caps_result != VK_SUCCESS) return m.caps_result; + caps->currentExtent = {(uint32_t)m.window_w, (uint32_t)m.window_h}; + return VK_SUCCESS; +} +VkResult vkGetPhysicalDeviceSurfaceSupportKHR(uint64_t, uint32_t family, VkSurfaceKHR s, VkBool32* ok) { + if (!s || !m.surfaces.count(s)) fail("present support asked of a surface that is not there"); + m.support_asked++; + *ok = family == dev.queue_family ? m.can_present : m.own_queue_can_present; + return VK_SUCCESS; +} +// (the window: Windows' calls, and the SDL bridge's) +HWND main_window = (HWND)&m; +int GetClientRect(HWND, RECT* r) { *r = {0, 0, m.window_w, m.window_h}; return 1; } +void* GetModuleHandleA(const char*) { return nullptr; } +VkResult vkCreateWin32SurfaceKHR(uint64_t, const VkWin32SurfaceCreateInfoKHR* si, void*, VkSurfaceKHR* out) { + m.surface_tries++; + if (si->hwnd != main_window) fail("a surface for another window"); + if (m.window_w <= 0 || m.window_h <= 0) fail("a surface asked for a window with no size"); + if (m.surface_result != VK_SUCCESS) return m.surface_result; + *out = ++m.serial, m.surfaces.insert(*out), m.surfaces_made++; + return VK_SUCCESS; +} +MkwVkWindow sdl{2, + [](void*, void*, void* out) -> int { + m.surface_tries++; + if (m.window_w <= 0 || m.window_h <= 0) fail("a surface asked for a window with no size"); + if (m.surface_result != VK_SUCCESS) return 0; + *(VkSurfaceKHR*)out = ++m.serial, m.surfaces.insert(m.serial), m.surfaces_made++; + return 1; + }, + [](void*, int* w, int* h) { *w = (int)m.window_w, *h = (int)m.window_h; }, nullptr}; +const MkwVkWindow* prep_window = &sdl; +/// vk_main.cpp's, whole +@CHOOSE@ + +// ---- vk_main.cpp's own code +bool open_window_surface(bool log_failure); +@SURFACE_RECOVER@ +@INVALIDATE@ +@OPEN@ + +/// create_swapchain: its first step and its creation step as they are, the surface's formats and modes between them left out +bool create_swapchain(uint32_t width, uint32_t height) { +@CREATE_HEAD@ + if (!width || !height) return false; + uint32_t n = 0; + struct { int format = 1; } fmt; + VkSwapchainCreateInfoKHR ci; + ci.surface = dev.surface; + int mode = 0; +@CREATE_TAIL@ + dev.swap_generation++; // (create_swapchain's own, after its log line: check_source finds it there) + swap_views_make(ci.imageUsage, 1); + return true; +} + +VkResult vkAcquireNextImageKHR(int, VkSwapchainKHR sc, uint64_t, VkSemaphore, VkFence fence, uint32_t* index) { + m.acquires++; + if (!sc || !m.swapchains.count(sc)) fail("an acquire from a swapchain that is not there"); + if (m.acquire_result != VK_SUCCESS) return m.acquire_result; + if (fence) m.fences.insert(fence); + *index = 1; + return VK_SUCCESS; +} +VkResult vkCreateFence(int, const VkFenceCreateInfo*, void*, VkFence* out) { *out = ++m.serial; return VK_SUCCESS; } +VkResult vkResetFences(int, uint32_t, const VkFence*) { return VK_SUCCESS; } +int fence_waits = 0; +bool image_back = true; +VkResult vkWaitForFences(int, uint32_t, const VkFence* f, int, uint64_t) { + if (*f >= 900 && *f < 903) return VK_SUCCESS; + fence_waits++; + return image_back ? VK_SUCCESS : VK_TIMEOUT; +} + +// ---- vk_replay.cpp's own code +std::mutex present_image_mu; +void present_thread_settle() { m.settles++; m.order.push_back("settle"); } +void present_note_game(int64_t) {} +struct { std::atomic paced_ns{0}; } present_fig; // (the acquire's wait, counted as the pacing's) +@HELPERS@ +int recorded = 0; +/// present_image: its first step, its acquire and its hold as they are; then the blit's recording and submit stood in for +void present_image_model() { +@BEGIN@ + Frame& f = frames[frame_index]; + frame_index = (frame_index + 1) % 3; +@ACQUIRE@ + ++recorded, m.submits++, m.in_flight = true, m.idle = false; +} +/// present_out's last step as it is: the present's result handled +struct QueueLock { + bool held = false; + bool owns_lock() const { return held; } + void lock() { held = true; } + void unlock() { held = false; } +}; +void present_out_model(VkResult res, QueueLock& qk, bool on_thread) { + m.presents++; +@OUT@ +} + +static bool nothing_published() { return !dev.swapchain && !dev.surface && dev.swap_rendered.empty() && dev.swap_images.empty() && dev.swap_views.empty(); } +static bool nothing_left() { return m.surfaces.empty() && m.swapchains.empty() && m.semaphores.empty() && m.views.empty(); } +static bool whole() { return m.surfaces.size() == 1 && m.swapchains.size() == 1 && m.semaphores.size() == 4 && m.views.size() == 4 && + dev.surface && dev.swapchain && dev.swap_rendered.size() == 4; } +/// The teardown's order: the device idle, then every view, every semaphore, the swapchain, the surface, each once +static bool in_order(size_t from) { + const std::vector want = {"idle", "reflex", "view", "view", "view", "view", "semaphore", "semaphore", "semaphore", + "semaphore", "swapchain", "surface"}; + return m.order.size() == from + want.size() && std::equal(want.begin(), want.end(), m.order.begin() + from); +} +/// Past the 250 ms between two tries: the cut code reads the steady clock, so the tries' own time is moved back +static void later() { tried_back(); } + +int main(int argc, char** argv) { + const bool want_on = argc > 1 && argv[1][0] == 'r'; + { // the rule + using namespace surface_lost; + CHECK(on(nullptr) && on("") && on("1") && !on("0")); + CHECK(recovers(true, true, false) && !recovers(false, true, false) && !recovers(true, false, false) && !recovers(true, true, true)); + CHECK(kRetryMs == 250 && retry_due(1000, 0) && !retry_due(1000, 900) && !retry_due(1149, 900) && retry_due(1150, 900)); + } + CHECK(surface_recover(VK_ERROR_SURFACE_LOST_KHR) == want_on && !surface_recover(VK_SUCCESS) && !surface_recover(VK_ERROR_OUT_OF_DATE_KHR) && + !surface_recover(VK_ERROR_DEVICE_LOST)); + // the start, as start() does it: the surface, then the first swapchain + CHECK(open_window_surface(true) && create_swapchain(640, 480) && whole() && dev.swap_generation == 1); + present_image_model(); + CHECK(recorded == 1 && m.acquires == 1); + + if (!want_on) { // RX_VK_SURFACE_RECOVER=0: the result is not handled, as before + m.acquire_result = VK_ERROR_SURFACE_LOST_KHR; + const size_t o = m.order.size(); + for (int i = 0; i < 5; i++) present_image_model(); + CHECK(recorded == 1 && whole() && m.invalidated == 0 && m.surfaces_made == 1 && m.swapchains_made == 1); + QueueLock qk; + present_out_model(VK_ERROR_SURFACE_LOST_KHR, qk, false), present_out_model(VK_ERROR_SURFACE_LOST_KHR, qk, true); + CHECK(whole() && !surface_lost_noted.load()); + for (size_t i = o; i < m.order.size(); i++) CHECK(m.order[i] == "settle"); + CHECK(m.bad == 0); + std::puts("=0: a lost surface is not handled: nothing let go, nothing made"); + return fails ? 1 : 0; + } + + { // lost at the acquire + const size_t o = m.order.size(); + const int r = recorded; + m.acquire_result = VK_ERROR_SURFACE_LOST_KHR; + present_image_model(); + CHECK(m.order[o] == "settle" && in_order(o + 1) && nothing_published() && nothing_left() && recorded == r); + CHECK(dev.swap_resize.load() == ((uint64_t)640 << 32 | 480)); + m.acquire_result = VK_SUCCESS; + later(); + present_image_model(); // (the next present: a surface and a swapchain again, and a frame) + CHECK(whole() && recorded == r + 1 && m.surfaces_made == 2 && m.swapchains_made == 2 && dev.swap_generation == 2 && !dev.swap_resize.load()); + CHECK(m.queue_chosen == 2 && dev.present_queue == 55); // (the presents' own queue asked of the new surface) + } + { // lost at the present, on the presenting thread: let go there, the queue's lock taken + const size_t o = m.order.size(); + QueueLock qk; + present_out_model(VK_ERROR_SURFACE_LOST_KHR, qk, false); + CHECK(qk.held && in_order(o) && nothing_published() && nothing_left() && !surface_lost_noted.load()); + later(); + present_image_model(); + CHECK(whole() && m.surfaces_made == 3); + } + { // lost at the present, on the present thread: only noted there; the next present_image lets go and makes again + const size_t o = m.order.size(); + const int r = recorded; + QueueLock qk; + m.on_present_thread = true; + present_out_model(VK_ERROR_SURFACE_LOST_KHR, qk, true); + m.on_present_thread = false; + CHECK(m.thread_calls == 0 && !qk.held && whole() && surface_lost_noted.load() && m.order.size() == o); + later(); + present_image_model(); + CHECK(m.order[o] == "settle" && whole() && !surface_lost_noted.load() && recorded == r + 1 && m.surfaces_made == 4); + // (let go between the settle and the new swapchain, in order) + const std::vector head(m.order.begin() + o + 1, m.order.begin() + o + 13); + CHECK(head.front() == "idle" && head.back() == "surface"); + } + { // lost at the creation: the old swapchain went with the attempt, and the surface goes too + m.create_result = VK_ERROR_SURFACE_LOST_KHR; + dev.swap_resize.store((uint64_t)800 << 32 | 600); + const int r = recorded; + present_image_model(); + CHECK(nothing_published() && nothing_left() && recorded == r && dev.swap_resize.load() == ((uint64_t)800 << 32 | 600)); + m.create_result = VK_SUCCESS; + later(); + present_image_model(); + CHECK(whole() && recorded == r + 1); + } + { // lost at the capabilities' query + m.caps_result = VK_ERROR_SURFACE_LOST_KHR; + dev.swap_resize.store((uint64_t)800 << 32 | 600); + present_image_model(); + CHECK(nothing_published() && nothing_left()); + m.caps_result = VK_SUCCESS; + later(); + present_image_model(); + CHECK(whole()); + } + { // a window with no size: it waits, and nothing is asked of the driver for it + m.acquire_result = VK_ERROR_SURFACE_LOST_KHR; + present_image_model(); + m.acquire_result = VK_SUCCESS; + CHECK(nothing_published()); + m.window_w = 0, m.window_h = 0; + const int tries = m.surface_tries, r = recorded; + for (int i = 0; i < 20; i++) later(), present_image_model(); + CHECK(m.surface_tries == tries && nothing_published() && nothing_left() && recorded == r); + m.window_w = 1280, m.window_h = 720; + later(); + present_image_model(); + CHECK(whole() && recorded == r + 1 && dev.swap_extent.width == 1280 && dev.swap_extent.height == 720); + } + { // a surface that cannot be made: one try each 250 ms at most, the first 8 failures said + m.acquire_result = VK_ERROR_SURFACE_LOST_KHR; + present_image_model(); + m.acquire_result = VK_SUCCESS; + m.surface_result = -1; + const int tries = m.surface_tries; + later(); + const int lines = log_lines; + for (int i = 0; i < 50; i++) present_image_model(); // (50 presents inside the 250 ms) + CHECK(m.surface_tries == tries + 1 && log_lines == lines + 1 && nothing_published() && nothing_left()); + later(); + present_image_model(); + CHECK(m.surface_tries == tries + 2 && log_lines == lines + 2); + // ...and one the replay's queue cannot present to: let go again, nothing leaked + m.surface_result = VK_SUCCESS, m.can_present = VK_FALSE; + for (int i = 0; i < 20; i++) later(), present_image_model(); + CHECK(m.surface_tries == tries + 22 && nothing_published() && nothing_left() && log_lines == lines + 8); + m.can_present = VK_TRUE; + later(); + present_image_model(); + CHECK(whole()); + } + { // the presents' own queue cannot present to the new surface: the replay's queue from then on, as at the start + m.acquire_result = VK_ERROR_SURFACE_LOST_KHR; + present_image_model(); + m.acquire_result = VK_SUCCESS, m.own_queue_can_present = VK_FALSE; + later(); + present_image_model(); + CHECK(whole() && dev.present_queue == 0 && dev.present_family == dev.queue_family); + m.own_queue_can_present = VK_TRUE; + } + { // many losses in a row, at each call in turn: a steady handle count + const int made = m.surfaces_made; + for (int i = 0; i < 30; i++) { + if (i % 3 == 0) { + m.acquire_result = VK_ERROR_SURFACE_LOST_KHR; + present_image_model(); + m.acquire_result = VK_SUCCESS; + } else { + QueueLock qk; + present_out_model(VK_ERROR_SURFACE_LOST_KHR, qk, i % 3 == 2); + } + later(); + present_image_model(); + CHECK(whole()); + } + CHECK(m.surfaces_made == made + 30 && m.surfaces.size() == 1 && m.swapchains.size() == 1 && m.semaphores.size() == 4); + } + { // a held image (RX_VK_ACQUIRE_FENCE) of a swapchain that went with its surface: left behind, not looked at again + image_back = false; + present_image_model(); + CHECK(held_image.on && held_image.generation == dev.swap_generation); + const VkFence left = acquire_fence_now; + QueueLock qk; + present_out_model(VK_ERROR_SURFACE_LOST_KHR, qk, false); + image_back = true; + later(); + const int a = m.acquires, r = recorded; + present_image_model(); + CHECK(whole() && !held_image.on && m.acquires == a + 1 && recorded == r + 1 && acquire_fence_now != left); + } + { // NVIDIA DLSS frame generation runs: the swapchain is Streamline's, the result is not handled + dlssg_active = true; + const int s = m.surfaces_made; + m.acquire_result = VK_ERROR_SURFACE_LOST_KHR; + present_image_model(); + QueueLock qk; + present_out_model(VK_ERROR_SURFACE_LOST_KHR, qk, false); + CHECK(whole() && m.surfaces_made == s && !surface_lost_noted.load()); + m.acquire_result = VK_SUCCESS, dlssg_active = false; + } + CHECK(m.bad == 0 && m.thread_calls == 0); + std::printf("recovered: %d surfaces and %d swapchains made, one of each left; %d presents drew\n", m.surfaces_made, m.swapchains_made, recorded); + return fails ? 1 : 0; +} +''' + +# (the cut's own clock: create_swapchain reads the steady clock for its 250 ms. The test moves the last try's time back +# instead of sleeping: its static is made the fixture's own variable, the statement that reads it left as it is) +TRIED = " static int64_t tried_ms;\n" + + +def cut_source(): + main = (VK / "vk_main.cpp").read_text(encoding="utf-8") + replay = (VK / "vk_replay.cpp").read_text(encoding="utf-8") + swap = block(main, "bool create_swapchain(uint32_t width, uint32_t height) {") + parts = {} + parts["SURFACE_RECOVER"] = block(main, "bool surface_recover(VkResult result) {") + parts["INVALIDATE"] = block(main, "void invalidate_lost_surface() {") + parts["OPEN"] = block(main, "bool open_window_surface(bool log_failure) {") + parts["CHOOSE"] = block(main, "static void choose_present_queue() {").replace( + "static void choose_present_queue() {", "static void choose_present_queue() {\n m.queue_chosen++;", 1) + head = between(swap, " // ((Triforce, 11-22, PRESENT-45) no surface:", " logf(\"vk: surface caps images", inclusive=False) + check(head.count(TRIED) == 1, "create_swapchain's retry clock changed") + parts["CREATE_HEAD"] = head.replace(TRIED, " int64_t& tried_ms = tried_ms_model;\n") + parts["CREATE_TAIL"] = between(swap, " ci.oldSwapchain = dev.swapchain;", " logf(\"vk: swapchain %ux%u", inclusive=False) + image = block(replay, "void present_image(VkImage src, uint32_t sw, uint32_t sh, const int crop[4], VkImageView view) {") + parts["BEGIN"] = between(image, " std::lock_guard lk(present_image_mu);", " if (src) {", inclusive=False) + parts["ACQUIRE"] = between(image, " uint32_t index = 0;\n", " if (vkResetFences(dev.device, 1, &f.done) == VK_ERROR_DEVICE_LOST) device_lost();\n") + # (RX_VK_ACQUIRE_FENCE's helpers, from its switch to the note of a surface lost on the present thread) + helpers = between(replay, "bool acquire_fence_on() {", "std::atomic surface_lost_noted{false};") + "\n" + check("std::atomic surface_lost_noted{false};" in replay, "vk_replay.cpp has no note of a surface lost on the present thread") + parts["HELPERS"] = helpers + out = block(replay, "static void present_out(const PresentOut& o, std::unique_lock& qk, bool on_thread) {") + parts["OUT"] = between(out, " if (surface_recover(res)) {", " if (res == VK_ERROR_OUT_OF_DATE_KHR || res == VK_SUBOPTIMAL_KHR) {", + inclusive=False) + return main, replay, swap, image, out, parts + + +def check_source(main, replay, swap, image, out): + # the default path: every new statement is behind the lost result or the missing surface + check(swap.count("surface_recover(result)") == 3 and swap.count("invalidate_lost_surface()") == 3, + "create_swapchain does not hand a lost surface on at its two queries and its creation") + for line in swap.splitlines(): + if "invalidate_lost_surface()" in line: + check(line.strip().startswith("if (surface_recover(result)) return invalidate_lost_surface(), false;"), + "create_swapchain lets go of the surface for another result than a lost surface") + check(swap.index(" if (!dev.surface) {") < swap.index("vkGetPhysicalDeviceSurfaceCapabilitiesKHR(") and + swap.count("open_window_surface(") == 1 and swap.index("open_window_surface(") < swap.index("vkGetPhysicalDeviceSurfaceCapabilitiesKHR("), + "create_swapchain makes a surface anywhere but first, for a missing one") + check(image.count("invalidate_lost_surface()") == 2 and out.count("invalidate_lost_surface()") == 1, + "present_image or present_out lets go of the surface in more places than the acquire, the note and the present") + at = [image.index(p) for p in ("present_thread_settle();", "if (surface_lost_noted.exchange(false, std::memory_order_acq_rel)) {", + "dev.swap_resize.exchange(0", "vkAcquireNextImageKHR(", "if (surface_recover(res)) {", + "if (res == VK_ERROR_OUT_OF_DATE_KHR) {", "if (vkResetFences(dev.device, 1, &f.done)")] + check(at == sorted(at), "present_image does not settle the present thread before it lets go, or handles the loss after it records") + thread = out[out.index(" if (surface_recover(res)) {"):out.index(" if (res == VK_ERROR_OUT_OF_DATE_KHR || res == VK_SUBOPTIMAL_KHR) {")] + check(thread.index("if (on_thread) {") < thread.index("surface_lost_noted.store(true") < thread.index("return;") < thread.index("invalidate_lost_surface();"), + "the present thread lets go of the swapchain itself") + recover = block(main, "bool surface_recover(VkResult result) {") + check('static const bool on = surface_lost::on(getenv("RX_VK_SURFACE_RECOVER"));' in recover and + "surface_lost::recovers(result == VK_ERROR_SURFACE_LOST_KHR, on, dlssg::active())" in recover, "RX_VK_SURFACE_RECOVER is not read once") + check(swap.count("dev.swap_generation++;") == 1 and swap.index(" dev.swapchain = sc;") < swap.index("dev.swap_generation++;"), + "create_swapchain does not count a new swapchain's generation where it publishes it") + # start() still makes the first surface itself, as it did + start = block(main, "int start(void* hwnd) {") + check("open_window_surface" not in start and "vkCreateWin32SurfaceKHR(dev.instance, &si, nullptr, &dev.surface)" in start and + "w->create_surface(w->window, (void*)dev.instance, (void*)&dev.surface)" in start and start.count("choose_present_queue();") == 2, + "start() was changed: the first surface is the startup's own code") + check(main.count("vkDestroySurfaceKHR(") == 2, "the surface is destroyed somewhere else than the two places of the recovery") + docs = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("| `RX_VK_SURFACE_RECOVER=0` |" in docs and "tests/test_present_surface_lost.py" in docs, "docs/development.md has no row for RX_VK_SURFACE_RECOVER") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + main_text, replay, swap, image, out, parts = cut_source() + check_source(main_text, replay, swap, image, out) + env = {k: v for k, v in os.environ.items() if k not in ("RX_VK_SURFACE_RECOVER", "RX_VK_ACQUIRE_FENCE")} + with tempfile.TemporaryDirectory(prefix="present-surface-lost-") as tmp: + for name in ("windows", "not-windows"): + text = FIXTURE + for key, value in parts.items(): + text = text.replace("@" + key + "@", value) + # (each platform's branch compiled here as code, whatever this PC is) + win = name == "windows" + text = text.replace("#ifdef _WIN32\n", "#if 1\n" if win else "#if 0\n").replace("#ifndef _WIN32\n", "#if 0\n" if win else "#if 1\n") + text = text.replace("// ---- vk_main.cpp's own code\n", + "// ---- vk_main.cpp's own code\nint64_t tried_ms_model = 0;\nvoid tried_back() { tried_ms_model -= 1000; }\n") + src, exe = Path(tmp) / (name + ".cpp"), Path(tmp) / (name + ".exe") + src.write_text(text, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-I", str(VK), str(src), "-o", str(exe)], check=True) + for value, mode in ((None, "recover"), ("1", "recover"), ("0", "off")): + run_env = dict(env) + if value is not None: + run_env["RX_VK_SURFACE_RECOVER"] = value + got = subprocess.run([str(exe), mode], env=run_env, stdout=subprocess.PIPE, text=True, timeout=60) + if got.returncode: + print(got.stdout) + raise SystemExit("FAIL: the %s branch with RX_VK_SURFACE_RECOVER %s" % (name, "unset" if value is None else "=" + value)) + if value != "1": + print("%s branch, RX_VK_SURFACE_RECOVER %s: %s" % (name, "unset" if value is None else "=" + value, + got.stdout.strip().splitlines()[-1])) + print("PASS: a lost surface at the acquire, the present, the creation or the query is let go in order and made again at the " + "next present; =0 and DLSS frame generation leave it as before") + + +if __name__ == "__main__": + main() diff --git a/tests/test_present_trace.py b/tests/test_present_trace.py index 3a54cc2..ddb1ea9 100644 --- a/tests/test_present_trace.py +++ b/tests/test_present_trace.py @@ -49,6 +49,9 @@ static void hos_log(const char *fmt, ...) { (void)fmt; } static uint64_t hos_ticks(void) { return 0; } static int fps_max(void) { return 60; } /* (nn_vi.c's Frame rate cap: the console's 60 here) */ static int64_t steady_period(int64_t base, int64_t now) { (void)now; return base; } /* ((Triforce) the Steady frame rate row off) */ +static void movie_pace_decide(void) {} /* ((Triforce, 11-22, PRESENT-30) no movie plays here: paced by the cap) */ +static int pace_fps(void) { return fps_max(); } +void hos_bench_present(int64_t now) { (void)now; } /* (benchmark mode's note of each present: not on here) */ ''' SUFFIX = r''' @@ -87,6 +90,7 @@ def main(): args = parser.parse_args() source = (args.source / "host/nn_vi.c").read_text(encoding="utf-8") helper = block(source, "static void present_trace(") if "static void present_trace(" in source else "" + helper += "\n" + block(source, "static int cmp_i64_desc(") # (the 1% low's sort, which the cut-out present calls) actual = block(source, "void hos_vi_native_presented(") states = [] with tempfile.TemporaryDirectory(prefix="present-trace-") as tmp: diff --git a/tests/test_prompt_guides.py b/tests/test_prompt_guides.py new file mode 100644 index 0000000..2df38c3 --- /dev/null +++ b/tests/test_prompt_guides.py @@ -0,0 +1,76 @@ +"""The pause menu controls page's leader lines on each face's controller picture (nx2/prompt_assets.py guide_layers), with no +game file: the game's line textures stand in as black images of their size. + +- Every ControllerGuide_Pro texture the page draws lines on over the Pro Controller is made again for the faces (GUIDES): + ZL, Minus, the sticks, Plus and the face buttons on _00 and _01, ZR on _02, the D-pad's frame on _03, L and R on _04. Left + the game's, a face's picture had them at the Pro's places: on the PlayStation pads, whose left stick is where the Pro's + d-pad is, the D-pad's frame went round the left stick ("Put d-pad arrows in the right place"). +- For each drawn face (PS4, PS5, Xbox, Steam Deck): the D-pad's label (Switch Up, on the d-pad) is a frame round that + face's own d-pad, clear of its left stick; L, R and ZR's lines end at that face's shoulder and trigger. + + python -I -B tests/test_prompt_guides.py +""" +import sys +from pathlib import Path + +RUN_ALL_MODULES = ["PIL", "numpy"] + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "nx2")) +import prompt_assets as P # noqa: E402 +from prompt_faces import FACES # noqa: E402 + +# the textures with the page's lines over the Pro Controller (UI/Layout2Archive/Common's bntx), and their size +PRO_GUIDES = ["ControllerGuide_Pro_0%d" % i for i in range(5)] +SIZE = (1555, 1511) + + +def check(cond, what): + if not cond: + raise SystemExit("FAIL: " + what) + + +def ink_box(im): + box = im.point(lambda v: 255 if v > 127 else 0).getbbox() + check(box is not None, "a line with no ink") + return box + + +def main(): + from PIL import Image + for name in PRO_GUIDES: + check(name in P.GUIDES, f"{name} is not made for the faces: its lines point at the Pro Controller's places") + tex = {name: i for i, name in enumerate(P.GUIDES)} + game = [Image.new("L", SIZE, 0) for _ in P.GUIDES] + faces = [f for f in FACES if P.parts(f)] + check({"ps4", "ps5", "xbox", "steamdeck"} <= set(faces), "a drawn face missing: " + ", ".join(faces)) + for face in faces: + where = P.parts(face) + keep, moving = P.guide_layers(game, face) + by = {(tx, end): lines for tx, entries in moving.items() for end, lines in entries} + dpad = by.get((tex["ControllerGuide_Pro_03"], "up")) + check(dpad is not None, f"{face}: no D-pad label on ControllerGuide_Pro_03") + cx, cy, e = P.guide_frame_at(where, ("up", "left", "right", "down")) + x0, y0, x1, y1 = ink_box(dpad["up"]) + # (the frame's right and top corners: the box's far side from the label's bar, at the left, and its top, the line + # coming from the bar's row below) + check(abs(x1 - (cx + e)) < 12 and abs(y0 - (cy - e)) < 12, + f"{face}: the D-pad's frame reaches x {x1} and y {y0}; the face's d-pad is at ({cx:.0f}, {cy:.0f}), corners " + f"{e:.0f} off") + sx, sy, sr = P.guide_at(where["lstick"]) + check(dpad["up"].getpixel((round(sx), round(sy))) == 0, f"{face}: the D-pad's frame is over the left stick") + check(dpad["down"] is dpad["up"], f"{face}: the d-pad's arms have frames of their own") + for tx, end, pad in ((tex["ControllerGuide_Pro_04"], "l", "lshoulder"), (tex["ControllerGuide_Pro_04"], "r", "rshoulder"), + (tex["ControllerGuide_Pro_02"], "zr", "rtrigger")): + lines = by.get((tx, end)) + check(lines is not None, f"{face}: no {end} label on {P.GUIDES[tx]}") + x, y, _ = P.guide_at(where[pad]) + im = lines[pad] + near = im.crop((round(x) - 20, round(y) - 30, round(x) + 21, round(y) + 11)) + check(near.getbbox() is not None, f"{face}: {end}'s line does not reach the {pad} at ({x:.0f}, {y:.0f})") + print(f"{face}: the D-pad's frame round ({cx:.0f}, {cy:.0f}), L, R and ZR at its shoulders") + print("ok") + + +if __name__ == "__main__": + main() diff --git a/tests/test_prompt_widen_cache.py b/tests/test_prompt_widen_cache.py new file mode 100644 index 0000000..62d120c --- /dev/null +++ b/tests/test_prompt_widen_cache.py @@ -0,0 +1,123 @@ +"""The button prompts' guide lines widen the lines that stay once per image, not once per route (nx2/prompt_assets.py +guide_widened, guide_route). + +(Triforce, 11-22, the performance audit's IO-42, after Source A's.) guide_route grew the "lines that stay" image by a 9 +by 9 maximum at every call, and every route of one line (the groups' frames, each pad input's line) is given the same +image. guide_widened keeps the last image's array: one entry, used only when it holds that very image. +HOOKSHOT_PROMPTS_WIDEN_EACH=1 widens at every route, as before. + +Game-free, on blank images of the guide's size (no game file is read): + - through guide_layers for the xbox face, the 9 by 9 maximum is made once per route with the switch (as before) and at + most 5 times by default, once for each image that differs; + - what guide_layers returns is the same either way, byte for byte: the lines that stay and every pad input's line, + for the xbox face and the Switch face; + - asked twice for one image the second answer is the kept array, with no filter; an entry that holds another image + under this image's id misses and is replaced; the cache never holds more than one image; + - guide_route asks guide_widened, and the switches table has the switch. + + python -I -B tests/test_prompt_widen_cache.py +""" +import os +import sys +from pathlib import Path +from unittest import mock + +RUN_ALL_MODULES = ["numpy", "PIL"] + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "nx2")) +import prompt_assets # noqa: E402 + +SIZE = (1552, 1440) # (the bars at 147 and 1407, the rows down to 1225: prompt_assets.BARS, GUIDE_LINES) + + +def check(cond, what): + if not cond: + raise SystemExit("FAIL: " + what) + + +def layers(face, each): + """guide_layers on blank game images: (what it returned as bytes, the 9 by 9 maxima made, the routes asked)""" + from PIL import Image, ImageFilter + game = [Image.new("L", SIZE, 0) for _ in prompt_assets.GUIDES] + made, routes = [], [] + real_filter, real_route = ImageFilter.MaxFilter, prompt_assets.guide_route + + def counting(size=3): + made.append(size) + return real_filter(size) + + def route(*a, **k): + routes.append(a[4]) + return real_route(*a, **k) + + prompt_assets._WIDENED.clear() + with mock.patch.object(prompt_assets, "WIDEN_EACH", each), mock.patch.object(ImageFilter, "MaxFilter", counting), \ + mock.patch.object(prompt_assets, "guide_route", route): + keep, moving = prompt_assets.guide_layers(game, face) + out = {"keep": [im.tobytes() for im in keep]} + for tx, ends in moving.items(): + for end, lines in ends: + for pad, im in lines.items(): + out["%d %s %s" % (tx, end, pad)] = im.tobytes() + return out, made.count(9), len(routes) + + +def main(): + import numpy as np + from PIL import Image, ImageFilter + + # ---- through guide_layers: fewer maxima, the same layers + for face in ("xbox", "switch"): + before, widened_before, routes_before = layers(face, True) + now, widened_now, routes_now = layers(face, False) + check(routes_before == routes_now and routes_now > 20, "%s: %d and %d routes" % (face, routes_before, routes_now)) + check(widened_before == routes_before, "%s: with the switch %d maxima for %d routes, not one each" + % (face, widened_before, routes_before)) + check(widened_now <= 5, "%s: %d maxima by default, more than the 5 images that differ" % (face, widened_now)) + check(sorted(before) == sorted(now), "%s: the layers' names differ" % face) + differ = [name for name in before if before[name] != now[name]] + check(not differ, "%s: layers differ from the ones widened at every route: %s" % (face, differ[:6])) + check(any(any(layer) for name, layer in now.items() if name != "keep"), "%s: no line was drawn at all" % face) + if face == "xbox": + counts = (widened_before, widened_now) + check(len(prompt_assets._WIDENED) <= 1, "the cache holds more than one image") + + # ---- the cache itself + a, b = Image.new("L", (64, 48), 0), Image.new("L", (64, 48), 0) + a.putpixel((10, 10), 255) + b.putpixel((40, 30), 255) + want_a, want_b = np.asarray(a.filter(ImageFilter.MaxFilter(9))), np.asarray(b.filter(ImageFilter.MaxFilter(9))) + prompt_assets._WIDENED.clear() + with mock.patch.object(prompt_assets, "WIDEN_EACH", False): + first = prompt_assets.guide_widened(a) + check(np.array_equal(first, want_a) and first[6:15, 6:15].all() and first.sum() == 81 * 255, "not a 9 by 9 maximum") + with mock.patch.object(ImageFilter, "MaxFilter", side_effect=AssertionError("widened again")): + check(prompt_assets.guide_widened(a) is first, "the same image was not answered from the cache") + # an entry under b's id that holds another image: it must miss, and be replaced by b's own + prompt_assets._WIDENED.clear() + prompt_assets._WIDENED[id(b)] = (a, first) + got = prompt_assets.guide_widened(b) + check(np.array_equal(got, want_b) and not np.array_equal(got, first), "an entry holding another image was used") + check(list(prompt_assets._WIDENED) == [id(b)] and prompt_assets._WIDENED[id(b)][0] is b, "the entry was not replaced") + prompt_assets.guide_widened(a) + check(list(prompt_assets._WIDENED) == [id(a)], "the cache keeps more than the last image") + with mock.patch.object(prompt_assets, "WIDEN_EACH", True): + prompt_assets._WIDENED.clear() + one, two = prompt_assets.guide_widened(a), prompt_assets.guide_widened(a) + check(one is not two and np.array_equal(one, two) and not prompt_assets._WIDENED, "the switch still keeps an array") + + # ---- the source and the docs + source = (ROOT / "nx2" / "prompt_assets.py").read_text(encoding="utf-8") + route = source[source.index("def guide_route("):source.index("def _square(")] + check("st = guide_widened(stays)" in route and "MaxFilter" not in route, "guide_route widens the image itself again") + check('WIDEN_EACH = os.environ.get("HOOKSHOT_PROMPTS_WIDEN_EACH", "0") not in ("", "0")' in source, "the switch is not read") + check("if held is not None and held[0] is stays:" in source, "the cache is not checked by the image itself") + docs = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + rows = [line for line in docs.splitlines() if line.startswith("| `HOOKSHOT_PROMPTS_WIDEN_EACH=1` |")] + check(len(rows) == 1 and "IO-42" in rows[0], "docs/development.md has no row for HOOKSHOT_PROMPTS_WIDEN_EACH") + print("ok: the xbox face's lines that stay are widened %d times, not %d; the layers are the same bytes" % (counts[1], counts[0])) + + +if __name__ == "__main__": + main() diff --git a/tests/test_prompts_background.py b/tests/test_prompts_background.py index 3df6067..e1f2932 100644 --- a/tests/test_prompts_background.py +++ b/tests/test_prompts_background.py @@ -114,12 +114,87 @@ class PromptsInTheBackground(unittest.TestCase): keys = (ROOT / "nx2/prompt_keys.py").read_text(encoding="utf-8") self.assertIn('"third_party", "fonts", "libertinus", "LibertinusSans-Bold.ttf"', keys) - def test_the_players_own_folders_are_drawn_at_every_build(self): - (self.out / ".triforce_romfs").unlink() # (romfs and exefs folders linked in: no stamp to tell them by) - self.assertIsNone(make_game.prompts_key(str(self.out))) - self.assertEqual(self.build()[1:], (True, "drawn in run 1")) - self.assertEqual(self.build()[1:], (True, "drawn in run 2")) - self.assertFalse((self.out / "prompts" / make_game.PROMPT_KEY).exists()) + def game_files(self): + """a romfs with every file the tool reads, as stand-ins; their names under the romfs""" + import prompt_assets + romfs = self.out / "romfs" + names = ["Pack/ZsDic.pack.zs", prompt_assets.FONT, prompt_assets.TEXTURES, prompt_assets.LAYOUTS, prompt_assets.CAMERA, + "System/Resource/ResourceSizeTable.Product.110.rsizetable.zs", + prompt_assets.MALS + "/EUen.Product.110.sarc.zs", prompt_assets.MALS + "/JPja.Product.110.sarc.zs"] + for n in names: + (romfs / n).parent.mkdir(parents=True, exist_ok=True) + (romfs / n).write_bytes(("the game's " + n).encode()) + return names + + def test_the_players_own_folders_are_kept_by_their_files_content(self): + # ((11-21, the performance audit's IO-43) romfs and exefs folders linked in have no stamp to tell them by: the + # prompts were drawn at every build. Their key is the content of the game's files the tool reads) + (self.out / ".triforce_romfs").unlink() + self.assertIsNone(make_game.prompts_key(str(self.out))) # (no romfs given: no key, as before) + names = self.game_files() + self.assertEqual(make_game.prompt_game_files(str(self.out / "romfs")), names) + root = Path(self.temp.name) + (root / "host").mkdir() + (root / "host" / "prompts.c").write_text("/* reads the prompts */\n") + with patch.object(make_game, "ROOT", self.temp.name): + self.assertEqual(self.build()[1:], (True, "drawn in run 1")) + self.assertEqual(self.build()[1:], (False, "drawn in run 1")) # (the same folders again: kept) + self.assertTrue((self.out / "prompts" / make_game.PROMPT_KEY).exists()) + run = 1 + for n in names: # (each file the tool reads, changed: drawn again, then kept) + (self.out / "romfs" / n).write_bytes(b"another dump's") + run += 1 + self.assertEqual(self.build()[1:], (True, "drawn in run %d" % run), n) + self.assertEqual(self.build()[1:], (False, "drawn in run %d" % run), n) + (self.out / "romfs" / "Model").mkdir() + (self.out / "romfs" / "Model" / "Link.bfres.zs").write_bytes(b"a file the tool does not read") + self.assertEqual(self.build()[1:], (False, "drawn in run %d" % run)) + (self.out / "romfs" / names[-1]).unlink() # (a language gone) + run += 1 + self.assertEqual(self.build()[1:], (True, "drawn in run %d" % run)) + (root / "host" / "prompts.c").write_text("/* reads the prompts, changed */\n") + run += 1 + self.assertEqual(self.build()[1:], (True, "drawn in run %d" % run)) + (root / "version.txt").write_text("v9.9.9\n") # (another release of the tools) + run += 1 + self.assertEqual(self.build()[1:], (True, "drawn in run %d" % run)) + self.assertEqual(self.build()[1:], (False, "drawn in run %d" % run)) + with patch.dict(os.environ, {"HOOKSHOT_PROMPTS_FOLDER_KEY": "0"}): # (the switch: at every build, as before) + self.assertIsNone(make_game.prompts_key(str(self.out), str(self.out / "romfs"))) + self.assertEqual(self.build()[1:], (True, "drawn in run %d" % (run + 1))) + self.assertEqual(self.build()[1:], (True, "drawn in run %d" % (run + 2))) + + def test_an_unpacked_games_key_holds_the_reader_and_the_release_too(self): + # ((11-21, IO-43) for every game: host/prompts.c, which reads what is drawn, and the release these tools are) + root = Path(self.temp.name) + (root / "host").mkdir() + (root / "host" / "prompts.c").write_text("/* reads the prompts */\n") + with patch.object(make_game, "ROOT", self.temp.name): + self.assertEqual(self.build()[1:], (True, "drawn in run 1")) + self.assertEqual(self.build()[1:], (False, "drawn in run 1")) + (root / "host" / "prompts.c").write_text("/* reads the prompts, changed */\n") + self.assertEqual(self.build()[1:], (True, "drawn in run 2")) + (root / "VERSION").write_text("v9.9.9\n") + self.assertEqual(self.build()[1:], (True, "drawn in run 3")) + self.assertEqual(self.build()[1:], (False, "drawn in run 3")) + + def test_every_game_file_the_tool_opens_is_in_the_key(self): + # ((11-21, IO-43) nx2/prompt_assets.py's own source: each place it opens a file under the romfs names it by one of + # the names prompt_game_files gives. A new read there has to join the key, or a changed dump keeps old prompts) + import re + src = (ROOT / "nx2/prompt_assets.py").read_text(encoding="utf-8") + opened = set(re.findall(r"read_zs\((?:a\.)?romfs, ([^,]+),", src)) + # ("rel" is read_zs's own definition) + self.assertEqual(opened, {"rel", "FONT", "TEXTURES", "LAYOUTS", "CAMERA", 'f"System/Resource/{name}"', 'f"{MALS}/{name}"'}, + "prompt_assets.py reads a game file this test does not know: add it to make_game.prompt_game_files") + joined = set(re.findall(r"os\.path\.join\(romfs, ([^)]+)\)", src)) + self.assertEqual(joined, {'"Pack", "ZsDic.pack.zs"', "rel", '"System", "Resource"', "MALS"}, + "prompt_assets.py opens a path under the romfs this test does not know") + key = (ROOT / "nx2/make_game.py").read_text(encoding="utf-8") + body = key[key.index("def prompt_game_files(romfs):"):key.index("def prompts_key(")] + for part in ('"Pack/ZsDic.pack.zs"', "prompt_assets.FONT", "prompt_assets.TEXTURES", "prompt_assets.LAYOUTS", + "prompt_assets.CAMERA", '"ResourceSizeTable."', "prompt_assets.MALS", '".sarc.zs"'): + self.assertIn(part, body) def test_a_drawing_that_fails_leaves_no_prompts(self): self.build() diff --git a/tests/test_push_state.py b/tests/test_push_state.py index 8ac381c..4f831c2 100644 --- a/tests/test_push_state.py +++ b/tests/test_push_state.py @@ -34,7 +34,7 @@ def main(): if "RX_VK_PUSH_STATE_OLD" not in bind: raise SystemExit("the old bookkeeping has no switch") full = bind[: bind.index("static thread_local std::vector changed")] - if "ps.words.clear(), ps.at.clear()" not in full or "ps.n = dw().writes.size()" not in full: + if "ps.words.clear(), ps.at.clear()" not in full or "ps.n = dwr.writes.size()" not in full: raise SystemExit("a full push does not remember its words") fixture = f'''#include #include diff --git a/tests/test_radv_gcn.py b/tests/test_radv_gcn.py new file mode 100644 index 0000000..4b8dbe5 --- /dev/null +++ b/tests/test_radv_gcn.py @@ -0,0 +1,120 @@ +"""RADV on a GCN 1 to 4 GPU told apart (gpu/vk/vk_vendor.h radv_gcn), on made-up devices; no game, no GPU. + +(Triforce, 11-22, the performance audit's MEM-23 and PIPE-56, after Source A's.) Source A tells AMD's GCN 1 to 4 GPUs +(HD 7000 to RX 500) under Mesa's RADV by RADV reporting no 16-bit storage input and output there; Vega and later have it. +This is the fact alone (VendorFacts::storage_io16, radv_gcn(), dev.radv_gcn): nothing is decided by it in this change. + +Compiles radv_gcn() with the Vulkan headers and runs it on the properties of GPUs players have: + RADV without 16-bit storage I/O (an RX 580, an R9 380) -> GCN + RADV with it (a Steam Deck, an RX 6800, a Vega 56) -> not + AMD's Windows driver and AMDVLK, with or without it -> not (not RADV) + NVIDIA's driver, NVK, Intel's Windows driver, ANV, llvmpipe, without it too -> not + facts made without the feature (the default) -> not, on any driver +and that vendor_fixes() is the same for a device whichever way the new fact is set. + +And reads the source: create_device fills the fact from VkPhysicalDeviceVulkan11Features and sets dev.radv_gcn before +the device is made, and logs only where it is true (so no other device's log changes). + + python -I -B tests/test_radv_gcn.py --headers [--cxx clang++] +""" +import argparse +from pathlib import Path +import subprocess +import tempfile + +RUN_ALL_ARGS = ["--cxx", "{cxx}", "--headers", "{vulkan_headers}"] + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" + +FIXTURE = r""" +#include +#include "vk_vendor.h" +using namespace mkwvk; + +static int checks, fails; +static void want(const char* what, bool got, bool wanted) { + checks++; + if (got != wanted) fails++, std::printf("FAIL %s: %d, want %d\n", what, (int)got, (int)wanted); +} + +int main() { + struct D { const char* name; uint32_t vendor; VkDriverId driver; VkPhysicalDeviceType type; const char* info; bool io16, gcn; }; + const D devices[] = { + {"RX 580, RADV", 0x1002, VK_DRIVER_ID_MESA_RADV, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, "Mesa 26.2.5", false, true}, + {"R9 380, RADV", 0x1002, VK_DRIVER_ID_MESA_RADV, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, "Mesa 25.3.0", false, true}, + {"Steam Deck, RADV", 0x1002, VK_DRIVER_ID_MESA_RADV, VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU, "Mesa 25.1.0", true, false}, + {"RX 6800, RADV", 0x1002, VK_DRIVER_ID_MESA_RADV, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, "Mesa 26.2.5", true, false}, + {"Vega 56, RADV", 0x1002, VK_DRIVER_ID_MESA_RADV, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, "Mesa 26.2.5", true, false}, + {"RX 580, AMD's Windows driver", 0x1002, VK_DRIVER_ID_AMD_PROPRIETARY, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, + "26.5.2 (AMD proprietary shader compiler)", false, false}, + {"RX 6700 XT, AMD's Windows driver", 0x1002, VK_DRIVER_ID_AMD_PROPRIETARY, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, + "26.10.1 (AMD proprietary shader compiler)", true, false}, + {"RX 580, AMDVLK", 0x1002, VK_DRIVER_ID_AMD_OPEN_SOURCE, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, "2025.Q2.1 (LLPC)", false, false}, + {"RTX 5090, NVIDIA's Windows driver", 0x10DE, VK_DRIVER_ID_NVIDIA_PROPRIETARY, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, "581.57", true, false}, + {"an NVIDIA card whose driver reported none", 0x10DE, VK_DRIVER_ID_NVIDIA_PROPRIETARY, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, "581.57", false, false}, + {"GTX 1650, NVK", 0x10DE, VK_DRIVER_ID_MESA_NVK, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, "Mesa 26.0.1", false, false}, + {"Arc B580, Intel's Windows driver", 0x8086, VK_DRIVER_ID_INTEL_PROPRIETARY_WINDOWS, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, "101.8132", false, false}, + {"Iris Xe, ANV", 0x8086, VK_DRIVER_ID_INTEL_OPEN_SOURCE_MESA, VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU, "Mesa 26.0.1", false, false}, + {"llvmpipe", 0x10005, VK_DRIVER_ID_MESA_LLVMPIPE, VK_PHYSICAL_DEVICE_TYPE_CPU, "Mesa 26.0.1", false, false}, + }; + for (const D& d : devices) { + VendorFacts f; + f.vendor_id = d.vendor, f.driver_id = d.driver, f.device_type = d.type, f.driver_info = d.info; + want("facts without the feature filled name no GCN card", radv_gcn(f), false); + const VendorFixes before = vendor_fixes(f); + f.storage_io16 = d.io16; + want(d.name, radv_gcn(f), d.gcn); + const VendorFixes after = vendor_fixes(f); + want("the vendor fixes do not read the new fact", + before.fetch_bounds == after.fetch_bounds && before.nan_bits == after.nan_bits && before.gather_offset == after.gather_offset, true); + } + if (fails) std::printf("radv gcn: FAILED (%d of %d checks)\n", fails, checks); + else std::printf("radv gcn: all %d checks passed\n", checks); + return fails != 0; +} +""" + + +def check_source(): + main = (VK / "vk_main.cpp").read_text(encoding="utf-8") + order = [main.find(n) for n in ( + "vkGetPhysicalDeviceFeatures2(dev.physical, &f);", + "facts.storage_io16 = f11.storageInputOutput16 != VK_FALSE;", + "dev.radv_gcn = radv_gcn(facts);", + 'if (dev.radv_gcn) logf("vk: RADV on a GCN 1 to 4 GPU', + "vkCreateDevice(dev.physical, &ci")] + if min(order) < 0 or order != sorted(order): + raise SystemExit("FAIL: create_device does not fill the 16-bit storage I/O fact and dev.radv_gcn after the features are " + "asked and before the device is made, logging only where it is true") + if main.count("dev.radv_gcn =") != 1: + raise SystemExit("FAIL: dev.radv_gcn is set in more than one place") + internal = (VK / "vk_internal.h").read_text(encoding="utf-8") + if "bool radv_gcn = false;" not in internal: + raise SystemExit("FAIL: vk_internal.h: Device has no radv_gcn, false until create_device says otherwise") + vendor = (VK / "vk_vendor.h").read_text(encoding="utf-8") + if "bool storage_io16 = true;" not in vendor or \ + "return f.driver_id == VK_DRIVER_ID_MESA_RADV && !f.storage_io16;" not in vendor: + raise SystemExit("FAIL: vk_vendor.h: radv_gcn is not RADV without 16-bit storage input and output") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--headers", type=Path, required=True) + parser.add_argument("--cxx", default="g++") + args = parser.parse_args() + check_source() + with tempfile.TemporaryDirectory(prefix="radv-gcn-") as tmp: + tmp = Path(tmp) + cpp, exe = tmp / "fixture.cpp", tmp / "fixture.exe" + cpp.write_text(FIXTURE, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Werror", "-I" + str(VK), "-I" + str(args.headers), str(cpp), + "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + print(r.stdout.rstrip()) + if r.returncode: + raise SystemExit("FAIL: the GCN rule's checks failed") + + +if __name__ == "__main__": + main() diff --git a/tests/test_radv_mesh_optimizer.py b/tests/test_radv_mesh_optimizer.py new file mode 100644 index 0000000..c2b317b --- /dev/null +++ b/tests/test_radv_mesh_optimizer.py @@ -0,0 +1,150 @@ +"""Task and mesh pipelines on RADV built with the driver's optimizer (gpu/vk/vk_draw.cpp mesh_opt_switch, mesh_unoptimized, +mesh_opt_recipe, build_graphics, warmed_stamp_of; RX_VK_MESH_OPT; no game/GPU). + +On AMD's own drivers the optimizer hung the GPU minutes into Kokiri (RX 6600), so their task and mesh pipelines are built without +it. On RADV it is the other way round: built without it, the culling task programs timed out the ring at the save load. + +The three functions are cut out of vk_draw.cpp unchanged and compiled with a stand-in device, one run a case: unset, RADV builds +with the optimizer and every other AMD driver without; =0 without on every AMD driver, RADV too (the rule before); =1 and any +other value with it everywhere; NVIDIA and Intel never without. The stamp word: only on RADV with native mesh shaders and the +switch unset (not on the task + mesh fallback, not with =0 or =1, not on AMD's own drivers). The source: build_graphics asks +mesh_unoptimized for task and mesh stages; the cloth rule and the retry without the optimizer are as they were; warmed_stamp_of +adds kMeshOptRecipe only when asked, warmed_stamp and libraries_stamp ask mesh_opt_recipe, warm_skip_carry tries the 8 +combinations of the three words; the docs row is there. + + python -I -B tests/test_radv_mesh_optimizer.py [--cxx clang++] +""" +import argparse +import os +from pathlib import Path +import subprocess +import tempfile + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] + +RADV, AMD_PRO, AMD_OPEN, NV, INTEL = 3, 1, 2, 4, 6 # (VkDriverId values: MESA_RADV, AMD_PROPRIETARY, AMD_OPEN_SOURCE, NVIDIA, Intel Mesa) + +FIXTURE = r'''#include +#include +#include +#include +enum VkDriverId { VK_DRIVER_ID_AMD_PROPRIETARY = 1, VK_DRIVER_ID_AMD_OPEN_SOURCE = 2, VK_DRIVER_ID_MESA_RADV = 3, + VK_DRIVER_ID_NVIDIA_PROPRIETARY = 4, VK_DRIVER_ID_INTEL_OPEN_SOURCE_MESA = 6 }; +struct { struct { uint32_t vendorID; } props; VkDriverId driver_id; bool mesh_shader, task_mesh_fallback; } dev; +@CUT@ +int main(int argc, char** argv) { + dev.props.vendorID = (uint32_t)strtoul(argv[1], nullptr, 16); + dev.driver_id = (VkDriverId)atoi(argv[2]); + dev.mesh_shader = atoi(argv[3]) != 0; + dev.task_mesh_fallback = atoi(argv[4]) != 0; + std::printf("%d %d\n", (int)mesh_unoptimized(), (int)mesh_opt_recipe()); + return 0; +} +''' + +# (vendor, driver, native mesh, task + mesh fallback, RX_VK_MESH_OPT or None) -> (built without the optimizer, stamp word) +CASES = [ + (("1002", RADV, 1, 0, None), (0, 1)), + (("1002", RADV, 0, 0, None), (0, 0)), # (no mesh shaders: the fallback's compute, no word) + (("1002", RADV, 1, 1, None), (0, 0)), # (the task + mesh fallback: no word) + (("1002", AMD_PRO, 1, 0, None), (1, 0)), + (("1002", AMD_OPEN, 1, 0, None), (1, 0)), + (("1002", RADV, 1, 0, "0"), (1, 0)), + (("1002", AMD_PRO, 1, 0, "0"), (1, 0)), + (("1002", RADV, 1, 0, "1"), (0, 0)), + (("1002", AMD_PRO, 1, 0, "1"), (0, 0)), + (("1002", AMD_PRO, 1, 0, "yes"), (0, 0)), + (("10DE", NV, 1, 0, None), (0, 0)), + (("10DE", NV, 1, 0, "0"), (0, 0)), + (("8086", INTEL, 1, 0, None), (0, 0)), +] + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def fail(message): + raise SystemExit("FAIL: " + message) + + +def cut(draw): + return "\n".join(block(draw, m) for m in ("int mesh_opt_switch() {", "bool mesh_unoptimized() {", "bool mesh_opt_recipe() {")) + + +def check_source(draw, dev): + build = block(draw, "VkPipeline build_graphics(const Program& p, const NvnState& s, const Targets& at, VkPrimitiveTopology topo, " + "int part, bool* cold) {") + rule = build[build.index("if (mesh_unoptimized()) {"):] + rule = rule[:rule.index("#ifdef _WIN32")] + if "VK_SHADER_STAGE_TASK_BIT_EXT | VK_SHADER_STAGE_MESH_BIT_EXT" not in rule or \ + "ci.flags |= VK_PIPELINE_CREATE_DISABLE_OPTIMIZATION_BIT;" not in rule: + fail("build_graphics no longer builds task and mesh stages without the optimizer where mesh_unoptimized says") + if 'getenv("RX_VK_MESH_OPT") != nullptr' in draw: + fail("RX_VK_MESH_OPT is read as set or not again (=0 would turn the optimizer on)") + for needle in ("if (p.key == 0x5fa56485a6b07619ull) {", + "if (driver.driverID == VK_DRIVER_ID_AMD_PROPRIETARY) ci.flags |= VK_PIPELINE_CREATE_DISABLE_OPTIMIZATION_BIT;", + "if (dev.props.vendorID == 0x1002 && dev.task_mesh_fallback && !p.mesh && part < 0 && r == VK_ERROR_UNKNOWN &&"): + if needle not in build: + fail("build_graphics's cloth rule or retry changed: " + needle) + stamp = block(draw, "uint64_t warmed_stamp_of(bool libraries, uint32_t patches, bool rate, bool mesh_opt) {") + if "if (mesh_opt) add(&kMeshOptRecipe, sizeof kMeshOptRecipe);" not in stamp: + fail("warmed_stamp_of does not add kMeshOptRecipe only when asked") + if "warmed_stamp_of(libraries, module_patches(), false, mesh_opt_recipe());" not in block(draw, "uint64_t warmed_stamp(bool libraries) {"): + fail("warmed_stamp does not ask mesh_opt_recipe") + if "libraries && dev.foliage_rate, mesh_opt_recipe());" not in block(draw, "uint64_t libraries_stamp(bool libraries) {"): + fail("libraries_stamp does not ask mesh_opt_recipe") + carry = block(draw, "void warm_skip_carry(const std::string& skip, bool libraries, uint64_t now) {") + if "for (int k = 0; k < 8; k++) {" not in carry or "(k & 4) != 0);" not in carry: + fail("warm_skip_carry does not try the stamp with and without kMeshOptRecipe") + if "`RX_VK_MESH_OPT=1` or `=0`" not in dev: + fail("docs/development.md has no RX_VK_MESH_OPT row") + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--cxx", default="g++") + a = p.parse_args() + draw = (ROOT / "gpu/vk/vk_draw.cpp").read_text(encoding="utf-8") + dev = (ROOT / "docs/development.md").read_text(encoding="utf-8") + try: + check_source(draw, dev) + text = FIXTURE.replace("@CUT@", cut(draw)) + except ValueError as e: + fail("a text this test reads moved, or is not there (%s)" % e) + flags = subprocess.BELOW_NORMAL_PRIORITY_CLASS if os.name == "nt" else 0 + problems = [] + with tempfile.TemporaryDirectory(prefix="radv-mesh-opt-") as tmp: + cpp = Path(tmp) / "fixture.cpp" + exe = Path(tmp) / ("fixture" + (".exe" if os.name == "nt" else "")) + cpp.write_text(text, encoding="utf-8") + built = subprocess.run([a.cxx, "-std=c++20", "-O1", "-Wall", "-Werror", str(cpp), "-o", str(exe)], + capture_output=True, text=True, creationflags=flags) + if built.returncode: + print(built.stdout + built.stderr) + fail("the fixture did not build") + base = {k: v for k, v in os.environ.items() if k != "RX_VK_MESH_OPT"} + for (vendor, driver, mesh, fallback, switch), want in CASES: + env = dict(base, **({"RX_VK_MESH_OPT": switch} if switch is not None else {})) + done = subprocess.run([str(exe), vendor, str(driver), str(mesh), str(fallback)], env=env, capture_output=True, text=True, + timeout=60, creationflags=flags) + got = tuple(int(x) for x in done.stdout.split()) if not done.returncode else None + if got != want: + problems.append("%s driver %d mesh %d fallback %d RX_VK_MESH_OPT=%r: %s, wanted %s" + % (vendor, driver, mesh, fallback, switch, got, want)) + if problems: + fail("\n ".join([""] + problems)) + print("PASS: task and mesh pipelines with the optimizer on RADV, without it on AMD's own drivers; =0 the rule before, =1 with " + "it everywhere; the stamp word on RADV with native mesh only; cloth rule and retry unchanged (%d cases)" % len(CASES)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_radv_vrs_drop.py b/tests/test_radv_vrs_drop.py new file mode 100644 index 0000000..c07523e --- /dev/null +++ b/tests/test_radv_vrs_drop.py @@ -0,0 +1,140 @@ +"""Game Mode's forced-VRS variable dropped before Vulkan loads, so prepared pipelines still match (host/hos_main.c +driver_env_choose; HOS_HALF_RATE_SHADING; Linux; no game). + +Steam's Game Mode gives every game RADV_FORCE_VRS_CONFIG_FILE (Half Rate Shading). While the file exists RADV puts a forced shading +rate into every pipeline's key, whatever it holds, so pipelines prepared without it miss the driver's caches and compile again. + +driver_env_choose is cut out of hos_main.c unchanged and compiled with a stand-in environment (getenv, unsetenv) over real files +(stat), so it runs on any system. Checked: a file holding 1x1 and an empty file: the variable dropped and the line says so; a +missing file, an empty value and no variable: left as it is, no line; HOS_HALF_RATE_SHADING=1 keeps it (the line says so), =0 +drops it; RADV_FORCE_VRS untouched in every case. The source: Linux's main calls it after the env file is loaded and before the +display (and before the builder's --prepare-shaders run returns), the line joins the boot: lines, all of it Linux's alone, and the +switch has its docs row. + + python -I -B tests/test_radv_vrs_drop.py [--cc clang] +""" +import argparse +import os +from pathlib import Path +import subprocess +import tempfile + +RUN_ALL_ARGS = ["--cc", "gcc"] + +ROOT = Path(__file__).resolve().parents[1] + +FIXTURE = r'''#include +#include +#include +#include +/* (the environment: a few names, set from the arguments) */ +static const char *names[3] = {"RADV_FORCE_VRS_CONFIG_FILE", "HOS_HALF_RATE_SHADING", "RADV_FORCE_VRS"}; +static const char *values[3]; +static const char *fake_getenv(const char *n) { + for (int i = 0; i < 3; i++) if (!strcmp(n, names[i])) return values[i]; + return NULL; +} +static int fake_unsetenv(const char *n) { + for (int i = 0; i < 3; i++) if (!strcmp(n, names[i])) values[i] = NULL; + return 0; +} +#define getenv fake_getenv +#define unsetenv fake_unsetenv +@CUT@ +int main(int argc, char **argv) { + for (int i = 0; i < 3 && i + 1 < argc; i++) values[i] = strcmp(argv[i + 1], "-") ? argv[i + 1] : NULL; + driver_env_choose(); + printf("vrs_file=%s\nvrs=%s\nnote=%s\n", values[0] ? values[0] : "(unset)", values[2] ? values[2] : "(unset)", driver_env_note); + return 0; +} +''' + + +def fail(message): + raise SystemExit("FAIL: " + message) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def check_source(src, dev): + choose = src.index("static void driver_env_choose(void) {") + linux = src.rfind("#ifdef __linux__", 0, choose) + if linux < 0 or "#endif" in src[linux:choose]: + fail("driver_env_choose is not Linux's alone") + main = block(src, "int main(int argc, char **argv) {\n int hos_display_init(void);") + load, call = main.index("load_hookshot_env();"), main.index("driver_env_choose();") + if not load < call < main.index("hos_display_init();") or call > main.index("--prepare-shaders"): + fail("Linux's main does not call driver_env_choose between the env file and the display, before the builder's run") + if "#ifdef __linux__" not in main[load:call]: + fail("Linux's main calls driver_env_choose outside Linux") + game = src[src.index(" memory_notice();\n#endif\n"):] + if not game.startswith(" memory_notice();\n#endif\n#ifdef __linux__\n if (driver_env_note[0]) hos_log(\"%s\", driver_env_note);"): + fail("driver_env_choose's line does not join the boot: lines in the log") + if "`HOS_HALF_RATE_SHADING=1`" not in dev: + fail("docs/development.md has no HOS_HALF_RATE_SHADING row") + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--cc", default="gcc") + a = p.parse_args() + src = (ROOT / "host/hos_main.c").read_text(encoding="utf-8") + dev = (ROOT / "docs/development.md").read_text(encoding="utf-8") + try: + check_source(src, dev) + cut = "static char driver_env_note[640];\n" + block(src, "static void driver_env_choose(void) {") + except ValueError as e: + fail("a text this test reads moved (%s)" % e) + flags = subprocess.BELOW_NORMAL_PRIORITY_CLASS if os.name == "nt" else 0 + problems = [] + with tempfile.TemporaryDirectory(prefix="radv-vrs-") as tmp: + one = Path(tmp) / "vrs_1x1.txt" + one.write_text("1x1\n", encoding="ascii") + empty = Path(tmp) / "vrs_empty.txt" + empty.write_text("", encoding="ascii") + missing = str(Path(tmp) / "no_such_file.txt") + c = Path(tmp) / "fixture.c" + exe = Path(tmp) / ("fixture" + (".exe" if os.name == "nt" else "")) + c.write_text(FIXTURE.replace("@CUT@", cut), encoding="utf-8") + built = subprocess.run([a.cc, "-std=gnu17", "-O1", "-Wall", "-Wno-unused-function", str(c), "-o", str(exe)], + capture_output=True, text=True, creationflags=flags) + if built.returncode: + print(built.stdout + built.stderr) + fail("the fixture did not build") + # (RADV_FORCE_VRS_CONFIG_FILE, HOS_HALF_RATE_SHADING, RADV_FORCE_VRS) -> (dropped, the note's text or "" for none) + cases = [ + ((str(one), "-", "2x2"), (True, "dropped")), + ((str(empty), "-", "-"), (True, "dropped")), + ((missing, "-", "2x2"), (False, "")), + (("", "-", "-"), (False, "")), + (("-", "-", "2x2"), (False, "")), + ((str(one), "1", "2x2"), (False, "kept (HOS_HALF_RATE_SHADING=1")), + ((str(one), "0", "-"), (True, "dropped")), + ] + for args, (dropped, note) in cases: + done = subprocess.run([str(exe)] + list(args), capture_output=True, text=True, timeout=60, creationflags=flags) + out = dict(line.split("=", 1) for line in done.stdout.splitlines() if "=" in line) + want_file = "(unset)" if dropped or args[0] == "-" else args[0] + want_vrs = "(unset)" if args[2] == "-" else args[2] + ok = (not done.returncode and out.get("vrs_file") == want_file and out.get("vrs") == want_vrs and + (note in out.get("note", "") if note else out.get("note") == "")) + if ok and dropped and "HOS_HALF_RATE_SHADING=1 keeps it" not in out.get("note", ""): + ok = False + if not ok: + problems.append("%s: %s" % (args, (done.stdout + done.stderr).strip())) + if problems: + fail("\n ".join([""] + problems)) + print("PASS: RADV_FORCE_VRS_CONFIG_FILE dropped when it names a file, kept with HOS_HALF_RATE_SHADING=1, left alone otherwise; " + "RADV_FORCE_VRS untouched; called before Vulkan on Linux (%d cases)" % len(cases)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_range_walk.py b/tests/test_range_walk.py new file mode 100644 index 0000000..15cbe38 --- /dev/null +++ b/tests/test_range_walk.py @@ -0,0 +1,202 @@ +"""Is a page in a sorted list of ranges, asked for a whole loop's pages by a cursor that only moves forward (gpu/vk/ +range_walk.h; vk_memory.cpp's capture and collection loops; no game, no GPU). + +(Triforce, 11-22, the performance audit's MEM-42, after Source A's.) memory_collect_written, capture_make and take_capture +asked of every page whether it lies in a range list (the captures' ranges, the texture storage's) with a binary search +a page. A loop's pages come ascending, or nearly, so each loop has a RangeWalk now. + +This compiles the header and asks the walk and std::upper_bound, the search as the loops made it, the same addresses: +over ranges with gaps, ranges that touch, a list that starts at 0, an empty list, one range, and random lists (also +with overlapping ranges, which the search allowed), asked ascending by page, ascending with repeats, at random, +descending, and ascending with steps back; and the walk made to search every time (the switch, the lab knob). Every +answer must be the search's. It also counts the walk's steps over an ascending loop: no more than the ranges and the +addresses together (one pass). + +And reads the source: none of the loops searches any more (no upper_bound over texr or the captures' ranges, in_tex_merged +only where one page is asked), each walk over tex_merged is made after shadow_mu is taken in its block, the switch is +read once and the lab knob at each walk, and the switches table has the row. + + python -I -B tests/test_range_walk.py [--cxx clang++] +""" +import argparse +import re +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + check(marker in text, "vk_memory.cpp has no " + marker) + start = text.index(marker) + end = text.index("{", start + len(marker) - 1) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +PROGRAM = r""" +#include +#include "range_walk.h" +using range_walk::Ranges; +using range_walk::RangeWalk; +static int fails = 0; +static long asked = 0; +static uint64_t rnd_state = 0x853C49E6748FEA9Bull; +static uint64_t rnd() { + rnd_state ^= rnd_state << 13, rnd_state ^= rnd_state >> 7, rnd_state ^= rnd_state << 17; + return rnd_state; +} +// (the search as the loops made it) +static bool searched(const Ranges& r, uint64_t a) { + auto it = std::upper_bound(r.begin(), r.end(), std::make_pair(a, ~uint64_t(0))); + return it != r.begin() && a < std::prev(it)->second; +} +static void ask(const Ranges& r, const std::vector& order, const char* what) { + for (int always = 0; always < 2; always++) { + RangeWalk w{r, always != 0}; + for (uint64_t a : order) { + asked++; + if (w.in(a) != searched(r, a)) { + if (fails++ < 20) std::printf("FAIL %s%s: %#llx is %s by the search\n", what, always ? " (searching)" : "", (unsigned long long)a, + searched(r, a) ? "in" : "out"); + } + } + } +} +static void every_order(const Ranges& r, uint64_t span, const char* what) { + std::vector asc, order; + for (uint64_t a = 0; a < span; a += 4096) asc.push_back(a); + ask(r, asc, what); + order.clear(); // (ascending with repeats: a page asked twice, as the shadow's loop does where it stops) + for (uint64_t a : asc) { + order.push_back(a); + if (rnd() % 3 == 0) order.push_back(a); + } + ask(r, order, what); + order = asc; // (at random) + for (size_t i = order.size(); i > 1; i--) std::swap(order[i - 1], order[rnd() % i]); + ask(r, order, what); + order.assign(asc.rbegin(), asc.rend()); // (descending) + ask(r, order, what); + order.clear(); // (ascending with steps back: an unsorted capture) + for (size_t i = 0; i < asc.size(); i++) { + order.push_back(asc[i]); + if (rnd() % 5 == 0) order.push_back(asc[rnd() % (i + 1)]); + } + ask(r, order, what); + order.clear(); // (addresses inside pages, and the ranges' own edges) + for (const auto& e : r) + for (uint64_t a : {e.first - 1, e.first, e.first + 1, e.second - 1, e.second, e.second + 1}) order.push_back(a); + ask(r, order, what); + std::sort(order.begin(), order.end()); + ask(r, order, what); +} + +int main() { + const uint64_t P = 4096; + every_order({{4 * P, 8 * P}, {16 * P, 17 * P}, {40 * P, 64 * P}, {100 * P, 101 * P}}, 128 * P, "ranges with gaps"); + every_order({{4 * P, 8 * P}, {8 * P, 12 * P}, {12 * P, 13 * P}, {13 * P, 40 * P}}, 64 * P, "ranges that touch"); + every_order({{0, 3 * P}, {5 * P, 6 * P}}, 16 * P, "a list from 0"); + every_order({{1, 3 * P}}, 16 * P, "one range"); + every_order({}, 16 * P, "no range"); + for (int t = 0; t < 300; t++) { + Ranges r; + const int n = 1 + (int)(rnd() % (t % 3 == 0 ? 256 : 16)); + uint64_t at = (rnd() % 8) * P; + for (int i = 0; i < n; i++) { + const uint64_t len = (1 + rnd() % 12) * P; + r.push_back({at, at + len}); + at += len + (rnd() % 4 == 0 ? 0 : (rnd() % 9) * P); + } + every_order(r, at + 8 * P, "random ranges"); + if (t % 10 == 0) { // (sorted by first but overlapping: the search's answers all the same) + for (auto& e : r) e.second += (rnd() % 20) * P; + every_order(r, at + 8 * P, "overlapping ranges"); + } + } + { // one pass: an ascending loop moves the cursor at most once a range + Ranges r; + for (uint64_t i = 0; i < 256; i++) r.push_back({i * 16 * P, i * 16 * P + 4 * P}); + RangeWalk w{r}; + size_t moved = 0, in = 0; + for (uint64_t a = 0; a < 256 * 16 * P; a += P) { + const size_t before = w.next; + in += w.in(a); + if (w.next < before) fails++, std::printf("FAIL the cursor went back in an ascending loop\n"); + moved += w.next - before; + } + if (moved != r.size() || in != 256 * 4) fails++, std::printf("FAIL an ascending loop: the cursor moved %zu times over %zu ranges, %zu pages in\n", moved, r.size(), in); + } + if (fails) std::printf("range walk: FAILED (%d of %ld answers)\n", fails, asked); + else std::printf("range walk: %ld answers, every one the search's\n", asked); + return fails != 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + check((VK / "range_walk.h").exists(), "gpu/vk/range_walk.h is missing") + mem = (VK / "vk_memory.cpp").read_text(encoding="utf-8") + check('#include "range_walk.h"' in mem, "vk_memory.cpp does not include range_walk.h") + + # the loops + collect = block(mem, "void memory_collect_written() {") + check("RangeWalk in_capture{owned, range_walk_search()};" in collect and "in_capture.in(guest)" in collect and "upper_bound" not in collect, + "memory_collect_written still searches the captures' ranges for every page") + make = block(mem, "std::unique_ptr capture_make(bool behind) {") + check(make.count("RangeWalk in_texr{texr, range_walk_search()};") == 2 and make.count("in_texr.in(g)") == 2 and + "std::upper_bound(texr" not in make, "capture_make's staging loop or its behind loop still searches the texture ranges") + for m in re.finditer(r"RangeWalk in_texr\{", make): # (each made right before its loop) + check(make[m.end():].lstrip().split("\n")[1].lstrip().startswith("for ("), "capture_make: a walk is not made right at its loop") + take = block(mem, "bool take_capture(std::unique_ptr c, uint64_t value, uint64_t completed) {") + walks = [m.start() for m in re.finditer(r"RangeWalk in_tex\{tex_merged, range_walk_search\(\)\};", take)] + check(len(walks) == 5 and take.count("in_tex.in(") == 6, + "take_capture: its loops over the texture storage's ranges are not the five walks (%d) and six asks (%d)" % (len(walks), take.count("in_tex.in("))) + check(take.count("in_tex_merged(") == 1 and "const auto to_need = [&](uint32_t i) {\n if (!in_tex_merged(" in take, + "take_capture: in_tex_merged is asked elsewhere than for the one page of to_need") + # (a walk over tex_merged is made inside a hold of shadow_mu: the nearest lock or unlock before it is shadow_mu's guard) + for at in walks: + before = take[:at] + lock = before.rfind("std::lock_guard sk(shadow_mu);") + check(lock >= 0, "take_capture: a walk over tex_merged with shadow_mu not taken") + depth = 0 # (the guard's block is still open where the walk is made) + for ch in before[lock:]: + depth += (ch == "{") - (ch == "}") + check(depth >= 0, "take_capture: a walk over tex_merged made after its shadow_mu block ended") + log_walk = "RangeWalk in_tex{tex_merged, range_walk_search()}; // ((Triforce, 11-22, MEM-42) under shadow_mu, for this loop)\n for (size_t j = 0; j < m; j++) is_tex[j] = !tex_merged.empty() && in_tex.in(" + check("std::lock_guard sk(shadow_mu);\n " + log_walk in mem, "the late wait's log does not walk the texture ranges under shadow_mu") + check(mem.count("in_tex_merged(") == 2, "in_tex_merged is asked in some other loop") + + # the switch and the knob + search = block(mem, "bool range_walk_search() {") + check('getenv("RX_VK_RANGE_WALK")' in search and "static const bool off" in search and "e && *e && atoi(e) == 0" in search and + 'return off || lab_ab::on("norangewalk");' in search, "RX_VK_RANGE_WALK is not read once, on unless 0, with the lab knob norangewalk") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`RX_VK_RANGE_WALK=0`" in dev and "`norangewalk`" in dev, "docs/development.md: RX_VK_RANGE_WALK=0 is not in the switches table") + + with tempfile.TemporaryDirectory(prefix="range-walk-") as tmp: + src, exe = Path(tmp) / "walk.cpp", Path(tmp) / "walk.exe" + src.write_text(PROGRAM, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O2", "-Wall", "-Wextra", "-Werror", "-I" + str(VK), str(src), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True, timeout=300) + print(r.stdout.rstrip()) + check(r.returncode == 0, "the walk's answers are not the search's") + print("PASS: the capture and collection loops ask a forward cursor, whose answers are the binary search's in any order") + + +if __name__ == "__main__": + main() diff --git a/tests/test_readback_late.py b/tests/test_readback_late.py new file mode 100644 index 0000000..ec0136a --- /dev/null +++ b/tests/test_readback_late.py @@ -0,0 +1,179 @@ +"""Late readbacks: ranges the CPU reads back without their write-back's wait (gpu/vk/readback_late.h; no game, no GPU). + +(Triforce, 11-22, the performance audit's SYNC-19, after Source A's.) The replay's invalidate waits for the write-back of +every range the game is about to read back. Source A names ranges read without that wait, from what guest memory holds: a size +in hex (its default 3980, the culling camera block) or `newest` (the writer is the newest submission handed out). +RX_VK_READBACK_LATE=[,]; unset there is NO rule here and every readback waits as before, until the lab has run it. + +Compiles a driver against the header and checks: + the text unset, empty and 0 give no rule; 3980 and 0x3980 the camera block's size; newest; several; a word, a size + with a tail and a ninth size are left out and counted; a size named twice is one rule; + the decision with no rule, 200,000 random readbacks are never late (the default decides as before); with a rule a range + is late only while its write-back is ahead of the GPU, by its size, or by `newest` only when its writer is + at or past the newest value handed out; + the range log counts by (address, size), sums a range's waits, keeps the rest together past its cap and lists the + longest waits first. +And reads vk_replay.cpp's invalidate: the rule is asked with the write-back's value the readback came with, the GPU's +progress and the newest fence; a late range skips the catch-up and the wait and nothing else (its note for the CPU's +write-back stays, before the decision); nothing is noted for the range log without its switch; the lab knob gives the +default's rules only while the variable is unset; the switches table has the row. + + python -I -B tests/test_readback_late.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +DRIVER = r''' +#include "readback_late.h" +#include + +namespace rl = mkwvk::readback_late; +static int checks, fails; +#define CHECK(what, cond) do { checks++; if (!(cond)) fails++, printf("FAIL %s (line %d)\n", what, __LINE__); } while (0) + +int main() { + // the text + CHECK("unset: no rule", !rl::parse(nullptr).any() && !rl::parse(nullptr).bad); + CHECK("empty: no rule", !rl::parse("").any()); + CHECK("0: none", !rl::parse("0").any() && !rl::parse("0").bad); + { const rl::Rules r = rl::parse("3980"); CHECK("3980", r.n == 1 && r.sizes[0] == 0x3980 && !r.newest && !r.bad); } + { const rl::Rules r = rl::parse("0x3980"); CHECK("0x3980", r.n == 1 && r.sizes[0] == 0x3980 && !r.bad); } + { const rl::Rules r = rl::parse("newest"); CHECK("newest", r.n == 0 && r.newest && r.any() && !r.bad); } + { const rl::Rules r = rl::parse("3980,newest,1040"); CHECK("several", r.n == 2 && r.sizes[0] == 0x3980 && r.sizes[1] == 0x1040 && r.newest && !r.bad); } + { const rl::Rules r = rl::parse("junk"); CHECK("a word: left out", !r.any() && r.bad == 1); } + { const rl::Rules r = rl::parse("3980z,newest2,,40"); CHECK("tails", r.n == 1 && r.sizes[0] == 0x40 && !r.newest && r.bad == 2); } + { const rl::Rules r = rl::parse("3980,3980,0x3980"); CHECK("named twice: one rule", r.n == 1 && !r.bad); } + { const rl::Rules r = rl::parse("1,2,3,4,5,6,7,8,9,a"); CHECK("past the eighth size", r.n == rl::kMaxSizes && r.bad == 2 && !r.has(9) && r.has(8)); } + { const rl::Rules r = rl::parse("123456789012345678901234567890"); CHECK("too long a part", !r.any() && r.bad == 1); } + { const rl::Rules d = rl::default_rules(); CHECK("the default's rules", d.n == 1 && d.sizes[0] == rl::kCameraBlock && rl::kCameraBlock == 0x3980 && !d.newest); } + + // the decision + uint64_t rng = 0x2545F4914F6CDD1Dull; + const auto rnd = [&] { rng ^= rng << 13, rng ^= rng >> 7, rng ^= rng << 17; return rng; }; + const rl::Rules none = rl::parse(nullptr), zero = rl::parse("0"), cam = rl::parse("3980"), newest = rl::parse("newest"), + both = rl::parse("newest,3980"); + for (int i = 0; i < 200000; i++) { + const uint64_t size = i % 5 == 0 ? 0x3980 : rnd() % 0x8000, done = rnd() % 1000, writer = rnd() % 1004, handed = done + rnd() % 6; + const bool pending = writer > done, is_newest = writer >= handed; + CHECK("no rule: never late", !rl::late(none, size, writer, done, handed) && !rl::late(zero, size, writer, done, handed)); + CHECK("by size", rl::late(cam, size, writer, done, handed) == (pending && size == 0x3980)); + CHECK("newest", rl::late(newest, size, writer, done, handed) == (pending && is_newest)); + CHECK("both", rl::late(both, size, writer, done, handed) == (pending && (is_newest || size == 0x3980))); + if (fails) break; + } + CHECK("a write-back already done is never late", !rl::late(both, 0x3980, 7, 7, 7) && !rl::late(both, 0x3980, 6, 7, 7)); + CHECK("wanted: the size alone", rl::wanted(cam, 0x3980, false) && !rl::wanted(cam, 0x3981, true)); + CHECK("wanted: newest alone", rl::wanted(newest, 1, true) && !rl::wanted(newest, 0x3980, false)); + + // the range log + rl::RangeLog log; + log.cap = 3; + log.note(0x1000, 0x3980, true, 2000000, true, false); + log.note(0x1000, 0x3980, false, 9000000, false, true); // (not waited: its time is not a wait's) + log.note(0x1000, 0x40, true, 500000, false, false); // (the same address, another size: another range) + log.note(0x9000, 0x1040, true, 7000000, true, false); + log.note(0xA000, 0x10, true, 1, false, false); // (past the cap: together) + log.note(0xB000, 0x10, false, 1, true, true); + log.note(0x9000, 0x1040, true, 1000000, false, false); // (a range already in the table still counts under itself) + CHECK("ranges", log.rows.size() == 3); + { const auto& r = log.rows[{0x1000, 0x3980}]; CHECK("a range's counts", r.readbacks == 2 && r.waits == 1 && r.wait_ns == 2000000 && r.newest == 1 && r.late == 1); } + CHECK("past the cap", log.other.readbacks == 2 && log.other.waits == 1 && log.other.wait_ns == 1 && log.other.newest == 1 && log.other.late == 1); + { const auto top = log.top(2); + CHECK("the longest waits first", top.size() == 2 && top[0].gpu == 0x9000 && top[0].size == 0x1040 && top[0].row.wait_ns == 8000000 && + top[1].gpu == 0x1000 && top[1].size == 0x3980); } + CHECK("top of more than there are", log.top(50).size() == 3); + + if (fails) printf("late readbacks: FAILED (%d of %d checks)\n", fails, checks); + else printf("late readbacks: all %d checks passed\n", checks); + return fails != 0; +} +''' + + +def check(ok, what): + if not ok: + raise SystemExit("FAIL: " + what) + + +def function(text, head): + at = text.index(head) + return text[at:text.index("\n}\n", at)] + + +def check_source(): + replay = (ROOT / "gpu/vk/vk_replay.cpp").read_text(encoding="utf-8") + check('#include "readback_late.h"' in replay, "vk_replay.cpp does not include readback_late.h") + rules = function(replay, "const readback_late::Rules& rb_late_rules() {") + check('const char* e = getenv("RX_VK_READBACK_LATE");' in rules and "readback_late::parse(e);" in rules and "static const" in rules, + "RX_VK_READBACK_LATE is not parsed once by the header") + late = function(replay, "bool rb_late(uint64_t size, uint64_t writer, uint64_t done, uint64_t handed_out) {") + check("if (r.any()) return readback_late::late(r, size, writer, done, handed_out);" in late, + "the variable's rules do not decide through the header") + # (the knob gives the default's rules only while the variable is unset; with neither, never late) + check('if (!unset || !lab_ab::on("rblate")) return false;' in late and + late.index('if (!unset || !lab_ab::on("rblate")) return false;') < late.index("readback_late::default_rules();"), + "the lab knob's rules are not kept to an unset variable, or something is late with no rule and no knob") + check(late.count("return ") == 3, "rb_late has another way to say late") + + inv = function(replay, "void invalidate(uint64_t gpu, uint64_t size) {") + note = "memory_note_readback(host->gpu_guest(gpu), size);" + ask = "const bool is_late = rb_late(size, v0, c0, f0);" + check("if (!off && (v0 = memory_writeback_value(host->gpu_guest(gpu), size, &ever), ever)) {" in inv, + "the write-back's value the readback came with is not kept for the rule") + check("const uint64_t f0 = fence(), c0 = completed();" in inv, "the newest fence and the GPU's progress are not read") + check(inv.count(ask) == 1 and inv.index(note) < inv.index("const uint64_t f0 = fence(), c0 = completed();") < inv.index(ask), + "the rule is not asked once, after the note for the CPU's write-back") + # (a late range skips the catch-up: the first branch of the chain the two catch-ups are in) + chain = inv[inv.index(ask):inv.index("const auto w1 = std::chrono::steady_clock::now();")] + check("if (is_late) lates.fetch_add(1, std::memory_order_relaxed);\n" + " else if (behind0 && !no_catchup && !unforced) wait_queue_empty(), w = true;" in chain and + " else if (behind0 && !no_catchup && !async_on() && rb_fix(2)) sync_settle_at(settle_gate::kReadback), w = true;" in chain, + "a late range does not skip the catch-up, or the catch-ups are no longer the rest of its chain") + check(chain.count("wait_queue_empty()") == 1 and chain.count("sync_settle_at(") == 1 and "sync_settle()" not in chain.replace("sync_settle() as it was", ""), "another catch-up crept in beside the chain") + # (and the wait: every wait for the write-back's value is inside the block a late range does not enter) + check(inv.count("if (!is_late && v > completed()) {") == 1 and "if (v > completed()) {" not in inv, "a late range does not skip the wait") + block = inv[inv.index("if (!is_late && v > completed()) {"):inv.index("if (w) {")] + rest = inv.replace(block, "") + for call in ("wait_recorded(", "wait_value(", "readback_fallback_wait("): + check(call in block and call not in rest[rest.index(ask):rest.index("// (textures the GPU wrote there")], + "invalidate waits for the write-back outside the block a late range skips (%s)" % call) + # (nothing else changes for a late range: is_late is read in these places only) + check(inv.count("is_late") == 4 and "rb_ranges.note(gpu, size, w, d, v0 >= f0, is_late);" in inv, + "is_late decides something besides the catch-up, the wait and the range log's count") + # (the range log: nothing noted or logged without its switch) + check("if (rb_ranges_on()) {" in inv and inv.index("if (rb_ranges_on()) {") < inv.index("rb_ranges.note(") and + replay.count("rb_ranges.note(") == 1, "the range log notes a readback without its switch") + check("if (rb_ranges_on()) rb_ranges_line();" in inv and replay.count("rb_ranges_line();") == 1, "the range log's lines are logged without its switch") + on = function(replay, "bool rb_ranges_on() {") + check('getenv("RX_VK_LOG_READBACK_RANGES") && atoi(getenv("RX_VK_LOG_READBACK_RANGES")) != 0' in on, "RX_VK_LOG_READBACK_RANGES is not read") + + dev = (ROOT / "docs/development.md").read_text(encoding="utf-8") + row = [line for line in dev.splitlines() if line.startswith("| `RX_VK_READBACK_LATE=[,]`, `RX_VK_LOG_READBACK_RANGES=1` |")] + check(len(row) == 1 and "**Unset: no rule**" in row[0] and "`rblate`" in row[0] and "`newest`" in row[0], + "docs/development.md: the switches table does not have RX_VK_READBACK_LATE's row with its default") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + check_source() + with tempfile.TemporaryDirectory(prefix="readback-late-") as tmp: + src = Path(tmp) / "driver.cpp" + src.write_text(DRIVER, encoding="utf-8") + exe = Path(tmp) / "driver.exe" + subprocess.run([args.cxx, "-std=c++20", "-Wall", "-Wextra", "-Werror", "-I", str(ROOT / "gpu/vk"), str(src), "-o", str(exe)], + check=True) + out = subprocess.run([str(exe)], capture_output=True, text=True) + print(out.stdout, end="") + check(out.returncode == 0, "the header's checks failed") + print("PASS: late readbacks: no rule unless asked, the catch-up and the wait skipped for a named range only") + + +if __name__ == "__main__": + main() diff --git a/tests/test_recorder_auto.py b/tests/test_recorder_auto.py new file mode 100644 index 0000000..a5cc72e --- /dev/null +++ b/tests/test_recorder_auto.py @@ -0,0 +1,120 @@ +"""The recorder thread's default by the GPU and the cores (gpu/vk/recorder_auto.h, vk_main.cpp pick_device; no game/GPU). + +The rule is compiled from the header and run on made-up PCs: NVIDIA with 24 logical processors on and with 8 off (the rule since +11-16, unchanged); AMD with 6 physical cores (12 logical) on, 8 on, 4 cores and 8 logical (a Steam Deck) off, 5 off, the cores not +known off, AMD's older Windows driver (amd_older_compiler) off with 16 cores; Intel and an unknown vendor off with 32 cores. The +count: 4 of 8 two-thread cores in the affinity counts 4, one logical processor of a core counts the core, none counts 0. The +source: vk_main.cpp fills VendorFacts with the driver's properties, counts the process's physical cores and calls the rule before +create_device; RX_VK_RECORDER still decides over it (mode_from_env); the lab knob norec leaves the hold with no producer. + + python -I -B tests/test_recorder_auto.py [--cxx clang++] +""" +import argparse +import os +from pathlib import Path +import subprocess +import tempfile + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] + +FIXTURE = r'''#include +#include "recorder_auto.h" +using namespace mkwvk::recorder_default; +static int bad = 0; +static void expect(bool got, bool want, const char* what) { + if (got != want) { std::printf("FAIL %s: %d, wanted %d\n", what, (int)got, (int)want); bad++; } +} +int main() { + expect(rule(0x10DE, 24, 12, false), true, "NVIDIA 24 logical"); + expect(rule(0x10DE, 12, 6, false), true, "NVIDIA 12 logical"); + expect(rule(0x10DE, 8, 8, false), false, "NVIDIA 8 logical"); + expect(rule(0x10DE, 11, 16, false), false, "NVIDIA 11 logical, 16 physical"); + expect(rule(0x10DE, 32, 0, false), true, "NVIDIA, physical unknown"); + expect(rule(0x1002, 12, 6, false), true, "AMD 6c/12t"); + expect(rule(0x1002, 16, 8, false), true, "AMD 8c/16t"); + expect(rule(0x1002, 6, 6, false), true, "AMD 6c/6t"); + expect(rule(0x1002, 8, 4, false), false, "AMD 4c/8t (Steam Deck)"); + expect(rule(0x1002, 10, 5, false), false, "AMD 5c/10t"); + expect(rule(0x1002, 24, 0, false), false, "AMD, physical unknown"); + expect(rule(0x1002, 32, 16, true), false, "AMD older Windows driver"); + expect(rule(0x8086, 32, 16, false), false, "Intel"); + expect(rule(0x13B5, 32, 16, false), false, "another vendor"); + expect(why(0x1002, 8, 4, false) == kAmdFew, true, "why: AMD few"); + expect(why(0x1002, 24, 0, false) == kAmdUnknown, true, "why: AMD unknown"); + expect(why(0x1002, 32, 16, true) == kAmdOlder, true, "why: AMD older"); + for (int w = kNvidia; w <= kOther; w++) expect(text((Why)w) && *text((Why)w), true, "a text for each why"); + const uint64_t smt[8] = {0x3, 0xC, 0x30, 0xC0, 0x300, 0xC00, 0x3000, 0xC000}; + expect(physical_cores(smt, 8, 0xFFFF) == 8, true, "8 of 8 cores"); + expect(physical_cores(smt, 8, 0x00FF) == 4, true, "4 of 8 cores in the affinity"); + expect(physical_cores(smt, 8, 0x5555) == 8, true, "one logical processor a core"); + expect(physical_cores(smt, 8, 0x30000) == 0, true, "none in the affinity"); + expect(physical_cores(smt, 0, ~0ull) == 0, true, "no cores read"); + if (bad) return 1; + std::printf("OK\n"); + return 0; +} +''' + + +def fail(message): + raise SystemExit("FAIL: " + message) + + +def check_source(): + main = (ROOT / "gpu/vk/vk_main.cpp").read_text(encoding="utf-8") + rec = (ROOT / "gpu/vk/vk_recorder.h").read_text(encoding="utf-8") + replay = (ROOT / "gpu/vk/vk_replay.cpp").read_text(encoding="utf-8") + dev = (ROOT / "docs/development.md").read_text(encoding="utf-8") + pick = main[main.index("bool pick_device() {"):main.index("bool create_device() {")] + for needle in ("facts.driver_id = drv.driverID;", "facts.driver_info = drv.driverInfo;", "process_physical_cores()", + "recorder_default::why(facts.vendor_id, logical, physical, amd_older_compiler(facts))", + "recorder_auto(recorder_default::on(why));"): + if needle not in pick: + fail("pick_device no longer has: " + needle) + if "recorder_auto(dev.props.vendorID == 0x10DE" in main: + fail("vk_main.cpp still has the NVIDIA-only rule of its own") + count = main[main.index("static unsigned process_physical_cores() {"):main.index("bool pick_device() {")] + for needle in ("GetLogicalProcessorInformation(", "GetProcessAffinityMask(", "RelationProcessorCore", + "recorder_default::physical_cores("): + if needle not in count: + fail("process_physical_cores no longer has: " + needle) + env = rec[rec.index("inline int mode_from_env() {"):rec.index("inline int mode_init() {")] + if "if (!e) return g_auto_thread ? THREAD : OFF;" not in env: + fail("mode_from_env no longer takes the default only with RX_VK_RECORDER unset") + if "if (mode() == THREAD && !(g_direct_hook && g_direct_hook())) t_prod = &g_rec;" not in rec: + fail("DrainingMutex::enter no longer leaves the producer out for the lab knob") + if 'g_direct_hook = [] { return lab_ab::on("norec"); }' not in replay: + fail("vk_replay.cpp no longer sets the norec hook") + if "on an AMD GPU with 6 physical cores or more" not in dev: + fail("docs/development.md's RX_VK_RECORDER row does not say the AMD default") + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--cxx", default="g++") + a = p.parse_args() + try: + check_source() + except ValueError as e: + fail("a text this test reads moved (%s)" % e) + flags = subprocess.BELOW_NORMAL_PRIORITY_CLASS if os.name == "nt" else 0 + with tempfile.TemporaryDirectory(prefix="recorder-auto-") as tmp: + cpp = Path(tmp) / "fixture.cpp" + exe = Path(tmp) / ("fixture" + (".exe" if os.name == "nt" else "")) + cpp.write_text(FIXTURE, encoding="utf-8") + built = subprocess.run([a.cxx, "-std=c++20", "-O1", "-Wall", "-Werror", "-I" + str(ROOT / "gpu/vk"), str(cpp), "-o", str(exe)], + capture_output=True, text=True, creationflags=flags) + if built.returncode: + print(built.stdout + built.stderr) + fail("the fixture did not build") + done = subprocess.run([str(exe)], capture_output=True, text=True, timeout=60, creationflags=flags) + if done.returncode or done.stdout.strip() != "OK": + fail((done.stdout + done.stderr).strip()) + print("PASS: the recorder's default: NVIDIA with 12 logical processors or more as before, AMD with 6 physical cores or more " + "but its older Windows driver, nothing else; the count of cores in the affinity; vk_main.cpp calls the rule") + + +if __name__ == "__main__": + main() diff --git a/tests/test_recorder_sync.py b/tests/test_recorder_sync.py index f3dba19..53217f4 100644 --- a/tests/test_recorder_sync.py +++ b/tests/test_recorder_sync.py @@ -94,6 +94,10 @@ def recorder_calls(text): B_DIRECT_STMT = "::mkwvk::rec::boundary(::mkwvk::rec::B_DIRECT);" # ((Triforce, P4) the boundaries whose arguments go through a counting macro of vk_work.h first: cmd_work.h work_begun, work_submitted) COUNTED_BOUNDARIES = {"vkBeginCommandBuffer": "RX_WORK_BEGUN", "vkQueueSubmit2": "RX_WORK_SUBMIT"} +# ((Triforce, 11-22, the performance audit's SYNC-27) the boundaries that block on the GPU: after the drain their macro hands the +# driver's call and its arguments to a wrapper that times it (vk_work.h gpu_waits_vk, gpu_wait_kinds.h; tests/test_gpu_waits.py) +TIMED_BOUNDARIES = {"vkQueueWaitIdle": "queue_idle", "vkDeviceWaitIdle": "device_idle", "vkWaitForFences": "fences", + "vkWaitSemaphores": "semaphores", "vkGetQueryPoolResults": "query_results"} C_COMMENT_OR_STRING = re.compile(r'//[^\n]*|/\*.*?\*/|"(?:\\.|[^"\\\n])*"|\'(?:\\.|[^\'\\\n])*\'', re.S) @@ -167,7 +171,14 @@ def check_sources(): # ((Triforce, P4) a begin and a submit pass their arguments through RX_WORK_BEGUN / RX_WORK_SUBMIT, which count them: cmd_work.h) bmacros = dict(re.findall(r"#define (\w+)\(\.\.\.\) \(::mkwvk::rec::boundary\(::mkwvk::rec::B_\w+\), (::vkx::|::)\w+\(" r"(?:RX_WORK_\w+\()?__VA_ARGS__\)?\)\)", work)) + for f, wrapper in TIMED_BOUNDARIES.items(): # ((Triforce, 11-22, SYNC-27) drained first, then the driver's own call, timed) + if "#define %s(...) (::mkwvk::rec::boundary(::mkwvk::rec::B_%s), ::mkwvk::gpu_waits_vk::%s(::vkx::%s, __VA_ARGS__))" % (f, f, wrapper, f) in work: + bmacros[f] = "::vkx::" + else: + problems.append("vk_work.h: %s's macro does not drain and then call the driver through its timed wrapper" % f) for f, q in bmacros.items(): + if f in TIMED_BOUNDARIES: + continue args = "%s(__VA_ARGS__)" % COUNTED_BOUNDARIES[f] if f in COUNTED_BOUNDARIES else "__VA_ARGS__" if "#define %s(...) (::mkwvk::rec::boundary(::mkwvk::rec::B_%s), %s%s(%s))" % (f, f, q, f, args) not in work: problems.append("vk_work.h: %s's macro calls another function" % f) @@ -274,6 +285,10 @@ def build(cxx, headers, out, recorder_text, name, extra=(), work_text=None): (d / "vk_loader.h").write_text(test_loader(), encoding="utf-8") (d / "vk_work.h").write_text(work_text or (VK / "vk_work.h").read_text(encoding="utf-8"), encoding="utf-8") (d / "cmd_work.h").write_text((VK / "cmd_work.h").read_text(encoding="utf-8"), encoding="utf-8") # ((Triforce, P4) vk_work.h's) + for h in ("gpu_wait_kinds.h", "lab_ab.h"): # ((Triforce, 11-22, SYNC-27) vk_work.h's wait wrappers, and the knob they read) + (d / h).write_text((VK / h).read_text(encoding="utf-8"), encoding="utf-8") + (d / "lock_census.h").write_text((VK / "lock_census.h").read_text(encoding="utf-8"), encoding="utf-8") # ((Triforce, THREAD-68) vk_recorder.h's) + (d / "new_wait.h").write_text((VK / "new_wait.h").read_text(encoding="utf-8"), encoding="utf-8") # ((Triforce, OS-04) vk_recorder.h's) (d / "vk_recorder.h").write_text(recorder_text, encoding="utf-8") exe = d / "recorder_sync_test.exe" cmd = [cxx, "-std=c++20", "-O2", "-Wall", "-Wextra", "-Werror", "-Wno-unused-function", "-Wno-missing-field-initializers", "-DVK_NO_PROTOTYPES", @@ -363,6 +378,10 @@ def exit_test(cxx, headers, out, recorder_text, name): (d / "vk_loader.h").write_text(test_loader(), encoding="utf-8") (d / "vk_work.h").write_text((VK / "vk_work.h").read_text(encoding="utf-8"), encoding="utf-8") (d / "cmd_work.h").write_text((VK / "cmd_work.h").read_text(encoding="utf-8"), encoding="utf-8") # ((Triforce, P4) vk_work.h's) + for h in ("gpu_wait_kinds.h", "lab_ab.h"): # ((Triforce, 11-22, SYNC-27) vk_work.h's wait wrappers, and the knob they read) + (d / h).write_text((VK / h).read_text(encoding="utf-8"), encoding="utf-8") + (d / "lock_census.h").write_text((VK / "lock_census.h").read_text(encoding="utf-8"), encoding="utf-8") # ((Triforce, THREAD-68) vk_recorder.h's) + (d / "new_wait.h").write_text((VK / "new_wait.h").read_text(encoding="utf-8"), encoding="utf-8") # ((Triforce, OS-04) vk_recorder.h's) (d / "vk_recorder.h").write_text(recorder_text, encoding="utf-8") src = str(ROOT / "tests" / "recorder_exit_test.cpp") flags = ["-std=c++20", "-O2", "-Wall", "-Wextra", "-Werror", "-DVK_NO_PROTOTYPES", "-D_CRT_SECURE_NO_WARNINGS"] diff --git a/tests/test_recording_admission.py b/tests/test_recording_admission.py index 58c48bd..23b701e 100644 --- a/tests/test_recording_admission.py +++ b/tests/test_recording_admission.py @@ -41,13 +41,16 @@ def main(): start = draw.index(' VkDescriptorSetAllocateInfo ai{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO};', draw.index('bool bind_descriptors(')) end = draw.index('\n return true;\n}', start) + len('\n return true;') fixture = fixture.replace('/* RES */', record).replace('/* STREAM */', stream) - fixture = fixture.replace('/* POOL */', block(draw, 'VkDescriptorPool make_pool(')) - fixture = fixture.replace('/* BEGIN */', block(draw, 'void draw_begin_recording(')).replace('/* DESCRIPTORS */', draw[start:end]) + # ((Triforce, 11-22, DESC-33) make_pool with the half-pool rule's state before it and its counter after it) + pool = draw[draw.index('bool pool_grow_on() {'):draw.index('void pool_took(')] + block(draw, 'void pool_took(') + fixture = fixture.replace('/* POOL */', pool) + fixture = fixture.replace('/* BEGIN */', block(draw, 'void slot_reset_locked(') + '\n' + block(draw, 'void draw_begin_recording(')).replace('/* DESCRIPTORS */', draw[start:end]) fixture = fixture.replace('/* BANK_CACHE */', block(draw, 'struct BankCache {') + ';') fixture = fixture.replace('/* BANK_BRANCH */', block(block(draw, 'VkDescriptorBufferInfo uniform_range('), 'if (index == 0 || index == 2)')) cases = ['healthy', 'no-recording', 'initial-buffer', 'initial-map', 'initial-commit', 'pool-retry', 'parallel', 'growth-buffer', 'growth-map', 'growth-commit', 'overflow', 'rounding', 'alignment-zero', - 'alignment-three', 'grow-off', 'reuse', 'retention', 'descriptor-pool', 'descriptor-set', 'descriptor-commit', 'descriptor-retained', 'bank-retry', 'bank-same'] + 'alignment-three', 'grow-off', 'reuse', 'retention', 'descriptor-pool', 'descriptor-set', 'descriptor-commit', 'descriptor-retained', 'bank-retry', 'bank-same', + 'pool-grow', 'pool-grow-off'] # ((Triforce, 11-22, DESC-33) the slots' pools at half: desc_pool_grow.h) if a.case != 'all': assert a.case in cases cases = [a.case] @@ -56,10 +59,12 @@ def main(): cpp.write_text(fixture, encoding='utf-8') flags = ['-fsanitize=address,undefined', '-fno-sanitize-recover=all'] if a.sanitize else [] subprocess.run([a.cxx, '-std=c++20', '-O1', '-pthread', '-Wall', '-Wextra', '-Werror', '-Wno-missing-field-initializers', *flags, - '-I' + str(a.headers), str(cpp), '-o', str(exe)], check=True) + '-I' + str(a.headers), '-I' + str(a.source / 'gpu' / 'vk'), str(cpp), '-o', str(exe)], check=True) env = {k: v for k, v in os.environ.items() if not k.startswith('RX_VK_')} for case in cases: - subprocess.run([str(exe), case], env=env, check=True, timeout=15) + # (pool-grow-off: the rule's switch back, RX_VK_DESC_POOL_GROW=0) + subprocess.run([str(exe), case], env={**env, 'RX_VK_DESC_POOL_GROW': '0'} if case == 'pool-grow-off' else env, + check=True, timeout=15) if __name__ == '__main__': diff --git a/tests/test_redraw_state.py b/tests/test_redraw_state.py new file mode 100644 index 0000000..1aaa0c2 --- /dev/null +++ b/tests/test_redraw_state.py @@ -0,0 +1,149 @@ +"""A redraw of the stored draws sets only what differs from what its pass has set (gpu/vk/redraw_state.h, +vk_temporal.cpp capture_motion; no game, no GPU). + +(Triforce, 11-21, the upscaler audit's UPS-03.) The stored draws are drawn again into the motion vectors in one pass, +about 900 a frame in the owner's log. Each set the primitive topology, the primitive restart and its index buffer before +it drew, whatever the draw before had set: nearly every draw of the scene is a triangle list without restart. + +This compiles redraw_state.h alone and checks the rule: a pass's first draw sets all three; a draw the same as the one +before sets none; each of the three is set again when it alone changes, and the index buffer when its buffer, its offset +or its type does; what was set is what the last command said, so a value that comes back is set again; a new pass knows +nothing of the one before; with RX_VK_REDRAW_STATE=0 every draw sets all three; and over a frame of 900 draws the commands +left out are counted. + +And reads the source: capture_motion asks the rule before each of the three commands, with one Pass of its own made after +begin_redraw and the switch read once; the push constants and the draw itself are recorded for every draw; and the switch +is in the switches table. + + python -I -B tests/test_redraw_state.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +PROGRAM = r""" +#include +#include +#include "redraw_state.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +using namespace redraw_state; +int main() { + CHECK(!every(nullptr) && !every("") && !every("1") && every("0")); + { // the first draw sets all three; the same again sets none + Pass p; + CHECK(topology(p, 3) && restart(p, false) && index(p, 0x1000, 64, 1)); + CHECK(!topology(p, 3) && !restart(p, false) && !index(p, 0x1000, 64, 1)); + CHECK(p.set == 3 && p.left == 3); + // (each alone, when it changes) + CHECK(topology(p, 4) && !restart(p, false) && !index(p, 0x1000, 64, 1)); + CHECK(!topology(p, 4) && restart(p, true) && !index(p, 0x1000, 64, 1)); + // (the index buffer: another buffer, another offset, another type) + CHECK(index(p, 0x2000, 64, 1) && !index(p, 0x2000, 64, 1)); + CHECK(index(p, 0x2000, 128, 1) && !index(p, 0x2000, 128, 1)); + CHECK(index(p, 0x2000, 128, 0) && !index(p, 0x2000, 128, 0)); + // (what is set is what the last command said: a value that comes back is set again) + CHECK(topology(p, 3) && topology(p, 4) && topology(p, 3) && !topology(p, 3)); + CHECK(restart(p, false) && restart(p, true) && !restart(p, true)); + CHECK(index(p, 0x1000, 64, 1) && index(p, 0x2000, 128, 0) && index(p, 0x1000, 64, 1)); + } + { // a draw without indices leaves the bound index buffer as it is: the next indexed draw of the same one sets nothing + Pass p; + CHECK(topology(p, 3) && restart(p, false) && index(p, 0x1000, 0, 1)); + CHECK(!topology(p, 3) && !restart(p, false)); // (a plain array of vertices: no index asked) + CHECK(!index(p, 0x1000, 0, 1)); + } + { // topology 0 (a point list) and a buffer at offset 0 are set too: nothing is taken as set before it was + Pass p; + CHECK(topology(p, 0) && restart(p, false) && index(p, 0, 0, 0)); + CHECK(!topology(p, 0) && !restart(p, false) && !index(p, 0, 0, 0)); + } + { // a new pass knows nothing of the one before + Pass a; + topology(a, 3), restart(a, false), index(a, 0x1000, 0, 1); + Pass b; + CHECK(topology(b, 3) && restart(b, false) && index(b, 0x1000, 0, 1)); + } + { // RX_VK_REDRAW_STATE=0: every draw sets all three, as before + Pass p; + p.all = true; + for (int i = 0; i < 50; i++) CHECK(topology(p, 3) && restart(p, false) && index(p, 0x1000, 64, 1)); + CHECK(p.set == 150 && p.left == 0); + } + { // a frame of 900 triangle lists without restart, a new index buffer place every third draw: 2 + 300 commands, not 2,700 + Pass p; + for (uint32_t i = 0; i < 900; i++) { + if (topology(p, 3)) {} + if (restart(p, false)) {} + if (index(p, 0x1000, (uint64_t)(i / 3) * 96, 1)) {} + } + CHECK(p.set == 2 + 300 && p.left == 2700 - 302); + } + return fails ? 1 : 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + src = (VK / "vk_temporal.cpp").read_text(encoding="utf-8") + + check('#include "redraw_state.h"' in src, "vk_temporal.cpp does not use the rule") + motion = block(src, "int capture_motion(VkCommandBuffer cmd, const Camera& cam, const float* before, const float* reproject, uint32_t rw,") + at = [motion.index(part) for part in ( + "if (!begin_redraw(cmd, s.capture, cam, before, to.jitter_before, reproject, rw, rh, dview, to.motion)) return 0;", + 'static const bool set_every = redraw_state::every(getenv("RX_VK_REDRAW_STATE"));', + "redraw_state::Pass set;", + "set.all = set_every;", + "for (const Redraw& d : draws) {", + "vkCmdPushConstants(cmd, s.actor_layout, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof d.push, &d.push);", + "if (redraw_state::topology(set, (uint32_t)k.topology)) vkCmdSetPrimitiveTopology(cmd, k.topology);", + "if (redraw_state::restart(set, k.restart)) vkCmdSetPrimitiveRestartEnable(cmd, k.restart ? VK_TRUE : VK_FALSE);", + "if (redraw_state::index(set, (uint64_t)d.ib.buffer, (uint64_t)d.ib.offset, (uint32_t)type))", + "vkCmdBindIndexBuffer(cmd, d.ib.buffer, d.ib.offset, type);", + "vkCmdDrawIndexed(cmd, (uint32_t)k.count, 1, 0, k.base_vertex, 0);", + "vkCmdDraw(cmd, (uint32_t)k.count, 1, (uint32_t)k.base_vertex, 0);", + "vkCmdEndRendering(cmd);")] + check(at == sorted(at), "capture_motion does not ask the rule before each of the three commands, in a pass of its own after begin_redraw") + loop = motion[motion.index("for (const Redraw& d : draws) {"):motion.index("vkCmdEndRendering(cmd);")] + check(loop.count("vkCmdSetPrimitiveTopology(") == 1 and loop.count("vkCmdSetPrimitiveRestartEnable(") == 1 and + loop.count("vkCmdBindIndexBuffer(") == 1, "a state command is recorded without the rule being asked") + check(motion.count("redraw_state::Pass set;") == 1 and "redraw_state::Pass" not in src.replace(motion, ""), + "the pass's state is not capture_motion's own, made anew for each pass") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`RX_VK_REDRAW_STATE=0`" in dev, "docs/development.md: RX_VK_REDRAW_STATE=0 is not in the switches table") + + with tempfile.TemporaryDirectory(prefix="redraw-state-") as tmp: + c, exe = Path(tmp) / "state.cpp", Path(tmp) / "state.exe" + c.write_text(PROGRAM, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(VK), str(c), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + check(r.returncode == 0, "the rule as made:\n" + r.stdout) + print("PASS: a redraw of the stored draws sets only what differs from what its pass has set") + + +if __name__ == "__main__": + main() diff --git a/tests/test_reflex_pace.py b/tests/test_reflex_pace.py index 1d72ae2..36c30bf 100644 --- a/tests/test_reflex_pace.py +++ b/tests/test_reflex_pace.py @@ -62,6 +62,7 @@ FIXTURE = r'''#define VK_NO_PROTOTYPES #include "dlssg_pace.h" #define CHECK(x) do { if (!(x)) { std::fprintf(stderr, "FAIL:%d %s (frame %d)\n", __LINE__, #x, frame_no); std::exit(1); } } while (0) static int frame_no; +namespace gpu_wait_kinds { struct Exclude { Exclude() {} }; } /* ((Triforce, 11-22, SYNC-27) the sleep's wait is left out of the GPU waits' count: tests/test_gpu_waits.py) */ /* ---- the clock, the host, the device, the menu */ static int64_t clock_ns = 1000000000ll; static int64_t now_ns() { return clock_ns; } diff --git a/tests/test_reflex_swapchain_lifetime.py b/tests/test_reflex_swapchain_lifetime.py index 5bbcec4..ab2927c 100644 --- a/tests/test_reflex_swapchain_lifetime.py +++ b/tests/test_reflex_swapchain_lifetime.py @@ -44,6 +44,7 @@ struct VkSetLatencyMarkerInfoNV {int type;uint64_t presentID=0;int marker=0;}; struct PresentId {const void* pNext=nullptr;unsigned swapchainCount=0;const uint64_t* pPresentIds=nullptr;}; void (*before_lock)()=nullptr; namespace lab_ab { inline bool on(const char*) { return false; } } // ((Triforce) the lab's in-run A/B knobs: none here) +namespace gpu_wait_kinds { struct Exclude { Exclude() {} }; } // ((Triforce, 11-22, SYNC-27) the sleep's wait is left out of the GPU waits' count: tests/test_gpu_waits.py) struct TestMutex { std::mutex native; void lock(){auto hook=before_lock;before_lock=nullptr;if(hook)hook();native.lock();} diff --git a/tests/test_region_free_vcore.py b/tests/test_region_free_vcore.py new file mode 100644 index 0000000..8b7c787 --- /dev/null +++ b/tests/test_region_free_vcore.py @@ -0,0 +1,349 @@ +"""A freed region's discard and the freeing thread's virtual core: HOS_DISCARD_KEEP_CORE (host/hos_mem.c discard_core, +hos_region_free, discard_pages; no game). + +(Triforce, 11-22, the performance audit's OS-40, after Source A's.) Not set, everything is as before: the core is given +up only for the discard's own kernel call and only with the memory limit on. 0 is Source A's: the freeing thread gives +its core up before the whole of the discard and takes it back once, after the region is unlisted. 1 keeps the core +through every discard. + +hos_region_free and the rule are cut out of hos_mem.c, hos_vcore_away and hos_vcore_back out of hos_thread.c, and run +over the real virtual cores (vc_lock to vc_yield, with tests/vcore_spin_test.c's stand-ins) and a discard that stands in +for the kernel's and takes the core as discard_pages does (the rule's answer for its own call). A process a switch +value, the switch being read once: + +- alone: a region with something to discard, the limit off and on; a region of the game's with none of its pages (a + bare free); an address that is no region. What the core is at the import's zero-fill and at the kernel's call, how + often it is taken back, that the region is still listed and marked while it is discarded and the regions' lock is + free then, and that the thread has its core again when the free returns; +- the wait this changes, on two threads: the freeing thread and another game thread allowed only on its core. With 0 the + other thread runs on the core while the discard is made, takes the regions' lock and finds the region still listed; + the freeing thread then waits for the core, having let the lock go first, so the other thread, still holding the core, + takes the lock again, finds the region gone, and gives the core back: nobody waits for good. Not set and with 1 the + other thread gets the core only after the free; +- the rule alone, over the switch's values and the limit. +Broken frees must fail: the core taken back under the regions' lock (the other thread, which holds the core, cannot get +the lock), never taken back, given up for a bare free, and given up when the switch is not set. + + python -I -B tests/test_region_free_vcore.py [--cc clang] +""" +import argparse +import os +import re +import shutil +import subprocess +import tempfile +from pathlib import Path + +RUN_ALL_ARGS = ["--cc", "clang"] + +ROOT = Path(__file__).resolve().parents[1] +FAR = "60000000" + + +def fail(message): + raise SystemExit("FAIL: " + message) + + +def block(source, marker): + start = source.index(marker) + end = source.index("{", start) + 1 + depth = 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[start:end] + + +CASES = r''' +/* ---- (Triforce, OS-40) hos_mem.c's freed regions over the real virtual cores ---- */ +static _Thread_local HosThread *cur; +@AWAY@ +@BACK@ +@RANGE@ +static Range store[8]; +static Range *regions = store; +static int nregions; +static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; +#define LOCK() pthread_mutex_lock(&lock) +#define UNLOCK() pthread_mutex_unlock(&lock) +static int64_t limit_on; +static int64_t hos_mem_limit(void) { return limit_on; } +static int hos_mem_census_on(void) { return 1; } +#define GC_SPACE (1ull << 40) +enum { GC_GAME = 1 }; +enum { HOS_MEM_DISCARDED = 3 }; +static int census, notes; +static uint64_t game_pages_in = 0x1000; +static uint64_t gc_count(int *m, int kind, uint64_t lo, uint64_t hi) { (void)m, (void)kind, (void)lo, (void)hi; return game_pages_in; } +static void hos_mem_note(int what, uint64_t addr, uint64_t size) { (void)addr, (void)size; CHECK(what == HOS_MEM_DISCARDED); notes++; } +@RULE@ +/* the discard, standing in for hos_mem.c's: what the core is at the import's zero-fill, then the call's own give-up by the + rule, the kernel's call (the case's hook: what another thread does meanwhile), and the core back if it gave it up */ +static int discards, core_at_zero_fill, core_at_kernel, listed_at_discard, lock_free_at_discard; +static void (*during)(void); +static void discard_pages(uint64_t addr, uint64_t size) { + (void)size; + discards++; + core_at_zero_fill = cur->vc_held; + lock_free_at_discard = pthread_mutex_trylock(&lock) == 0; + if (lock_free_at_discard) { + listed_at_discard = 0; + for (int i = 0; i < nregions; i++) listed_at_discard |= regions[i].addr == addr && regions[i].freeing; + pthread_mutex_unlock(&lock); + } + const int away = discard_core_call_away(discard_core(), hos_mem_limit() != 0) ? hos_vcore_away() : 0; + core_at_kernel = cur->vc_held; + if (during) during(); + if (away) hos_vcore_back(); +} +@NEEDED@ +@PAGES@ +@MARK@ +static int64_t region_frees_bare; +static int64_t region_frees_away, region_back_longest_ns; +@FREE@ + +static HosThread f, w; +static const uint64_t R = 0x100000000ull; +static int listed(uint64_t addr) { + int n = 0; + for (int i = 0; i < nregions; i++) n += regions[i].addr == addr; + return n; +} +static void region(uint64_t addr, int game) { + regions[nregions].addr = addr, regions[nregions].size = 0x40000, regions[nregions].freeing = 0, regions[nregions].game = game; + nregions++; +} +static int64_t takes(void) { + AcquireSRWLockExclusive(&vc_lock); + const int64_t v = vc_st.takes; + ReleaseSRWLockExclusive(&vc_lock); + return v; +} +/* the other game thread, allowed only on the freeing thread's core */ +static int w_ran, w_saw_listed, w_saw_gone, with_region_away; +static void *other(void *arg) { + (void)arg; + cur = &w; + hos_vcore_back(); /* (waits for core 0: until the freeing thread gives it up) */ + LOCK(); + w_saw_listed = listed(R); + UNLOCK(); + __atomic_store_n(&w_ran, 1, __ATOMIC_RELEASE); + if (with_region_away) { + wait_for(&f.vc_queued, 1); /* the freeing thread waits for its core, which this thread holds */ + int got = 0; /* it must have let the lock go before it queued */ + for (int i = 0; i < 10000 && !(got = pthread_mutex_trylock(&lock) == 0); i++) sleep_us(200); + if (!got) { + printf("FAIL: the freeing thread waits for its core with the regions' lock held: the thread that holds the core cannot go on\n"); + fflush(stdout); + _Exit(1); + } + w_saw_gone = !listed(R); + UNLOCK(); + } + hos_vcore_away(); + return NULL; +} +static void let_other_run(void) { /* the kernel's milliseconds: the other thread runs on the core meanwhile */ + wait_for(&w_ran, 1); +} +static void other_still_waits(void) { + CHECK(w.vc_queued == 1 && !__atomic_load_n(&w_ran, __ATOMIC_ACQUIRE)); +} +int main(int argc, char **argv) { + const char *c = argc > 1 ? argv[1] : ""; + const int mode = discard_core(); + { + pthread_t dog; + pthread_create(&dog, NULL, watchdog, NULL); + pthread_detach(dog); + } + cores = 2; + test_clock = 1000000000000LL; /* (standing still: no time slice, no escape) */ + thread_init(&f, "freeing", 20, 1); + thread_init(&w, "other", 20, 1); + cur = &f; + hos_vcore_back(); + CHECK(f.vc_held == 1); + if (!strcmp(c, "alone")) { + const int region_away = discard_core_region_away(mode); + for (limit_on = 0; limit_on <= 1; limit_on++) { + const int call_away = discard_core_call_away(mode, (int)limit_on); + region(R + 0x1000000, 0), region(R, 0); + const int64_t t0 = takes(); + discards = notes = 0; + hos_region_free(R); + CHECK(discards == 1 && f.vc_held == 1 && !f.vc_queued); /* the thread has its core when the free returns */ + CHECK(listed_at_discard && lock_free_at_discard); /* listed and marked meanwhile, the lock free */ + CHECK(nregions == 1 && !listed(R) && regions[0].addr == R + 0x1000000); + CHECK((core_at_zero_fill != 0) == !region_away); /* held over the zero-fill unless the region gave it up */ + CHECK((core_at_kernel != 0) == !(region_away || call_away)); /* and over the kernel's call unless either did */ + CHECK(takes() - t0 == (region_away || call_away ? 1 : 0)); /* taken back once, by whoever gave it up */ + nregions = 0; + } + /* a region of the game's with none of its pages, the limit on: nothing to discard, the core kept in every mode */ + limit_on = 1, game_pages_in = 0; + region(R, 1); + const int64_t t1 = takes(); + discards = notes = 0; + hos_region_free(R); + CHECK(discards == 0 && notes == 1 && region_frees_bare == 1 && takes() == t1 && f.vc_held == 1 && nregions == 0); + hos_region_free(R), hos_region_free(R + 0x5000); /* freed already, and no region: nothing */ + CHECK(discards == 0 && notes == 1 && takes() == t1 && f.vc_held == 1); + game_pages_in = 0x1000; + CHECK(region_frees_away == (region_away ? 2 : 0)); + /* the rule */ + CHECK(discard_core_rule(NULL) == DISCARD_CORE_AS_BEFORE && discard_core_rule("") == DISCARD_CORE_AS_BEFORE); + CHECK(discard_core_rule("0") == DISCARD_CORE_GIVEN_UP && discard_core_rule("1") == DISCARD_CORE_KEPT && discard_core_rule("2") == DISCARD_CORE_KEPT); + CHECK(!discard_core_region_away(DISCARD_CORE_AS_BEFORE) && discard_core_region_away(DISCARD_CORE_GIVEN_UP) && !discard_core_region_away(DISCARD_CORE_KEPT)); + CHECK(!discard_core_call_away(DISCARD_CORE_AS_BEFORE, 0) && discard_core_call_away(DISCARD_CORE_AS_BEFORE, 1)); /* (as before: the limit's) */ + CHECK(discard_core_call_away(DISCARD_CORE_GIVEN_UP, 0) && discard_core_call_away(DISCARD_CORE_GIVEN_UP, 1)); + CHECK(!discard_core_call_away(DISCARD_CORE_KEPT, 0) && !discard_core_call_away(DISCARD_CORE_KEPT, 1)); + } else if (!strcmp(c, "two")) { + with_region_away = discard_core_region_away(mode); + limit_on = 0; + region(R, 0); + pthread_t th; + pthread_create(&th, NULL, other, NULL); + wait_for(&w.vc_queued, 1); /* the other thread waits for the core this one holds */ + during = with_region_away ? let_other_run : other_still_waits; + hos_region_free(R); + CHECK(f.vc_held == 1 && !listed(R)); + if (with_region_away) { + pthread_join(th, NULL); + CHECK(w_ran && w_saw_listed == 1 && w_saw_gone == 1); /* it ran during the discard, and again once the region was gone */ + CHECK(core_at_zero_fill == 0 && core_at_kernel == 0); + } else { + CHECK(!__atomic_load_n(&w_ran, __ATOMIC_ACQUIRE) && core_at_zero_fill == 1 && core_at_kernel == 1); + hos_vcore_away(); /* (a block of the freeing thread's: only now the other one runs) */ + pthread_join(th, NULL); + CHECK(w_ran && w_saw_listed == 0); + hos_vcore_back(); + } + CHECK(f.vc_held == 1); + } else { + FAIL("unknown case %s", c); + } + hos_vcore_away(); + end_state(); + if (failures) return 1; + printf("PASS %s\n", c); + return 0; +} +''' + +AWAY = " if (discard_core_region_away(discard_core())) away = hos_vcore_away();\n" +NEEDED = " if (hos_region_discard_needed(game && hos_mem_limit(), hos_mem_census_on(), game_pages)) {\n" +BACK = " hos_vcore_back();\n" +LAST_UNLOCK = " UNLOCK();\n /* ((Triforce, 11-22, OS-40) the core back, after the lock is let go:" +# (each: what the broken free does, the switch's value and the case that must fail, and the change to the cut hos_region_free) +MUTANTS = [ + ("takes the core back under the regions' lock", "0", "two", [(BACK, ""), (LAST_UNLOCK, " if (away) hos_vcore_back();\n" + LAST_UNLOCK)]), + ("never takes the core back", "0", "alone", [(BACK, "")]), + ("gives the core up for a bare free", "0", "alone", [(AWAY, ""), (NEEDED, " if (discard_core_region_away(discard_core())) away = hos_vcore_away();\n" + NEEDED)]), + ("gives the core up with the switch not set", None, "alone", [(AWAY, " away = hos_vcore_away();\n")]), + ("gives the core up with the switch at 1", "1", "two", [(AWAY, " away = hos_vcore_away();\n")]), +] + + +def read_sources(mem, dev): + free = block(mem, "void hos_region_free(uint64_t addr) {") + order = [NEEDED, AWAY, " discard_pages(addr, freed_size);\n", " nregions--;\n", LAST_UNLOCK, " if (away) {\n", BACK] + at = -1 + for needle in order: + nxt = free.find(needle, at + 1) + if nxt < 0: + fail("hos_mem.c hos_region_free: %r is not there, or not after what comes before it" % needle.strip()) + at = nxt + if free.count("hos_vcore_away()") != 1 or free.count("hos_vcore_back()") != 1 or free.count("LOCK();") != 4: + fail("hos_mem.c hos_region_free: more than one give-up or take-back, or its two locked parts changed") + discard = block(mem, "static void discard_pages(uint64_t addr, uint64_t size) {") + if "const int away = discard_core_call_away(discard_core(), hos_mem_limit() != 0) ? hos_vcore_away() : 0;" not in discard: + fail("hos_mem.c discard_pages: the call's own give-up is not the rule's") + core = block(mem, "static int discard_core(void) {") + if 'mode = discard_core_rule(getenv("HOS_DISCARD_KEEP_CORE"));' not in core or "static int mode = -1;" not in core: + fail("hos_mem.c discard_core: HOS_DISCARD_KEEP_CORE is not read once through the rule") + if "| `HOS_DISCARD_KEEP_CORE=0`, `=1` |" not in dev: + fail("docs/development.md: HOS_DISCARD_KEEP_CORE is not in the switches table") + return free + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cc", default="clang") + args = parser.parse_args() + try: + mem = (ROOT / "host/hos_mem.c").read_text(encoding="utf-8") + thread = (ROOT / "host/hos_thread.c").read_text(encoding="utf-8") + free = read_sources(mem, (ROOT / "docs/development.md").read_text(encoding="utf-8")) + range_t = re.findall(r"^typedef struct \{[^}]*\} Range;$", mem, re.M) + pages_used = re.findall(r"^static uint64_t pages_used\[[^\n]*\];$", mem, re.M) + if len(range_t) != 1 or len(pages_used) != 1: + fail("hos_mem.c's Range or pages_used not found once") + pieces = {"@AWAY@": block(thread, "int hos_vcore_away(void) {"), "@BACK@": block(thread, "void hos_vcore_back(void) {"), + "@RANGE@": range_t[0], "@PAGES@": pages_used[0], "@MARK@": block(mem, "static void pages_mark_used_locked("), + "@RULE@": mem[mem.index("enum { DISCARD_CORE_AS_BEFORE, DISCARD_CORE_GIVEN_UP, DISCARD_CORE_KEPT };"):mem.index("static void discard_pages(")], + "@NEEDED@": block(mem, "int hos_region_discard_needed(int game_region, int census_on, uint64_t game_pages) {")} + cores = thread[thread.index("static SRWLOCK vc_lock = SRWLOCK_INIT;"):thread.index("int hos_vcore_away(void) {")] + spin = (ROOT / "tests/vcore_spin_test.c").read_text(encoding="utf-8") + stand_ins = spin[:spin.index("/* ---- storm: many hand-offs")] + except (ValueError, OSError) as e: + fail("a text this test reads is not there (%s)" % e) + compiler = shutil.which(args.cc) + if not compiler: + raise SystemExit("C compiler not found: " + args.cc) + env = {k: v for k, v in os.environ.items() if not k.startswith("HOS_")} + env.update(HOS_VCORE_ESCAPE_US=FAR, HOS_VCORE_ESCAPE_EQ_US=FAR, HOS_VCORE_SLICE_US=FAR, HOS_VCORE_SPIN_US="0") + flags = subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0 + with tempfile.TemporaryDirectory(prefix="region-free-") as temporary: + top = Path(temporary) + + def build(name, free_text): + folder = top / name + folder.mkdir() + (folder / "vcore_block.inc").write_text(cores, encoding="utf-8") + text = stand_ins + CASES + for mark, piece in dict(pieces, **{"@FREE@": free_text}).items(): + text = text.replace(mark, piece) + (folder / "free.c").write_text(text, encoding="utf-8") + exe = folder / "free.exe" + built = subprocess.run([compiler, "-std=gnu17", "-O1", "-Wall", "-Werror", "-Wno-unused-function", "-Wno-unused-variable", + "-Wno-unknown-warning-option", "-Wno-unused-but-set-variable", "-Wno-unused-but-set-global", "-pthread", + "-I" + str(folder), str(folder / "free.c"), "-o", str(exe)], capture_output=True, text=True, timeout=180) + if built.returncode: + print(built.stdout + built.stderr) + fail("the fixture did not build (%s)" % name) + return exe + + def run(exe, case, switch): + case_env = dict(env) if switch is None else dict(env, HOS_DISCARD_KEEP_CORE=switch) + try: + return subprocess.run([str(exe), case], env=case_env, capture_output=True, text=True, timeout=120, creationflags=flags) + except subprocess.TimeoutExpired: + return subprocess.CompletedProcess([], 1, "no end in 120 s (a thread waits for good)", "") + + exe = build("as-made", free) + for switch, what in ((None, "not set: as before"), ("0", "0: the core given up for the whole discard"), ("1", "1: the core kept")): + for case in ("alone", "two"): + done = run(exe, case, switch) + if done.returncode or "PASS " + case not in done.stdout: + print(done.stdout[-3000:] + done.stderr[-1000:]) + fail("HOS_DISCARD_KEEP_CORE %s, the case %s" % (what, case)) + print("PASS HOS_DISCARD_KEEP_CORE " + what) + for i, (what, switch, case, changes) in enumerate(MUTANTS): + text = free + for old, new in changes: + if text.count(old) != 1: + fail("mutation %r: %r is not in the cut hos_region_free once (update the test with the code)" % (what, old.strip())) + text = text.replace(old, new) + done = run(build("mutant%d" % i, text), case, switch) + if done.returncode == 0: + fail("a free that %s still passed" % what) + said = [line for line in done.stdout.splitlines() if line.startswith("FAIL")] or [done.stdout.strip()[-120:]] + print("PASS mutation: a free that %s fails (%s)" % (what, said[0][:120])) + print("PASS: a freed region's discard keeps or gives up the core by HOS_DISCARD_KEEP_CORE, as before when it is not set, and nobody waits for good") + + +if __name__ == "__main__": + main() diff --git a/tests/test_release_queue.py b/tests/test_release_queue.py new file mode 100644 index 0000000..a806fa5 --- /dev/null +++ b/tests/test_release_queue.py @@ -0,0 +1,236 @@ +"""Texture and program releases queued for the replaying thread (gpu/vk/release_queue.h, vk_replay.cpp texture_released_of, +program_released_locked, releases_apply_locked; no game, no GPU). + +(Triforce, 11-22, the performance audit's MEM-85, after Source B's idea.) Without the replay thread a release was held for +the replaying thread only while a queue was stopped (PR 430), and otherwise waited for sync_drain and the replay's lock on +the game's thread. RX_VK_RELEASE_QUEUE=1 (OFF unless set; lab knob relq) pushes every such release to a list the replaying +thread applies at a submission's replay start or a present with no queue stopped, and update_blocked after the stopped +work; a texture's release carries the write generation of its call. A count of the old way's waits is on in every run. + +Compiles the header: the switch's text and knob; way() against the old decision over every input with the switch off, and +Queued whatever is stopped with it on; apply_now; the count's numbers, its line and its reset. Then a model: random streams +of queues stopping and going on, releases, submissions and presents, applied by the header's rule: with the switch on no +release is applied while a queue is stopped, none is left once the replaying thread has run with none stopped, and each is +applied with the generation of its call; with it off, the model's releases go the old ways (held while stopped, else at once). + +And reads the source: both release entries decide by way() with the old decision's parts, the queued push under held_mu +with the call's generation and the count, the held branch as it was, the old way's sync_drain and lock timed, the job path +of the replay thread after them unchanged; releases_apply_locked asks apply_now, swaps under held_mu and applies as the +do_ functions do without taking replay_mu; run_submit and do_present call it first under replay_mu, update_blocked after +the stopped work's releases under replay_mu; texture_released_at takes the generation it is given; the switches table. + + python -I -B tests/test_release_queue.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +PROGRAM = r""" +#include "release_queue.h" +#include +#include +#include +using namespace release_queue; +static int fails = 0; +#define CHECK(c) do { if (!(c)) { std::printf("FAIL line %d: %s\n", __LINE__, #c); fails++; } } while (0) + +int main() { + CHECK(!on(nullptr, false) && on(nullptr, true) && !on("0", true) && on("1", false) && on("2", false) && !on("x", true)); + for (int q = 0; q < 2; q++) + for (int h = 0; h < 2; h++) + for (int s = 0; s < 2; s++) + for (int b = 0; b < 2; b++) { + const Way w = way(q, h, s, b); + if (q) CHECK(w == Way::Queued); // (the switch's push ignores stopped queues) + else CHECK(w != Way::Queued && (w == Way::Held) == held_before(h, s, b)); + } + CHECK(!apply_now(0, 0) && apply_now(1, 0) && !apply_now(3, 1) && !apply_now(0, 2)); + { + Waits w; + w.note(100), w.note(Waits::kWaitNs), w.note(Waits::kWaitNs * 3), w.note_queued(); + CHECK(w.n == 3 && w.waited == 2 && w.ns == 100 + Waits::kWaitNs * 4 && w.max_ns == Waits::kWaitNs * 3 && w.queued == 1); + CHECK(!w.due() && w.worth_a_line()); + Waits q; + for (int i = 0; i < Waits::kEvery - 1; i++) q.note(10); + CHECK(!q.due() && !q.worth_a_line()); + q.note_queued(); + CHECK(q.due() && q.worth_a_line()); + q.reset(); + CHECK(q.n == 0 && q.queued == 0 && q.max_ns == 0 && !q.due()); + } + // (the model: 4 queues; a stream of stops, resumes (update_blocked), releases, submissions' starts and presents) + std::mt19937_64 rng(85); + for (int run = 0; run < 4000; run++) { + const bool qon = run % 2; + bool stopped[4] = {}; + int blocked = 0; + uint64_t gen = 1; + struct Rel { int id; uint64_t gen; bool stopped_at_call[4]; }; + std::vector queued, held; + std::vector applied; + int next_id = 0; + auto apply = [&](std::vector& list) { + for (const Rel& r : list) { + for (int k = 0; k < 4; k++) CHECK(!(r.stopped_at_call[k] && stopped[k])); // (never before stopped work) + applied.push_back(r.id); + } + list.clear(); + }; + auto replaying = [&]() { // (run_submit's start, do_present: replay_mu held) + if (apply_now(queued.size(), blocked)) apply(queued); + }; + for (int step = 0; step < 200; step++) { + const int e = (int)(rng() % 6); + if (e == 0) { // (a queue stops at a wait) + const int k = (int)(rng() % 4); + if (!stopped[k]) stopped[k] = true, blocked++; + } else if (e == 1) { // (a queue goes on; update_blocked) + const int k = (int)(rng() % 4); + if (stopped[k]) stopped[k] = false, blocked--; + if (!blocked) { + apply(held); + if (!queued.empty()) apply(queued); // (update_blocked's, after the held ones) + } + } else if (e == 2 || e == 3) { // (a release on the game's thread) + Rel r{next_id++, gen, {}}; + for (int k = 0; k < 4; k++) r.stopped_at_call[k] = stopped[k]; + const Way w = way(qon, true, true, blocked != 0); + if (w == Way::Queued) queued.push_back(r); + else if (w == Way::Held) held.push_back(r); + else { CHECK(blocked == 0); applied.push_back(r.id); } // (the old way: at once, nothing stopped) + if (!qon) CHECK(queued.empty()); + } else if (e == 4) { // (a submission's replay starts; the CPU's writes make generations) + replaying(); + gen += 1 + rng() % 3; + } else { + replaying(); // (a present) + } + } + // (the replaying thread with nothing stopped: nothing left) + for (int k = 0; k < 4; k++) stopped[k] = false; + blocked = 0; + apply(held); + replaying(); + CHECK(queued.empty() && held.empty() && (int)applied.size() == next_id); + } + // (each queued release keeps its call's generation through the list, as releases_apply_locked hands it on) + { + struct Q { int id; uint64_t gen; }; + std::vector list; + for (int i = 0; i < 10; i++) list.push_back({i, 100 + (uint64_t)i * 7}); + std::vector taken; + taken.swap(list); + for (int i = 0; i < 10; i++) CHECK(taken[i].gen == 100 + (uint64_t)i * 7); + CHECK(list.empty()); + } + std::printf(fails ? "%d failed\n" : "ok\n", fails); + return fails ? 1 : 0; +} +""" + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def in_order(text, steps, what): + at = -1 + for s in steps: + k = text.find(s, at + 1) + if k < 0: + raise SystemExit("FAIL: %s: %r is missing or out of its order" % (what, s)) + at = k + + +def check_source(): + replay = (VK / "vk_replay.cpp").read_text(encoding="utf-8") + texture = (VK / "vk_texture.cpp").read_text(encoding="utf-8") + memory = (VK / "vk_memory.cpp").read_text(encoding="utf-8") + way = "const release_queue::Way way = release_queue::way(release_queue_on(), hold_releases(), sync_waits_on(), blocked_n.load() != 0);" + tex = block(replay, "void texture_released_of(const nvn_tex* t) {") + in_order(tex, ["tex_pages_add(t, -1);", "if (!async_on()) {", way, "if (way == release_queue::Way::Queued) {", + "std::lock_guard hl(held_mu);", "queued_tex_releases.push_back({*t, memory_generation_any_thread()});", + "queued_releases.fetch_add(1, std::memory_order_release);", "release_count(-1);", "return;", + "if (way == release_queue::Way::Held) {", "held_tex_releases.push_back(*t);", "return;", + "const auto r0 = std::chrono::steady_clock::now();", "sync_drain(1);", "do_texture_released(t);", + "release_count(", "return;", "Job j{JOB_TEXTURE_RELEASED};", "enqueue(std::move(j));"], + "vk_replay.cpp texture_released_of") + prog = block(replay, "void program_released_locked(const nvn_prog* p) {") + in_order(prog, ["if (!async_on()) {", way, "if (way == release_queue::Way::Queued) {", "std::lock_guard hl(held_mu);", + "queued_prog_releases.push_back(*p);", "queued_releases.fetch_add(1, std::memory_order_release);", + "release_count(-1);", "return;", "if (way == release_queue::Way::Held) {", "held_prog_releases.push_back(*p);", + "st_held_releases++;", "return;", "const auto r0 = std::chrono::steady_clock::now();", "sync_drain(1);", + "do_program_released(p);", "release_count(", "return;", "Job j{JOB_PROGRAM_RELEASED};"], + "vk_replay.cpp program_released_locked") + on = block(replay, "bool release_queue_on() {") + if 'static const char* env = getenv("RX_VK_RELEASE_QUEUE");' not in on or \ + 'return release_queue::on(env, lab_ab::on("relq"));' not in on: + raise SystemExit("FAIL: release_queue_on does not read RX_VK_RELEASE_QUEUE once with the knob relq") + apply = block(replay, "void releases_apply_locked() {") + in_order(apply, ["if (!release_queue::apply_now(queued_releases.load(std::memory_order_acquire), blocked_n.load())) return;", + "std::lock_guard hl(held_mu);", "ts.swap(queued_tex_releases);", "ps.swap(queued_prog_releases);", + "queued_releases.store(0, std::memory_order_release);", "mkwvk::texture_released(q.t.serial);", + "mkwvk::texture_released_at(&q.t, q.gen);", "program_released_after_reads(&p);"], + "vk_replay.cpp releases_apply_locked") + if "(replay_mu)" in apply or "do_texture_released(" in apply or "do_program_released(" in apply: + raise SystemExit("FAIL: releases_apply_locked takes replay_mu again (its callers hold it)") + run = block(replay, "bool run_submit(const uint64_t* handles, int n, uint64_t value, uint64_t queue, bool memory_work,") + in_order(run, ["std::lock_guard lk(replay_mu);", "releases_apply_locked();", "ReplayPin pin;"], "vk_replay.cpp run_submit") + present = block(replay, "void do_present(const nvn_tex* t, const int crop[4]) {") + in_order(present, ["std::lock_guard lk(replay_mu);", "releases_apply_locked();", "bench_frame(presents);"], + "vk_replay.cpp do_present") + blocked = block(replay, "void update_blocked() {") + in_order(blocked, ["if (!n) {", "for (const nvn_tex& t : ts) do_texture_released(&t);", + "for (const nvn_prog& p : ps) do_program_released(&p);", + "if (queued_releases.load(std::memory_order_acquire)) {", "std::lock_guard lk(replay_mu);", + "releases_apply_locked();"], "vk_replay.cpp update_blocked") + if replay.count("releases_apply_locked();") != 3: + raise SystemExit("FAIL: the queued releases are applied somewhere else") + if replay.count("queued_tex_releases.push_back(") != 1 or replay.count("queued_prog_releases.push_back(") != 1: + raise SystemExit("FAIL: a release is queued somewhere else") + released = block(texture, "void texture_released(const nvn_tex* t) {") + if "texture_released_at(t, memory_generation());" not in released: + raise SystemExit("FAIL: vk_texture.cpp texture_released does not release from the generation now") + at = block(texture, "void texture_released_at(const nvn_tex* t, WriteGen gen) {") + if "it->second->released = true, it->second->release_gen = gen;" not in at or "memory_generation()" in at: + raise SystemExit("FAIL: vk_texture.cpp texture_released_at does not take the generation it is given") + any_thread = block(memory, "WriteGen memory_generation_any_thread() {") + if "return gen_load(gens.now);" not in any_thread: + raise SystemExit("FAIL: memory_generation_any_thread is not an atomic load of gens.now") + dev_md = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + if not any(line.startswith("| `RX_VK_RELEASE_QUEUE=1` |") and "**off unless set**" in line and "`relq`" in line + for line in dev_md.splitlines()): + raise SystemExit("FAIL: docs/development.md has no row for RX_VK_RELEASE_QUEUE with its default") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + check_source() + with tempfile.TemporaryDirectory(prefix="release-queue-") as tmp: + src = Path(tmp) / "t.cpp" + exe = Path(tmp) / "t.exe" + src.write_text(PROGRAM, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(VK), str(src), "-o", str(exe)], + check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True, timeout=120) + print(r.stdout.strip()) + if r.returncode: + raise SystemExit("FAIL: release_queue.h") + print("ok: test_release_queue") + + +if __name__ == "__main__": + main() diff --git a/tests/test_replay_hot_trims.py b/tests/test_replay_hot_trims.py new file mode 100644 index 0000000..879c725 --- /dev/null +++ b/tests/test_replay_hot_trims.py @@ -0,0 +1,319 @@ +"""Small per-draw and per-part costs cut on the replay thread, with the same results (vk_draw.cpp, vk_replay.cpp, +vk_dump.cpp, vk_memory.cpp; no game, no GPU). + +(Triforce, 11-22, the performance audit's DESC-06, after Source A's.) Five places did work at every draw, uniform buffer, +dispatch or part of a submission that nothing needed there: + 1. a uniform buffer's offset tested against its alignment with a division: a mask (the alignment is a power of two); + 2. ub_overlay's allocator a std::function, made of a capturing lambda at every uniform buffer: a plain function and + its alignment; + 3. dump_on() a call into vk_dump.cpp for one bool, at every draw: inline; + 4. dyn_same's 35 words and up to 160 bytes compared in loops: two memcmp (tests/test_same_dyn.py runs it); + 5. RX_VK_LOG_PUSH's program key formatted at every dispatch: only while the switch is set and its line not written; + 6. retired_imports_give_back summing every live pool and walking every retired pool at every part, for a line logged + every 1200 epochs: summed when the line is due, and the walk begins past the retired pools with no import left. + +Reads the source for 1 to 5. For 6, cuts retired_imports_give_back out of vk_memory.cpp and runs it beside the function +as it was (kept below) over the same authored session of 6000 parts, with the imports given back (NVIDIA's default, +RX_VK_KEEP_RETIRED=0) and kept (=1): the same imports given back at the same parts with the same values, the same +log lines, and the pools' state the same at the end; and the new one looks at far fewer retired pools. + +Not done from the entry, and checked absent here so that nobody takes them for done: the read-back refresh's skip +without mirrors (its test, tests/test_parallel_rb_refresh.py, does not pass on v0.6.0 either) and the Linux-only lock +type of pools_mu (not compilable on Windows). + + python -I -B tests/test_replay_hot_trims.py [--cxx clang++] +""" +import argparse +import os +import re +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +# retired_imports_give_back as it was before DESC-06 (v0.6.0), the oracle +BEFORE = r''' +void retired_imports_give_back() { // (pools_mu held) +#ifdef _WIN32 + static const bool keep = getenv("RX_VK_KEEP_RETIRED") ? getenv("RX_VK_KEEP_RETIRED")[0] != '0' : dev.props.vendorID != 0x10DE; +#else + static const bool keep = getenv("RX_VK_KEEP_RETIRED") && getenv("RX_VK_KEEP_RETIRED")[0] != '0'; +#endif + uint64_t live_bytes = 0, retired_bytes = 0; + size_t retired_alive = 0; + for (auto& [lo, p] : imported) live_bytes += p.hi - p.lo; + for (auto& nh : retired_pools) { + Pool& p = nh.mapped(); + if (!p.memory) continue; + if (!keep && p.mstate == Pool::MIRROR_NONE && !p.inherit_pending && p.retired_value <= mem_completed && + p.last_value <= mem_completed) { + dead_imports.push_back({p.buffer, p.memory, std::max(mem_rec_value, p.last_value), p.hi - p.lo, std::move(p.pieces), p.lo, p.hi}); + p.pieces.clear(); + p.buffer = VK_NULL_HANDLE, p.memory = VK_NULL_HANDLE, p.address = 0; + pools_gen++; + continue; + } + retired_alive++, retired_bytes += p.hi - p.lo; + } + static uint64_t logged_epoch; + if (mirror_epoch - logged_epoch >= 1200) { + logged_epoch = mirror_epoch; + mirror_use_report(); + char touched[160] = ""; + if (const int64_t pages = st_touch_pages.load(std::memory_order_relaxed)) + snprintf(touched, sizeof touched, "; %lld MB of pages read before their imports in %.0f ms (RX_VK_IMPORT_TOUCH; %lld reads " + "stopped for a waiting lookup)", (long long)(pages >> 8), st_touch_ns.load(std::memory_order_relaxed) / 1e6, + (long long)st_touch_cut.load(std::memory_order_relaxed)); + logf("vk: imports: %zu live (%llu MB), %zu retired still imported (%llu MB), %llu retired given back (%llu MB)%s", + imported.size(), (unsigned long long)(live_bytes >> 20), retired_alive, (unsigned long long)(retired_bytes >> 20), + (unsigned long long)st_retired_imports_freed, (unsigned long long)(st_retired_import_bytes_freed >> 20), touched); + } +} +''' + +FIXTURE = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define VK_NULL_HANDLE 0 +using VkBuffer = uint64_t; +using VkDeviceMemory = uint64_t; +struct { struct { uint32_t vendorID = 0x10DE; } props; } dev; +static uint64_t looked; // (how many retired pools the walk asked for their memory) +struct Memory { // (a pool's memory handle, counting each look at it) + uint64_t h = 0; + Memory& operator=(uint64_t v) { h = v; return *this; } + explicit operator bool() const { looked++; return h != 0; } + operator uint64_t() const { return h; } + bool operator!() const { looked++; return h == 0; } +}; +struct Pool { + enum MState { MIRROR_NONE, MIRROR_ON, MIRROR_REFUSED }; + uint64_t lo = 0, hi = 0, address = 0, retired_value = 0, last_value = 0; + int64_t retired_ms = 0; // ((Triforce, 11-22) MEM-21's, merged into this walk) + VkBuffer buffer = 0; + Memory memory; + MState mstate = MIRROR_NONE; + bool inherit_pending = false; + std::vector pieces; +}; +struct DeadImport { VkBuffer buffer; VkDeviceMemory memory; uint64_t value, bytes; std::vector pieces; uint64_t lo, hi; }; +std::map imported; +std::deque::node_type> retired_pools; +std::deque dead_imports; +uint64_t mem_rec_value = 0, mem_completed = 0, mirror_epoch = 0, pools_gen = 0; +uint64_t st_retired_imports_freed = 0, st_retired_import_bytes_freed = 0; +std::atomic st_touch_pages{0}, st_touch_ns{0}, st_touch_cut{0}; +// ((Triforce, 11-22) what MEM-21, merged into the walk, reads: its header, and RX_VK_KEEP_RETIRED read as the game reads +// it on Windows. Its wait (RX_VK_KEEP_RETIRED=wait) is MEM-21's own test's; here the three old readings must give the trace +// they gave before both changes) +uint64_t dead_import_bytes = 0; +bool system_heap_usage(uint64_t* used, uint64_t* budget) { *used = 0, *budget = 0; return false; } +/* RG_HEADER */ +retired_give_back::Mode retired_mode() { + return retired_give_back::mode(getenv("RX_VK_KEEP_RETIRED"), true, dev.props.vendorID == 0x10DE); +} +bool retired_keep_set() { return getenv("RX_VK_KEEP_RETIRED") != nullptr; } +static uint64_t trace = 1469598103934665603ull; // (everything that happened, hashed in order) +static void note(const void* d, size_t n) { + for (size_t i = 0; i < n; i++) trace = (trace ^ ((const uint8_t*)d)[i]) * 1099511628211ull; +} +static int lines, reports; +void mirror_use_report() { reports++, note("report", 6); } +void logf(const char* f, ...) { + char b[1024]; + va_list ap; + va_start(ap, f); + vsnprintf(b, sizeof b, f, ap); + va_end(ap); + lines++, note(b, strlen(b)); + if (lines <= 2) printf("%s\n", b); +} +// (the system heap's rule for kept retired imports, (Triforce) 11-22: never pressed here, as with RX_VK_KEEP_RETIRED set, +// where it is not asked; NVIDIA's default gives them back without it) +std::atomic import_room_refused_ms{0}; +int64_t import_now_ms() { return 0; } +namespace mem_parts { +[[maybe_unused]] constexpr int64_t kRefusedHoldMs = 60000; +bool retired_pressed(uint64_t, uint64_t, int, int64_t, int64_t) { return false; } +} +/* FUNCTION */ + +static uint64_t seed = 88172645463325252ull; +static uint32_t rnd(uint32_t n) { + seed ^= seed << 13, seed ^= seed >> 7, seed ^= seed << 17; + return (uint32_t)((seed >> 11) % n); +} +static uint64_t next_lo = 0x10000000; +static void retire() { + Pool p; + p.lo = next_lo, p.hi = next_lo + ((uint64_t)(1 + rnd(64)) << 20), next_lo = p.hi + (1u << 20); + p.buffer = p.lo + 1, p.address = p.lo; + if (rnd(10)) p.memory = p.lo + 2; // (one in ten retired with its import gone already) + if (!rnd(6)) p.mstate = Pool::MIRROR_ON; // (mirrored: its import stays) + if (!rnd(5)) p.inherit_pending = true; // (cleared later, below) + p.retired_value = mem_rec_value, p.last_value = mem_rec_value + rnd(4); + if (!rnd(3)) p.pieces = {p.lo + 3, p.lo + 4}; + std::map one; + one.emplace(p.lo, std::move(p)); + retired_pools.push_back(one.extract(one.begin())); +} + +int main() { + for (int i = 0; i < 40; i++) { // (the live pools: their sizes are in the line) + Pool p; + p.lo = 0x1000000ull * (uint64_t)(i + 1), p.hi = p.lo + ((uint64_t)(1 + rnd(32)) << 20), p.memory = p.lo + 2; + imported.emplace(p.lo, std::move(p)); + } + size_t dead_seen = 0; + for (int part = 0; part < 6000; part++) { + mem_rec_value++; + if (rnd(3)) mem_completed = mem_rec_value > 2 ? mem_rec_value - rnd(3) : 0; + if (!rnd(4)) retire(); + if (!rnd(7)) { // (the oldest inheritance ends, the oldest mirror goes: that pool can be given back from now) + static size_t held; + while (held < retired_pools.size() && !retired_pools[held].mapped().inherit_pending && + retired_pools[held].mapped().mstate == Pool::MIRROR_NONE) + held++; + if (held < retired_pools.size()) retired_pools[held].mapped().inherit_pending = false, retired_pools[held].mapped().mstate = Pool::MIRROR_NONE; + } + if (part == 2000) st_touch_pages = 5 << 8, st_touch_ns = 3000000, st_touch_cut = 2; + retired_imports_give_back(); + mirror_epoch++; + for (; dead_seen < dead_imports.size(); dead_seen++) { + const DeadImport& d = dead_imports[dead_seen]; + const uint64_t w[] = {(uint64_t)part, d.buffer, d.memory, d.value, d.bytes, d.lo, d.hi, d.pieces.size()}; + note(w, sizeof w); + st_retired_imports_freed++, st_retired_import_bytes_freed += d.bytes; // (as reap_mirrors counts them) + } + } + for (auto& nh : retired_pools) { // (and what is left of every retired pool) + const Pool& p = nh.mapped(); + const uint64_t w[] = {p.lo, p.buffer, (uint64_t)p.memory, p.address, p.pieces.size(), (uint64_t)p.mstate, (uint64_t)p.inherit_pending}; + note(w, sizeof w); + } + printf("trace %016llx retired %zu dead %zu lines %d reports %d gen %llu\n", (unsigned long long)trace, retired_pools.size(), + dead_imports.size(), lines, reports, (unsigned long long)pools_gen); + printf("looked %llu\n", (unsigned long long)looked); + return 0; +} +''' + + +def block(text, marker): + start = text.index(marker) + at = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[at] == "{") - (text[at] == "}") + at += 1 + return text[start:at] + + +def need(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def check_source(): + draw = (ROOT / "gpu/vk/vk_draw.cpp").read_text(encoding="utf-8") + replay = (ROOT / "gpu/vk/vk_replay.cpp").read_text(encoding="utf-8") + uniform = block(draw, "VkDescriptorBufferInfo uniform_range(") + # 1 + need("% align" not in uniform and "if (b && !(b.offset & (align - 1))) return {b.buffer, b.offset, n};" in uniform, + "uniform_range still divides a buffer's offset by its alignment") + need("const uint64_t align = std::max(dev.props.limits.minUniformBufferOffsetAlignment, 16);" in uniform, + "uniform_range's alignment is no longer the device's (a power of two): the mask needs that") + # 2 + need("std::function" not in block(replay, "uint64_t ub_overlay(") and + "uint64_t ub_overlay(uint64_t addr, uint64_t n, uint8_t* base, uint64_t (*alloc)(uint64_t, uint64_t), uint64_t align) {" in replay and + "const uint64_t at = alloc(n, align);" in block(replay, "uint64_t ub_overlay("), + "ub_overlay still takes a std::function, or does not hand its allocator the alignment") + need("uint64_t ub_overlay(uint64_t addr, uint64_t n, uint8_t* base, uint64_t (*alloc)(uint64_t, uint64_t), uint64_t align);" + in (ROOT / "gpu/vk/vk_draw.h").read_text(encoding="utf-8"), "vk_draw.h declares another ub_overlay") + need("ub_overlay(u.addr, n, cur->mapped, stream_alloc, align); at != UINT64_MAX" in uniform and + "uint64_t stream_alloc(uint64_t n, uint64_t align) {" in draw, + "uniform_range does not hand ub_overlay stream_alloc and its alignment") + need(not re.search(r"ub_overlay\([^;]*\[&\]", draw), "a caller of ub_overlay builds a capturing lambda for it") + # 3 + dump_h = (ROOT / "gpu/vk/vk_dump.h").read_text(encoding="utf-8") + dump = (ROOT / "gpu/vk/vk_dump.cpp").read_text(encoding="utf-8") + need("extern bool dump_armed;" in dump_h and re.search(r"inline bool dump_on\(\) \{\s*return dump_armed;\s*\}", dump_h), + "vk_dump.h's dump_on is not inline over dump_armed") + need("bool dump_on()" not in dump and "\nbool dump_armed = false;\n" in dump and not re.search(r"(?(*cp);", at)] + # ((DESC-30) and in the lab's "diagold" phases, for what it cost) + need("if ((want && !done) || dispatch_diag_old()) {" in push and push.index("if ((want && !done) || dispatch_diag_old()) {") < push.index("snprintf(") and + "if (want && !done && !strncmp(hex, want, strlen(want))) {" in push and 'logf("%s", line);' in push, + "dispatch formats RX_VK_LOG_PUSH's key before it knows the switch is set and its line not written") + # what the entry also lists and this port left alone + memory = (ROOT / "gpu/vk/vk_memory.cpp").read_text(encoding="utf-8") + need("using SrwMutex = std::mutex;" not in memory and "++unmirrored & 63" not in memory, + "the lock type or the read-back skip changed: this test and its docstring say they did not") + return memory + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + memory = check_source() + now = block(memory, "void retired_imports_give_back() {") + for line in ("const bool due = mirror_epoch - logged_epoch >= 1200;", "if (due)\n for (auto& [lo, p] : imported) live_bytes += p.hi - p.lo;", + "static size_t gone = 0;", "if (gone > retired_pools.size()) gone = 0;", "if (give_back || due)", + "for (size_t i = gone; i < retired_pools.size(); i++) {", "if (i == gone) gone++;", "if (due) {"): + need(line in now, "retired_imports_give_back lost: " + line) + need("retired_pools.push_back(imported.extract(it));" in memory and + not re.search(r"retired_pools\.(erase|pop_front|pop_back|clear|insert|resize|emplace_front|push_front)\(", memory), + "the retired pools' list no longer only grows at its end: the walk's start past the given back ones needs that") + env = {k: v for k, v in os.environ.items() if not k.startswith("RX_VK_")} + with tempfile.TemporaryDirectory(prefix="replay-hot-trims-") as tmp: + exes = {} + for name, fn in (("before", BEFORE), ("now", now)): + cpp, exe = Path(tmp) / (name + ".cpp"), Path(tmp) / (name + ".exe") + rg = (ROOT / "gpu" / "vk" / "retired_give_back.h").read_text(encoding="utf-8").replace("#pragma once", "") + cpp.write_text(FIXTURE.replace("/* RG_HEADER */", rg).replace("/* FUNCTION */", fn), encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", str(cpp), "-o", str(exe)], check=True) + exes[name] = exe + for keep in ("0", "1", None): + out = {} + for name, exe in exes.items(): + r = subprocess.run([str(exe)], capture_output=True, text=True, check=True, timeout=120, + env=env if keep is None else {**env, "RX_VK_KEEP_RETIRED": keep}) + rows = r.stdout.strip().splitlines() + out[name] = ("\n".join(rows[:-1]), int(rows[-1].split()[1])) + what = "RX_VK_KEEP_RETIRED " + ("unset (NVIDIA: given back)" if keep is None else keep) + need(out["before"][0] == out["now"][0], "%s: the retired imports are not given back, or the line not logged, as before:\n%s\n-- now:\n%s" + % (what, out["before"][0], out["now"][0])) + need("vk: imports: 40 live" in out["now"][0], what + ": the imports line is gone") + dead = int(re.search(r" dead (\d+) ", out["now"][0]).group(1)) + need((dead == 0) == (keep == "1") and (keep == "1" or dead > 1000), what + ": %d imports given back in the session" % dead) + need(out["now"][1] * 4 < out["before"][1], "%s: the walk looks at %d retired pools where it looked at %d: not the cut" + % (what, out["now"][1], out["before"][1])) + print("%s: same trace; %d looks at retired pools where there were %d" % (what, out["now"][1], out["before"][1])) + print("PASS: the replay thread's small costs are cut and the results are the same") + + +if __name__ == "__main__": + main() diff --git a/tests/test_replay_pin_soft.py b/tests/test_replay_pin_soft.py new file mode 100644 index 0000000..5c440d2 --- /dev/null +++ b/tests/test_replay_pin_soft.py @@ -0,0 +1,454 @@ +"""A replay's pin is soft where the cores are of more than one kind (gpu/vk/replay_pin.h, vk_replay.cpp ReplayPin; no game, no GPU, +no processor of the kind). + +(Triforce, 11-22, the performance audit's THREAD-09, after Source A's.) A thread that replays is pinned to the core the host keeps +for the replay, and that core is kept from the game's threads only: another program's thread on it holds the pinned replay up. On +a processor of two kinds of core the pin lets the thread also run on the fastest-kind cores among its own processors, with the +reserved core as its ideal processor, and leaves out the core it is on so that it moves at once. + +Three parts: + +- the rule, replay_pin.h, compiled and run over made-up processors, with the reserved core chosen by the host's own + pick_reserved_core cut out of host/hos_thread.c: a 6+8-core hybrid (reserved 0xc00, from core 0: 0xffc), an 8+4 one, a 12-core + and an 8-core 16-thread processor of one kind (this PC's kind: strict, the reserved core alone, exactly as before), a two-die + X3D (soft only with the switch at 2, and then inside its die), a laptop's 2+8 (no core reserved: no pin), a thread held to the + small cores, a thread on the reserved core already; the switch's values; +- vk_replay.cpp's ReplayPin, cut out and run over a stand-in thread that behaves as Windows' does (a thread whose processor is + no longer allowed moves, to its ideal processor when that is allowed and free; an ideal processor outside a new affinity is + replaced): the widening, the ideal processor, the move, pins inside a pin, and everything given back in the order that works. + Each step taken out or reordered (a mutation of the cut code) must make a check fail; +- on Windows, the code that reads the processor (replay_pin_cpu and the four small calls) compiled against Windows itself and + run on this PC: what it reads agrees with itself, whatever the processor. + + python -I -B tests/test_replay_pin_soft.py [--cxx clang++] +""" +import argparse +import subprocess +import sys +import tempfile +from pathlib import Path + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +RULE_TEST = r''' +#include +#include +#include "replay_pin.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +using namespace mkwvk::replay_pin; +/* PICK */ +static uint64_t masks[256]; +static unsigned char classes[256]; +static int n; +static void fresh() { n = 0; } +// count cores of `per` logical processors each from logical processor `first`, of one efficiency class +static void add(int count, int per, int first, unsigned char cls) { + for (int i = 0; i < count; i++) masks[n] = ((1ull << per) - 1) << (first + i * per), classes[n++] = cls; +} +// what a pin sets for a thread with affinity `own` on logical processor `lp`: the mask it ends with +static uint64_t pinned(int mode_, uint64_t reserved, uint64_t own, int lp) { + const Kinds k = kinds(masks, classes, n); + if (!reserved) return 0; // (no core reserved: no pin at all) + if (!soft(mode_, k.hybrid)) return reserved; + uint64_t core = 0; + for (int i = 0; i < n; i++) if (masks[i] >> lp & 1) core = masks[i]; + return moved(wide(reserved, k.fastest, own), reserved, core); +} +int main() { + // the switch + CHECK(mode(nullptr) == kHybrid && mode("") == kHybrid && mode("1") == kHybrid); + CHECK(mode("0") == kStrict && mode("-1") == kStrict && mode("off") == kStrict); + CHECK(mode("2") == kAlways && mode("3") == kAlways); + CHECK(soft(kHybrid, true) && !soft(kHybrid, false) && !soft(kStrict, true) && !soft(kStrict, false)); + CHECK(soft(kAlways, false) && soft(kAlways, true)); + + // a 6+8-core hybrid: 6 performance cores of 2 logical processors (0-11), 8 efficiency cores (12-19) + fresh(), add(6, 2, 0, 1), add(8, 1, 12, 0); + { + const Kinds k = kinds(masks, classes, n); + const uint64_t reserved = pick_reserved_core(masks, classes, n), all = 0xfffff, own = all & ~reserved; + CHECK(k.hybrid && k.fastest == 0xfff && reserved == 0xc00); + CHECK((reserved & k.fastest) == reserved); // (the reserved core is one of the fastest kind) + CHECK(wide(reserved, k.fastest, own) == 0xfff); + CHECK(pinned(kHybrid, reserved, own, 0) == 0xffc); // (from core 0: every performance core but that one) + CHECK(pinned(kHybrid, reserved, own, 1) == 0xffc); // (core 0's other logical processor: the same core) + CHECK(pinned(kHybrid, reserved, own, 5) == 0xfcf); + CHECK(pinned(kHybrid, reserved, own, 14) == 0xfff); // (from an efficiency core: it is outside already, nothing to leave out) + CHECK(pinned(kHybrid, reserved, all, 10) == 0xfff); // (on the reserved core already: it stays, nothing left out) + CHECK(pinned(kStrict, reserved, own, 0) == 0xc00); // (=0: the reserved core alone, as before) + CHECK(pinned(kAlways, reserved, own, 0) == 0xffc); + CHECK(ideal(reserved) == 11); // (the reserved core's last logical processor) + // a thread held to the efficiency cores (HOS_PIN): nothing of its own is of the fastest kind, so the reserved core alone + CHECK(pinned(kHybrid, reserved, 0xff000, 13) == 0xc00); + // a thread held to two performance cores: those and the reserved one, never more than it had beside it + CHECK(pinned(kHybrid, reserved, 0x0f, 0) == 0xc0c); + } + // an 8+4-core hybrid: 8 performance cores (0-15), 4 efficiency cores (16-19) + fresh(), add(8, 2, 0, 1), add(4, 1, 16, 0); + { + const Kinds k = kinds(masks, classes, n); + const uint64_t reserved = pick_reserved_core(masks, classes, n); + CHECK(k.hybrid && k.fastest == 0xffff && reserved == 0xc000 && ideal(reserved) == 15); + CHECK(pinned(kHybrid, reserved, 0xfffff & ~reserved, 6) == 0xff3f); + } + // one kind of core, 12 cores with SMT: the strict pin, whatever the thread is on + fresh(), add(12, 2, 0, 0); + { + const Kinds k = kinds(masks, classes, n); + const uint64_t reserved = pick_reserved_core(masks, classes, n); + CHECK(!k.hybrid && k.fastest == 0xffffff && reserved == 0xc00000); + for (int lp = 0; lp < 24; lp++) CHECK(pinned(kHybrid, reserved, 0xffffff & ~reserved, lp) == 0xc00000); + CHECK(pinned(kStrict, reserved, 0x3fffff, 0) == 0xc00000); + CHECK(pinned(kAlways, reserved, 0x3fffff, 0) == 0xfffffc); // (the lab's 2: soft there too) + } + // this PC's kind: 8 cores, 16 logical processors, one kind, one cache domain: nothing may change + fresh(), add(8, 2, 0, 0); + { + const Kinds k = kinds(masks, classes, n); + const uint64_t reserved = pick_reserved_core(masks, classes, n); + CHECK(!k.hybrid && k.fastest == 0xffff && reserved == 0xc000); + for (int lp = 0; lp < 16; lp++) { + CHECK(pinned(kHybrid, reserved, 0x3fff, lp) == 0xc000); // (unset and 1) + CHECK(pinned(kStrict, reserved, 0x3fff, lp) == 0xc000); + } + CHECK(!soft(mode(nullptr), k.hybrid) && !soft(mode("1"), k.hybrid) && !soft(mode("0"), k.hybrid)); + } + // the same core class reported as a non-zero number (some processors of one kind do): still one kind + fresh(), add(8, 2, 0, 3); + CHECK(!kinds(masks, classes, n).hybrid && kinds(masks, classes, n).fastest == 0xffff); + // a two-die X3D, 16 cores with SMT: one kind, so strict; with the switch at 2 a game thread, which the host holds to the + // cache die (logical processors 0-15) without the reserved core, stays on that die + fresh(), add(16, 2, 0, 0); + { + const uint64_t die = 0xffff, reserved = 0xc000, own = die & ~reserved; + CHECK(!kinds(masks, classes, n).hybrid); + CHECK(pinned(kHybrid, reserved, own, 2) == reserved); + CHECK(pinned(kAlways, reserved, own, 2) == 0xfff3); + CHECK((pinned(kAlways, reserved, own, 2) & ~die) == 0); + } + // a laptop's 2 performance and 8 efficiency cores: the host keeps no core (fewer than 4 of the fastest kind), so no pin + fresh(), add(2, 2, 0, 1), add(8, 1, 4, 0); + { + const uint64_t reserved = pick_reserved_core(masks, classes, n); + CHECK(kinds(masks, classes, n).hybrid && reserved == 0 && pinned(kHybrid, reserved, 0xfff, 0) == 0 && ideal(reserved) == -1); + } + // three kinds of core: only the highest class is the fastest; a core with no processors is not one + fresh(), add(2, 1, 0, 2), add(4, 1, 2, 1), add(4, 1, 6, 0); + masks[n] = 0, classes[n++] = 9; + CHECK(kinds(masks, classes, n).hybrid && kinds(masks, classes, n).fastest == 0x3); + // nothing listed (the cores could not be read): no fastest kind, not hybrid, and a pin stays strict + fresh(); + CHECK(!kinds(masks, classes, n).hybrid && kinds(masks, classes, n).fastest == 0); + // moved: the processor not known, or nothing else left: whole + CHECK(moved(0xfff, 0xc00, 0) == 0xfff && moved(0x3, 0, 0x3) == 0x3 && moved(0xc03, 0xc00, 0x3) == 0xc00); + if (!fails) std::printf("PASS: the soft pin's rule over made-up processors\n"); + return fails ? 1 : 0; +} +''' + +PIN_TEST = r''' +#include +#include +#include "replay_prio.h" +#include "replay_pin.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +typedef uintptr_t DWORD_PTR; +typedef void* HANDLE; +namespace mkwvk { +// One thread as Windows keeps it: its affinity, its ideal processor, the logical processor it is on; and the processors another +// program holds (busy). A thread whose processor is no longer allowed moves at once: to its ideal processor when that is allowed +// and free, else to the lowest free allowed one. An ideal processor outside a new affinity is replaced by one inside it. +uint64_t affinity, busy, reserved_mask, fast_mask, core_of[64]; +int ideal, cur, aff_sets, ideal_sets, ideal_reads, prio; +bool refuse_aff, refuse_ideal, async; +void place() { + if (affinity >> cur & 1) return; + if ((affinity >> ideal & 1) && !(busy >> ideal & 1)) { cur = ideal; return; } + for (int i = 0; i < 64; i++) if ((affinity >> i & 1) && !(busy >> i & 1)) { cur = i; return; } + cur = __builtin_ctzll(affinity); +} +HANDLE GetCurrentThread() { return (HANDLE)1; } +DWORD_PTR SetThreadAffinityMask(HANDLE, DWORD_PTR m) { + aff_sets++; + if (refuse_aff || !m) return 0; + const DWORD_PTR was = affinity; + affinity = m; + if (!(affinity >> ideal & 1)) ideal = __builtin_ctzll(affinity); + place(); + return was; +} +int GetThreadPriority(HANDLE) { return prio; } +int SetThreadPriority(HANDLE, int p) { prio = p; return 1; } +bool async_on() { return async; } +uint64_t replay_pin_mask() { return reserved_mask; } +int replay_prio_level() { return 2; } +uint64_t replay_pin_soft(bool = true) { return fast_mask; } +uint64_t replay_pin_own() { return affinity; } +uint64_t replay_pin_core_now() { return core_of[cur]; } +int replay_pin_ideal_now() { ideal_reads++; return ideal; } +bool replay_pin_set_ideal(int lp) { + ideal_sets++; + if (refuse_ideal || lp < 0 || !(affinity >> lp & 1)) return false; // (Windows takes an ideal processor only among the thread's own) + ideal = lp; + return true; +} +/* PIN */ +} // namespace mkwvk +using namespace mkwvk; +// the 6+8-core hybrid: a game thread on every processor but the reserved core's, its ideal processor an efficiency core +void hybrid(int on, int ideal_was) { + for (int lp = 0; lp < 12; lp++) core_of[lp] = 3ull << (lp & ~1); + for (int lp = 12; lp < 20; lp++) core_of[lp] = 1ull << lp; + reserved_mask = 0xc00, fast_mask = 0xfff, affinity = 0xff3ff, busy = 0, ideal = ideal_was, cur = on; + aff_sets = ideal_sets = ideal_reads = 0, prio = 0, refuse_aff = refuse_ideal = async = false; +} +int main() { + hybrid(0, 13); // (on core 0, the reserved core free: widened, the ideal processor set, and moved there at once) + { + ReplayPin outer; + CHECK(affinity == 0xffc && ideal == 11 && cur == 11 && aff_sets == 2 && ideal_sets == 1 && prio == 2); + { + ReplayPin inner; // (a pin inside a pin sets nothing) + CHECK(affinity == 0xffc && ideal == 11 && cur == 11 && aff_sets == 2 && ideal_sets == 1 && ideal_reads == 1); + } + CHECK(affinity == 0xffc && ideal == 11 && aff_sets == 2 && ideal_sets == 1); // (still replaying) + } + CHECK(affinity == 0xff3ff && ideal == 13 && prio == 0 && aff_sets == 3 && ideal_sets == 2); // (all of it back) + CHECK(affinity >> cur & 1); // (and it is on one of its own processors again) + + hybrid(0, 13), busy = 0xc00; // (another program holds the reserved core: the replay runs on another fast core, not core 0) + { + ReplayPin p; + CHECK(affinity == 0xffc && ideal == 11 && cur == 2); + CHECK((1ull << cur & fast_mask) && !(1ull << cur & reserved_mask)); + } + CHECK(affinity == 0xff3ff && ideal == 13); + + hybrid(14, 13); // (on an efficiency core: the widening itself moves it to a fast core, and the pin on to the reserved one) + { + ReplayPin p; + CHECK((affinity & reserved_mask) == reserved_mask && !(affinity & ~fast_mask) && cur == 11 && ideal == 11); + } + CHECK(affinity == 0xff3ff && ideal == 13); + + hybrid(0, 4), affinity = 0xfffff, cur = 10; // (a thread already on the reserved core: it stays) + { + ReplayPin p; + CHECK(affinity == 0xfff && cur == 10 && ideal == 11 && aff_sets == 1); + } + CHECK(affinity == 0xfffff && ideal == 4); + + hybrid(0, 13), fast_mask = 0; // (strict: a processor of one kind, or the switch at 0: the reserved core alone, one call each way) + { + ReplayPin p; + CHECK(affinity == 0xc00 && aff_sets == 1 && ideal_sets == 0 && ideal_reads == 0); + CHECK(cur == 10 || cur == 11); + } + CHECK(affinity == 0xff3ff && aff_sets == 2 && ideal_sets == 0); + + hybrid(0, 13), reserved_mask = 0; // (no core reserved: nothing) + { ReplayPin p; CHECK(prio == 2); } + CHECK(affinity == 0xff3ff && aff_sets == 0 && ideal_sets == 0 && ideal_reads == 0 && cur == 0); + + hybrid(0, 13), async = true; // (the replay on its own thread: the pin does nothing) + { ReplayPin p; } + CHECK(affinity == 0xff3ff && aff_sets == 0 && ideal_sets == 0 && cur == 0); + + hybrid(0, 13), refuse_aff = true; // (Windows refused the widening: nothing else is set, nothing given back) + { ReplayPin p; CHECK(aff_sets == 1 && ideal_sets == 0); } + CHECK(affinity == 0xff3ff && ideal == 13 && aff_sets == 1 && ideal_sets == 0); + + hybrid(0, 13), refuse_ideal = true; // (Windows refused the ideal processor: the thread still moves; no ideal given back) + { + ReplayPin p; + CHECK(affinity == 0xffc && cur != 0 && ideal_sets == 1); + } + CHECK(affinity == 0xff3ff && ideal_sets == 1); + + hybrid(0, 13); + for (int i = 0; i < 3; i++) { ReplayPin p; CHECK(cur == 11); } // (pin after pin) + CHECK(affinity == 0xff3ff && ideal == 13 && aff_sets == 9 && ideal_sets == 6); + if (!fails) std::printf("PASS: ReplayPin widens, prefers the reserved core, moves the thread and gives everything back\n"); + return fails ? 1 : 0; +} +''' + +# (each: what is taken out or reordered, the text in the cut ReplayPin, what replaces it) +RESTORE_AFF = " if (replay_pin_prev) SetThreadAffinityMask(GetCurrentThread(), replay_pin_prev), replay_pin_prev = 0;\n" +RESTORE_IDEAL = " if (replay_pin_ideal_prev != kIdealNone) replay_pin_set_ideal(replay_pin_ideal_prev), replay_pin_ideal_prev = kIdealNone;\n" +WIDEN = " if ((replay_pin_prev = SetThreadAffinityMask(GetCurrentThread(), (DWORD_PTR)wide))) {\n" +READ = " const int ideal = replay_pin_ideal_now();\n" +MUTATIONS = [ + ("the move off the core it is on", "if (moved != wide) SetThreadAffinityMask(GetCurrentThread(), (DWORD_PTR)moved);", ""), + ("the reserved core as the ideal processor", + "if (ideal >= 0 && replay_pin_set_ideal(replay_pin::ideal(m))) replay_pin_ideal_prev = ideal;", ""), + ("the old ideal processor read before the widening", READ + WIDEN, WIDEN + READ), + ("the ideal processor given back", RESTORE_IDEAL, ""), + ("the affinity given back before the ideal processor", None, None), + ("the strict pin where the rule says so", "const uint64_t fast = replay_pin_soft(), own = fast ? replay_pin_own() : 0;", + "const uint64_t fast = 0xfff, own = replay_pin_own();"), + ("only the outermost pin", "if (replay_pin_depth++) return;", "replay_pin_depth++;"), + ("nothing more after a refused widening", WIDEN, + " replay_pin_prev = SetThreadAffinityMask(GetCurrentThread(), (DWORD_PTR)wide); {\n"), + ("no ideal processor given back for a refused one", + "if (ideal >= 0 && replay_pin_set_ideal(replay_pin::ideal(m))) replay_pin_ideal_prev = ideal;", + "if (ideal >= 0) replay_pin_set_ideal(replay_pin::ideal(m)), replay_pin_ideal_prev = ideal;"), +] + +CPU_TEST = r''' +#include +#include +#include +#include +#include "replay_pin.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +namespace mkwvk { +/* CPU */ +} // namespace mkwvk +using namespace mkwvk; +int main() { + const ReplayPinCpu& c = replay_pin_cpu(); + DWORD_PTR proc = 0, sys = 0; + GetProcessAffinityMask(GetCurrentProcess(), &proc, &sys); + CHECK(c.kinds.fastest != 0); // (the cores were read) + CHECK((c.kinds.fastest & ~(uint64_t)sys) == 0); // (and are this PC's) + uint64_t all = 0; + for (int lp = 0; lp < 64; lp++) { + if (!c.core_of[lp]) continue; + all |= 1ull << lp; + CHECK(c.core_of[lp] >> lp & 1); // (a logical processor is on its own core) + for (int o = 0; o < 64; o++) + if (c.core_of[lp] >> o & 1) CHECK(c.core_of[o] == c.core_of[lp]); // (and its siblings name the same core) + } + CHECK(all == (uint64_t)sys); + CHECK(c.kinds.hybrid == (c.kinds.fastest != all)); + const uint64_t own = replay_pin_own(); + CHECK(own == (uint64_t)proc); // (a thread nobody pinned: the process's processors) + const uint64_t now = replay_pin_core_now(); + CHECK(now && (now & all) == now); + const int ideal = replay_pin_ideal_now(); + CHECK(ideal >= 0 && ideal < 64 && (own >> ideal & 1)); + CHECK(replay_pin_set_ideal(ideal) && replay_pin_ideal_now() == ideal); // (set to what it is: nothing changes) + CHECK(!replay_pin_set_ideal(-1)); + // the whole of a soft pin on this thread, by the real calls, and back: the lab's 2 on any processor + const uint64_t reserved = c.core_of[63 - __builtin_clzll(own)] & own; // (this thread's last core, standing in for the reserved one) + const uint64_t wide = replay_pin::wide(reserved, c.kinds.fastest, own); + const DWORD_PTR was = SetThreadAffinityMask(GetCurrentThread(), (DWORD_PTR)wide); + CHECK(was == proc); + CHECK(replay_pin_set_ideal(replay_pin::ideal(reserved))); + const uint64_t moved = replay_pin::moved(wide, reserved, replay_pin_core_now()); + CHECK(SetThreadAffinityMask(GetCurrentThread(), (DWORD_PTR)moved) == (DWORD_PTR)wide); + CHECK(replay_pin_ideal_now() == replay_pin::ideal(reserved)); + CHECK(moved >> GetCurrentProcessorNumber() & 1); // (it runs inside the pin's processors at once) + CHECK(SetThreadAffinityMask(GetCurrentThread(), was) == (DWORD_PTR)moved); + CHECK(replay_pin_set_ideal(ideal) && replay_pin_ideal_now() == ideal && replay_pin_own() == (uint64_t)proc); + std::printf("this PC: fastest-kind processors %#llx of %#llx, %s\n", (unsigned long long)c.kinds.fastest, (unsigned long long)all, + c.kinds.hybrid ? "cores of more than one kind (a soft pin by default)" : "one kind of core (the strict pin, as before)"); + if (!fails) std::printf("PASS: the processor as Windows lists it here, and a soft pin's calls on a real thread\n"); + return fails ? 1 : 0; +} +''' + + +def cut_pin(replay): + """ReplayPin with its thread-locals, from the first of them to the struct's end""" + start = replay.index("thread_local int replay_pin_depth = 0;") + struct = block(replay, "struct ReplayPin {") + return replay[start:replay.index(struct) + len(struct)] + ";\n" + + +def check_source(replay, dev): + if '#include "replay_pin.h"' not in replay: + raise SystemExit("vk_replay.cpp does not include replay_pin.h") + soft = block(replay, "uint64_t replay_pin_soft(bool lab = true) {") + for need, why in (('replay_pin::mode(getenv("RX_VK_REPLAY_PIN_SOFT"))', "the switch is not the header's rule over RX_VK_REPLAY_PIN_SOFT"), + ("static const int mode", "the switch is not read once"), + ("replay_pin::soft(mode, c.kinds.hybrid) || (lab && lab_ab::on(\"pinsoft\"))", "soft is not the rule's, or the lab's knob is gone"), + ('logf("vk: a replay\'s pin is %s', "the choice leaves no line in the log")): + if need not in soft: + raise SystemExit("vk_replay.cpp replay_pin_soft: " + why) + cpu = block(replay, "const ReplayPinCpu& replay_pin_cpu() {") + for need, why in (("e->Processor.GroupMask[0].Group == group", "cores of another processor group are taken"), + ("classes[n] = e->Processor.EfficiencyClass;", "the cores' kinds are not read"), + ("c.kinds = replay_pin::kinds(masks, classes, n);", "the fastest kind is not the header's"), + ("static const ReplayPinCpu cpu", "the processor is read more than once")): + if need not in cpu: + raise SystemExit("vk_replay.cpp replay_pin_cpu: " + why) + pin = cut_pin(replay) + for need in ("replay_pin::wide(m, fast, own)", "replay_pin::moved(wide, m, replay_pin_core_now())", "replay_pin::ideal(m)"): + if need not in pin: + raise SystemExit("vk_replay.cpp ReplayPin: %r is missing: the soft pin is not the header's" % need) + thread = block(replay, "void replay_thread_main() {") + if "replay_pin_soft(false)" not in thread or "replay_pin::wide(m, fast, own)" not in thread or \ + "if (on != m) replay_pin_set_ideal(replay_pin::ideal(m));" not in thread: + raise SystemExit("vk_replay.cpp replay_thread_main: the replay thread of RX_VK_ASYNC=1 does not take the soft pin's processors") + if "| `RX_VK_REPLAY_PIN_SOFT=0`, `=2` |" not in dev: + raise SystemExit("docs/development.md: RX_VK_REPLAY_PIN_SOFT is not in the switches table") + + +def build_run(cxx, d, name, src, expect_ok=True, extra=()): + path = d / (name + ".cpp") + path.write_text(src, encoding="utf-8") + exe = d / (name + ".exe") + r = subprocess.run([cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-Wno-unused-function", "-Wno-unused-variable", + "-Wno-unused-but-set-variable", "-I" + str(VK), str(path), "-o", str(exe), *extra], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + if r.returncode: + raise SystemExit("%s: the build failed:\n%s" % (name, r.stdout)) + r = subprocess.run([str(exe)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=60) + if expect_ok and r.returncode: + raise SystemExit("%s failed:\n%s" % (name, r.stdout)) + return r + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--cxx", default="clang++") + args = ap.parse_args() + replay = (VK / "vk_replay.cpp").read_text(encoding="utf-8") + host = (ROOT / "host" / "hos_thread.c").read_text(encoding="utf-8") + check_source(replay, (ROOT / "docs" / "development.md").read_text(encoding="utf-8")) + print("PASS sources: the switch read once through the rule, the pin and the replay thread take the rule's processors") + pin = cut_pin(replay) + with tempfile.TemporaryDirectory(prefix="replay-pin-soft-") as tmp: + d = Path(tmp) + pick = block(host, "static uint64_t pick_reserved_core(") + print(build_run(args.cxx, d, "rule", RULE_TEST.replace("/* PICK */", pick)).stdout.strip()) + print(build_run(args.cxx, d, "pin", PIN_TEST.replace("/* PIN */", pin)).stdout.strip()) + for i, (what, old, new) in enumerate(MUTATIONS): + if old is None: # (the two lines that give things back, the other way round) + old, new = RESTORE_AFF + " // ((Triforce, THREAD-09) after the affinity, which holds the old ideal processor again)\n" + RESTORE_IDEAL, \ + RESTORE_IDEAL + RESTORE_AFF + if old not in pin: + raise SystemExit("mutation %r: %r is not in the cut ReplayPin (update the test with the code)" % (what, old)) + r = build_run(args.cxx, d, "mutant%d" % i, PIN_TEST.replace("/* PIN */", pin.replace(old, new, 1)), expect_ok=False) + if r.returncode == 0: + raise SystemExit("mutation %r: the checks still passed without it" % what) + print("PASS mutation: without %s a check fails (%s)" % (what, r.stdout.strip().splitlines()[0])) + if sys.platform == "win32": + cpu = replay[replay.index("struct ReplayPinCpu {"):replay.index("// ---- (the soft pin's processors end) ----")] + print(build_run(args.cxx, d, "cpu", CPU_TEST.replace("/* CPU */", cpu)).stdout.strip()) + else: + print("SKIP: the processor's reading is Windows' (not this system)") + print("ALL PASS") + + +if __name__ == "__main__": + main() diff --git a/tests/test_replay_prio.py b/tests/test_replay_prio.py index a7afbdf..ceb161f 100644 --- a/tests/test_replay_prio.py +++ b/tests/test_replay_prio.py @@ -97,6 +97,7 @@ PIN_TEST = r''' #include #include #include "replay_prio.h" +#include "replay_pin.h" #define CHECK(x) do { if (!(x)) { std::fprintf(stderr, "FAIL:%d %s\n", __LINE__, #x); std::exit(1); } } while (0) typedef uintptr_t DWORD_PTR; typedef void* HANDLE; @@ -115,6 +116,15 @@ DWORD_PTR SetThreadAffinityMask(HANDLE, DWORD_PTR m) { aff_sets++; DWORD_PTR was bool async_on() { return async; } uint64_t replay_pin_mask() { return mask; } int replay_prio_level() { return level_now; } +// ((Triforce, THREAD-09) the soft pin's calls, replay_pin.h: soft_fast 0 is the strict pin, as on a processor of one kind of core; +// tests/test_replay_pin_soft.py has the soft pin's own cases) +uint64_t soft_fast = 0, core_now = 0; +int ideal = 3, ideal_sets = 0; +uint64_t replay_pin_soft(bool = true) { return soft_fast; } +uint64_t replay_pin_own() { return affinity; } +uint64_t replay_pin_core_now() { return core_now; } +int replay_pin_ideal_now() { return ideal; } +bool replay_pin_set_ideal(int lp) { ideal_sets++; ideal = lp; return true; } /* PIN */ } // namespace mkwvk using namespace mkwvk; @@ -159,6 +169,17 @@ int main() { { ReplayPin q; CHECK(aff_sets == 1); } } CHECK(affinity == 0xffff && prio == 0 && aff_sets == 2); + CHECK(ideal_sets == 0 && ideal == 3); // (the strict pin sets no ideal processor) + // ((Triforce, THREAD-09) the soft pin beside the priority: widened, the reserved core's last processor as the ideal one, the + // core it is on left out; a pin inside it sets nothing; the priority, the affinity and the ideal processor all come back) + reset(0, 2), mask = 0xc00, soft_fast = 0xfff, affinity = 0xff3ff, core_now = 0x3; + { + ReplayPin p; + CHECK(affinity == 0xffc && ideal == 11 && prio == 2 && aff_sets == 2 && ideal_sets == 1); + { ReplayPin q; CHECK(aff_sets == 2 && ideal_sets == 1 && sets == 1); } + } + CHECK(affinity == 0xff3ff && ideal == 3 && prio == 0 && aff_sets == 3 && ideal_sets == 2 && sets == 2); + soft_fast = 0, affinity = 0xffff; reset(0, 2), mask = 0; for (int i = 0; i < 3; i++) { ReplayPin p; } // (pin after pin: raised and given back each time) CHECK(prio == 0 && sets == 6); diff --git a/tests/test_replay_threads_row.py b/tests/test_replay_threads_row.py index d2fd75f..beab18f 100644 --- a/tests/test_replay_threads_row.py +++ b/tests/test_replay_threads_row.py @@ -143,7 +143,7 @@ def main(): automatic = top.index("if (replay_threads_at_start == replay_threads::kRowAutomatic) return replay_threads_automatic();") check(order[1] < automatic < order[2], "par_threads_max: Automatic is not asked after the switches, before the fixed counts") auto = function(replay, "int replay_threads_automatic() {") - check("replay_threads::automatic(logical, amd_older)" in auto and "std::thread::hardware_concurrency()" in auto and + check("replay_threads::automatic(logical, amd_older)" in auto and "cpu_count()" in auto and "amd_older_compiler(facts)" in auto and "facts.driver_info = drv.driverInfo;" in auto, "replay_threads_automatic no longer asks the rule with the processors and the driver") check('logf("vk: replay helper threads: Automatic gives this PC %d (%u logical processors: %s). %s"' in auto and diff --git a/tests/test_replay_threads_tuner.py b/tests/test_replay_threads_tuner.py index 5294836..789b2c9 100644 --- a/tests/test_replay_threads_tuner.py +++ b/tests/test_replay_threads_tuner.py @@ -16,10 +16,13 @@ par_tune_present at each present). No game, no GPU. every 8,000 presents fewer trials, the better way in use as often and as soon after a change; a short trial never changes the way and one that finds the other way shorter is followed at once by a full one; a change the scene's means cannot see found by a backed-off trial; loads are no scene change; the rule's no-trial verdicts hold through scene changes. + "settled": (11-22) the rule gives an area's load window, and the first after it, no verdict (with settled off: on at the + load, as Kaefergeneral's log had it); a settled heavy scene like test4's still turns the helpers on without a trial, a + window after it settled; a trial the scene changed under changes nothing and the next comes once the scene settles. 2. The source read: par_tune_present returns at once unless the tuner decides (the Debug row's Off, RX_VK_PARALLEL), hands each frame to Tuner::step with the way in use, and applies its change; the rule is RX_VK_PARALLEL_RULE's to turn off, and counts the replay thread among the threads that record segments only where it does (RX_VK_PARALLEL_OWN); the back-off is - RX_VK_PARALLEL_BACKOFF's to turn off. + RX_VK_PARALLEL_BACKOFF's to turn off, settled RX_VK_TUNER_SETTLED's. python -I -B tests/test_replay_threads_tuner.py [--cxx g++] """ @@ -61,7 +64,7 @@ def main(): replay = (VK / "vk_replay.cpp").read_text(encoding="utf-8") tune = function(replay, "void par_tune_present() {") in_order(tune, ["if (!par_auto()) return;", "const replay_threads::Frame fr = par_frame_now(ms);", - "t.use_rule = par_rule_on();", "t.backoff = par_backoff_on();", + "t.use_rule = par_rule_on();", "t.backoff = par_backoff_on();", "t.settled = par_settled_on();", "t.threads = par_threads_max() + (par_own_on() ? 1 : 0);", "tuner.step(fr, par_auto_on, par_stream_order_only());", "if (s.flip) par_auto_on = !par_auto_on;"], "par_tune_present") @@ -72,6 +75,9 @@ def main(): backoff = function(replay, "bool par_backoff_on() {") check('getenv("RX_VK_PARALLEL_BACKOFF")' in backoff and 'atoi(getenv("RX_VK_PARALLEL_BACKOFF")) != 0' in backoff, "par_backoff_on no longer reads RX_VK_PARALLEL_BACKOFF (on unless 0)") + settled = function(replay, "bool par_settled_on() {") + check('getenv("RX_VK_TUNER_SETTLED")' in settled and 'atoi(getenv("RX_VK_TUNER_SETTLED")) != 0' in settled, + "par_settled_on no longer reads RX_VK_TUNER_SETTLED (on unless 0)") own = function(replay, "bool par_own_on() {") check('getenv("RX_VK_PARALLEL_OWN")' in own, "par_own_on no longer reads RX_VK_PARALLEL_OWN") check("bool par_auto() {\n static const bool v = !getenv(\"RX_VK_PARALLEL\") && par_threads_max() >= 2;" in replay, @@ -79,19 +85,21 @@ def main(): guide = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") check("`RX_VK_PARALLEL_RULE=0`" in guide, "docs/development.md does not say what RX_VK_PARALLEL_RULE=0 does") check("`RX_VK_PARALLEL_BACKOFF=0`" in guide, "docs/development.md does not say what RX_VK_PARALLEL_BACKOFF=0 does") + check("`RX_VK_TUNER_SETTLED=0`" in guide, "docs/development.md does not say what RX_VK_TUNER_SETTLED=0 does") with tempfile.TemporaryDirectory(prefix="replay-tuner-") as tmp: exe = Path(tmp) / "tuner.exe" built = subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I", str(VK), str(ROOT / "tests" / "replay_threads_tuner_test.cpp"), "-o", str(exe)], capture_output=True, text=True) check(built.returncode == 0, "the fixture did not build:\n" + built.stderr[-4000:]) env = {k: v for k, v in os.environ.items() if not k.startswith("RX_VK_")} - for case in ("same", "prev", "rule", "backoff"): + for case in ("same", "prev", "rule", "backoff", "settled"): r = subprocess.run([str(exe), case], env=env, capture_output=True, text=True, timeout=300) print((r.stdout + r.stderr).strip()) check(r.returncode == 0, "replay_threads_tuner_test %s" % case) print("PASS: the helper threads' tuner decides as it did inline with the rule off; with it, on without a trial where the " "replay is plainly the frame's limit, no trial where the GPU or the game's threads are, trials in between; the " - "back-off makes them rarer and shorter once the way holds, early where the scene changes, off as before") + "back-off makes them rarer and shorter once the way holds, early where the scene changes, off as before; both judge " + "settled scenes only") if __name__ == "__main__": diff --git a/tests/test_report_words.py b/tests/test_report_words.py index 7dcd89a..f285bfa 100644 --- a/tests/test_report_words.py +++ b/tests/test_report_words.py @@ -41,6 +41,7 @@ def main(): struct Pool { uint64_t lo; }; struct VkBufferCopy { uint64_t srcOffset, dstOffset, size; }; std::mutex cpu_words_mu; +bool wb_notes_off() { return false; } // ((Triforce, 11-22, MEM-51) the mirrors are on here: tests/test_mirror_bookkeeping.py has the case without std::vector cpu_words, cpu_words_recent, cpu_words_new; // (the big and the recent sorted sets, and noted since: vk_memory.cpp) #define CHECK(ok) do { if (!(ok)) { std::fprintf(stderr, "failed line %d\n", __LINE__); std::exit(1); } } while (0) ''' + helper + r''' diff --git a/tests/test_retired_give_back.py b/tests/test_retired_give_back.py new file mode 100644 index 0000000..ac81e65 --- /dev/null +++ b/tests/test_retired_give_back.py @@ -0,0 +1,218 @@ +"""When a retired pool's import is given back (gpu/vk/retired_give_back.h, vk_memory.cpp retired_imports_give_back; no game, +no GPU). + +(Triforce, 11-22, the performance audit's MEM-21, after Source A's.) A pool remade larger is retired; once the GPU is past +it, its import is pinned RAM nothing binds. Before: given back at once on Linux and with NVIDIA's driver on Windows, kept +with AMD's and Intel's on Windows, and never when the pool's mirror was refused. Now, OFF unless set: +RX_VK_KEEP_RETIRED=wait gives them back after a wait (RX_VK_RETIRE_SECS, 30; RX_VK_RETIRE_MIN_SECS, 2 while the system heap +is past 85%), and RX_VK_RETIRED_REFUSED=1 (on with wait unless 0) gives back the refused mirrors' imports too. + +Compiles the header: the modes by OS, vendor and switch; on 200,000 random texts, vendors and OSes the mode keeps where the +old reading kept and gives back at once where it did, unless the text is "wait"; on 200,000 random pools, with the wait and +the refused rule off, want() is the old condition; with the wait, nothing goes back before it is due, nor while the pool +that took it in still copies from its mirror, nor before the GPU is past it; due() with and without pressure; settled(); +a tour of 20 areas, three pools retired in each, where the wait keeps at most one area's and keep keeps them all. + +And reads the source: retired_imports_give_back decides by want() with the old condition's parts and the push's shape, +the switch is read once through mode() with Windows and NVIDIA as before, retire_pool notes the time, the dead imports' +bytes are added at the push and taken off where reap_mirrors frees them, the pressure give-back reads settled() only with +the wait, the switches table has the row. + + python -I -B tests/test_retired_give_back.py [--cxx clang++] +""" +import argparse +import re +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +PROGRAM = r""" +#include "retired_give_back.h" +#include +#include +#include +#include +#include +using namespace retired_give_back; +static int fails = 0; +#define CHECK(c) do { if (!(c)) { std::printf("FAIL line %d: %s\n", __LINE__, #c); fails++; } } while (0) + +int main() { + // (the modes by OS, vendor and switch) + CHECK(mode(nullptr, true, false) == Mode::Keep && mode(nullptr, true, true) == Mode::AtOnce); + CHECK(mode(nullptr, false, false) == Mode::AtOnce && mode(nullptr, false, true) == Mode::AtOnce); + for (int w = 0; w < 2; w++) + for (int n = 0; n < 2; n++) { + CHECK(mode("wait", w, n) == Mode::Wait); + CHECK(mode("0", w, n) == Mode::AtOnce && mode("00", w, n) == Mode::AtOnce && mode("0x", w, n) == Mode::AtOnce); + CHECK(mode("1", w, n) == Mode::Keep && mode("", w, n) == Mode::Keep && mode("keep", w, n) == Mode::Keep); + CHECK(mode("Wait", w, n) == Mode::Keep && mode("wait ", w, n) == Mode::Keep && mode("waits", w, n) == Mode::Keep); + } + // (old reading against new on random texts: the same everywhere but "wait") + std::mt19937_64 rng(2111); + const char* words[] = {"wait", "0", "1", "", "keep", "w", "00", "wai", "waitt", "2", "-1", " 0", "0 "}; + const char alpha[] = "wait01 x"; + for (int i = 0; i < 200000; i++) { + std::string s; + const bool unset = rng() % 8 == 0; + if (rng() % 2) s = words[rng() % (sizeof words / sizeof *words)]; + else for (int k = (int)(rng() % 6); k > 0; k--) s += alpha[rng() % (sizeof alpha - 1)]; + const char* e = unset ? nullptr : s.c_str(); + const bool w = rng() % 2, n = rng() % 2; + const Mode m = mode(e, w, n); + if (e && s == "wait") { CHECK(m == Mode::Wait); continue; } + CHECK(m != Mode::Wait); + CHECK((m == Mode::Keep) == keep_before(e, w, n)); + // (and the refused mirrors' rule off unless asked, outside the wait) + CHECK(!refused_too(nullptr, m)); + } + CHECK(refused_too(nullptr, Mode::Wait) && !refused_too("0", Mode::Wait) && refused_too("1", Mode::AtOnce) && !refused_too("0", Mode::Keep)); + // (the seconds) + CHECK(secs(nullptr, 30) == 30 && secs("", 30) == 30 && secs("5", 30) == 5 && secs("0", 2) == 0); + CHECK(secs("-3", 30) == 30 && secs("x", 30) == 30 && secs("4s", 30) == 30); + // (the pressure: past 85% of a known budget) + CHECK(!pressure(85, 100) && pressure(86, 100) && !pressure(1000, 0)); + CHECK(pressure((uint64_t)7 << 30, (uint64_t)8 << 30) && !pressure((uint64_t)6 << 30, (uint64_t)8 << 30)); + // (due: 30 s, or 2 s while the heap is short) + CHECK(!due(29999, 0, 30, 2, false) && due(30000, 0, 30, 2, false)); + CHECK(!due(1999, 0, 30, 2, true) && due(2000, 0, 30, 2, true)); + CHECK(due(5000, 5000, 0, 0, false)); + // (settled: the dead imports out of the heap's use, never below 0) + CHECK(settled(100, 30) == 70 && settled(30, 100) == 0 && settled(5, 5) == 0); + // (want against the old condition on random pools, and the wait's guards) + for (int i = 0; i < 200000; i++) { + const Mode m = (Mode)(rng() % 3); + const bool refused = rng() % 2, none = rng() % 2, ref = !none && rng() % 2, inherit = rng() % 2, past = rng() % 2, + is_due = rng() % 2; + const bool w = want(m, refused, none, ref, inherit, past, is_due); + if (m != Mode::Wait && !refused) CHECK(w == want_before(m == Mode::Keep, none, inherit, past)); + if (m == Mode::Keep) CHECK(!w); + if (m == Mode::Wait && !is_due) CHECK(!w); + if (inherit || !past) CHECK(!w); + if (!none && !(refused && ref)) CHECK(!w); + if (m == Mode::Wait && is_due && !inherit && past && (none || (refused && ref))) CHECK(w); + } + // (a tour: 20 areas a minute apart, three pools retired in each; checked a second at a time for two minutes past the last) + for (int mi = 0; mi < 3; mi++) { + const Mode m = (Mode)mi; + struct P { int64_t at; bool alive; }; + std::vector

pools; + size_t most = 0; + for (int64_t t = 0; t <= 21 * 60000; t += 1000) { + if (t % 60000 == 0 && t < 20 * 60000) + for (int k = 0; k < 3; k++) pools.push_back({t, true}); + size_t alive = 0; + for (P& p : pools) { + if (!p.alive) continue; + const bool d = due(t, p.at, 30, 2, false); + if (want(m, refused_too(nullptr, m), true, false, false, true, d)) p.alive = false; + else alive++; + } + most = std::max(most, alive); + } + size_t left = 0; + for (const P& p : pools) left += p.alive; + if (m == Mode::Keep) CHECK(left == 60 && most == 60); + if (m == Mode::AtOnce) CHECK(left == 0 && most == 0); + if (m == Mode::Wait) CHECK(left == 0 && most == 3); + } + std::printf(fails ? "%d failed\n" : "ok\n", fails); + return fails ? 1 : 0; +} +""" + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def in_order(text, steps, what): + at = -1 + for s in steps: + k = text.find(s, at + 1) + if k < 0: + raise SystemExit("FAIL: %s: %r is missing or out of its order" % (what, s)) + at = k + + +def check_source(): + mem = (VK / "vk_memory.cpp").read_text(encoding="utf-8") + if '#include "retired_give_back.h"' not in mem: + raise SystemExit("FAIL: vk_memory.cpp does not include retired_give_back.h") + pool = block(mem, "struct Pool {") + if "int64_t retired_ms = 0;" not in pool: + raise SystemExit("FAIL: Pool has no retired_ms") + retire = block(mem, "void retire_pool(std::map::iterator it) {") + in_order(retire, ["old->retired_value = std::max(old->last_value, mem_rec_value);", + "old->retired_ms = std::chrono::duration_cast(std::chrono::steady_clock::now()"], + "vk_memory.cpp retire_pool") + mode = block(mem, "retired_give_back::Mode retired_mode() {") + in_order(mode, ["static const retired_give_back::Mode m = [] {", "#ifdef _WIN32", "const bool windows = true;", "#else", + "const bool windows = false;", "#endif", + 'retired_give_back::mode(getenv("RX_VK_KEEP_RETIRED"), windows, dev.props.vendorID == 0x10DE);'], + "vk_memory.cpp retired_mode") + give = block(mem, "void retired_imports_give_back() {") + if "static const bool keep" in give or 'getenv("RX_VK_KEEP_RETIRED")' in give: + raise SystemExit("FAIL: retired_imports_give_back reads RX_VK_KEEP_RETIRED itself, not through retired_mode") + in_order(give, ["const rg::Mode mode = retired_mode();", 'rg::refused_too(getenv("RX_VK_RETIRED_REFUSED"), mode);', + 'rg::secs(getenv("RX_VK_RETIRE_SECS"), 30)', 'rg::secs(getenv("RX_VK_RETIRE_MIN_SECS"), 2)', + "if (!p.memory) {", # (DESC-06's walk: a pool with no import left is passed, and the walk's start moves past it) + "const bool gpu_past = p.retired_value <= mem_completed && p.last_value <= mem_completed;", + "if (mode == rg::Mode::Wait && gpu_past && !p.inherit_pending &&", + "is_due = rg::due(now_ms, p.retired_ms, wait_s, min_s, false);", + "if (!is_due && rg::due(now_ms, p.retired_ms, wait_s, min_s, true)) {", + "heap_short = system_heap_usage(&used, &budget) && rg::pressure(rg::settled(used, dead_import_bytes), budget);", + "if (pressed_back || rg::want(mode, refused, p.mstate == Pool::MIRROR_NONE, p.mstate == Pool::MIRROR_REFUSED, p.inherit_pending, gpu_past, is_due)) {", + "dead_import_bytes += p.hi - p.lo;", "dead_imports.push_back({p.buffer, p.memory,", "continue;", + "retired_alive++, retired_bytes += p.hi - p.lo;"], "vk_memory.cpp retired_imports_give_back") + if give.count("dead_imports.push_back(") != 1 or mem.count("dead_imports.push_back(") != 1 or \ + mem.count("dead_import_bytes += ") != 1: + raise SystemExit("FAIL: a retired import goes into dead_imports somewhere its bytes are not counted") + reap = block(mem, "void reap_mirrors() {") + in_order(reap, ["while (!dead_imports.empty() && dead_imports.front().value <= mem_completed) {", + "vkFreeMemory(dev.device, dead_imports.front().memory, nullptr);", + "dead_import_bytes -= std::min(dead_import_bytes, dead_imports.front().bytes);", "dead_imports.pop_front();"], + "vk_memory.cpp reap_mirrors") + valve = block(mem, "void imports_give_back() {") + in_order(valve, ["if (!system_heap_usage(&used, &budget)) return;", + "if (retired_mode() == retired_give_back::Mode::Wait) used = retired_give_back::settled(used, dead_import_bytes);", + "const bool by_heap = small && used > measure / 100 * (uint64_t)high;"], # (MEM-22: against its cap) + "vk_memory.cpp imports_give_back") + if valve.count("settled(") != 1: + raise SystemExit("FAIL: imports_give_back reads settled() somewhere else") + dev_md = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + if not any(line.startswith("| `RX_VK_KEEP_RETIRED=wait`, `RX_VK_RETIRED_REFUSED=1` |") and "**off unless set**" in line + and "RX_VK_RETIRE_SECS" in line and "RX_VK_RETIRE_MIN_SECS" in line for line in dev_md.splitlines()): + raise SystemExit("FAIL: docs/development.md has no row for RX_VK_KEEP_RETIRED=wait with its default") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + check_source() + with tempfile.TemporaryDirectory(prefix="retired-give-back-") as tmp: + src = Path(tmp) / "t.cpp" + exe = Path(tmp) / "t.exe" + src.write_text(PROGRAM, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(VK), str(src), "-o", str(exe)], + check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True, timeout=120) + print(r.stdout.strip()) + if r.returncode: + raise SystemExit("FAIL: retired_give_back.h") + print("ok: test_retired_give_back") + + +if __name__ == "__main__": + main() diff --git a/tests/test_rt_pool_minimum.py b/tests/test_rt_pool_minimum.py index 99fd1c7..0d55143 100644 --- a/tests/test_rt_pool_minimum.py +++ b/tests/test_rt_pool_minimum.py @@ -42,6 +42,8 @@ static const char *fixture_env(const char *name) { /* (local, RenoDX port: host/nn_vi.c's, whether the scene's colour target is RGBA16F for the run) */ static int scene_rgba16f; int hos_scene_rgba16f(void) { return scene_rgba16f; } +/* ((Triforce, 11-21, TEX-85) host/nn_applet.c's, beside the function: whether the heap is held to a size, which it notes) */ +static int rt_pool_pinned = -1; ''' + body + r''' #define WIDE(w, h) ((uint64_t)(h) << 32 | (w)) static const struct { diff --git a/tests/test_same_dyn.py b/tests/test_same_dyn.py index 06cc70d..1fc4295 100644 --- a/tests/test_same_dyn.py +++ b/tests/test_same_dyn.py @@ -44,9 +44,16 @@ def main(): guard = body[:body.index("dyn_same(")] if "last_dyn_valid" not in guard: raise SystemExit("a clear's dynamic state is left unset") - fixture = f'''#include + # ((Triforce, 11-22, the performance audit's DESC-06, after Source A's) the words and the bytes by one memcmp each, the + # floats still by value: the same answers as the three loops gave, at the last word and the last byte too, with -0 the + # same as 0 and a NaN the same as nothing + if fn.count("std::memcmp(") != 2 or "if (fp[i] != cfp[i]) return false;" not in fn: + raise SystemExit("dyn_same does not compare its floats by value and its words and bytes by two memcmp") + fixture = f'''#include +#include #include #include +#include #define CHECK(x) do {{ if (!(x)) {{ std::fprintf(stderr, "FAIL:%d %s\\n", __LINE__, #x); std::exit(1); }} }} while (0) {fn} int main() {{ @@ -63,6 +70,33 @@ int main() {{ craw[1] = 7; CHECK(!dyn_same(fp, 2, up, 2, raw, 2, cfp, cup, craw)); CHECK(!dyn_same(fp, 0, up, 2, raw, 2, cfp, cup, craw)); + craw[1] = 6; + CHECK(!dyn_same(fp, 2, up, 0, raw, 2, cfp, cup, craw)); + CHECK(!dyn_same(fp, 2, up, 2, raw, 0, cfp, cup, craw)); + CHECK(dyn_same(fp, 2, up, 2, raw, 2, cfp, cup, craw)); + // (the sizes the draw uses: 13 floats, 35 words, 160 bytes; a difference in the last of each, and only there) + float f13[13], g13[13]; + uint32_t u35[35], v35[35]; + uint8_t b160[160], c160[160]; + for (int i = 0; i < 13; i++) f13[i] = g13[i] = (float)i * 0.5f; + for (int i = 0; i < 35; i++) u35[i] = v35[i] = 0x01010101u * (uint32_t)i; + for (int i = 0; i < 160; i++) b160[i] = c160[i] = (uint8_t)(i * 7); + CHECK(dyn_same(f13, 13, u35, 35, b160, 160, g13, v35, c160)); + v35[34] ^= 0x80000000u; + CHECK(!dyn_same(f13, 13, u35, 35, b160, 160, g13, v35, c160)); + CHECK(dyn_same(f13, 13, u35, 34, b160, 160, g13, v35, c160)); // (the last word is past the count: not looked at) + v35[34] = u35[34], v35[0] ^= 1; + CHECK(!dyn_same(f13, 13, u35, 35, b160, 160, g13, v35, c160)); + v35[0] = u35[0], c160[159] ^= 1; + CHECK(!dyn_same(f13, 13, u35, 35, b160, 160, g13, v35, c160)); + CHECK(dyn_same(f13, 13, u35, 35, b160, 159, g13, v35, c160)); // (the last byte is past the count: not looked at) + c160[159] = b160[159], g13[12] += 0.25f; + CHECK(!dyn_same(f13, 13, u35, 35, b160, 160, g13, v35, c160)); + g13[12] = f13[12]; + f13[0] = 0.0f, g13[0] = -0.0f; // (-0 is 0: by value, not by its bits) + CHECK(std::memcmp(&f13[0], &g13[0], sizeof(float)) != 0 && dyn_same(f13, 13, u35, 35, b160, 160, g13, v35, c160)); + f13[5] = g13[5] = std::nanf(""); // (a NaN is the same as nothing, itself included: the state is built again) + CHECK(!dyn_same(f13, 13, u35, 35, b160, 160, g13, v35, c160)); std::printf("PASS: the same dynamic state is not built again\\n"); }} ''' diff --git a/tests/test_segment_heap.py b/tests/test_segment_heap.py new file mode 100644 index 0000000..b98e878 --- /dev/null +++ b/tests/test_segment_heap.py @@ -0,0 +1,154 @@ +"""Windows' segment heap for the exe, at the build's choice (HOOKSHOT_SEGMENT_HEAP=1, nx2/win_manifest.py). + +On AMD's Windows driver the shader preparation's compiles waited on the NT heap's lock (76% of the driver's samples in +RtlAllocateHeap/RtlFreeHeap's critical section): heapType SegmentHeap in the exe's manifest puts the whole process on the +segment heap. The heap is fixed when the process starts, so the two arms are two exes; on by default since the 11-22 play A/B (HOOKSHOT_SEGMENT_HEAP=0: off). + +What must hold: with HOOKSHOT_SEGMENT_HEAP=0 the manifest is as it was (no heapType); unset or 1, one heapType SegmentHeap in +the 2020 WindowsSettings namespace inside windowsSettings, the UTF-8 code page and longPathAware kept; write() follows the +variable, and nx2/build.py links write()'s .res. gpu/vk/prepare_threads.h's heap_name and the log line of vk_draw.cpp's +prepare_workers are checked by tests/test_prepare_threads.py. On Windows a probe that prints the signature at byte 16 of +GetProcessHeap() is linked with each .res: 0xDDEEDDEE (the segment heap) with the element, 0xFFEEFFEE (the NT heap) without. + + python -I -B tests/test_segment_heap.py [--cc clang] +""" +import argparse +import os +import shutil +import subprocess +import sys +import tempfile +import xml.etree.ElementTree as ET +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "nx2")) +import win_manifest # noqa: E402 + +WS2016 = "{http://schemas.microsoft.com/SMI/2016/WindowsSettings}" +WS2019 = "{http://schemas.microsoft.com/SMI/2019/WindowsSettings}" +WS2020 = "{http://schemas.microsoft.com/SMI/2020/WindowsSettings}" +fails = [] + +PROBE = r""" +#include +#include +#include +int main(void) { + unsigned sig = 0; + memcpy(&sig, (const char*)GetProcessHeap() + 16, sizeof sig); + void* p[64]; + for (int i = 0; i < 64; i++) p[i] = HeapAlloc(GetProcessHeap(), 0, 64 + i * 100); + for (int i = 0; i < 64; i++) HeapFree(GetProcessHeap(), 0, p[i]); + printf("0x%08X\n", sig); + return 0; +} +""" + + +def check(ok, what): + print(("ok " if ok else "FAIL ") + what) + if not ok: + fails.append(what) + + +def settings(text): + """[(tag, text)] of a manifest's windowsSettings, in order""" + root = ET.fromstring(text) + for el in root.iter(): + if el.tag.endswith("}windowsSettings"): + return [(s.tag, (s.text or "").strip()) for s in el] + return [] + + +def check_manifest(): + check(hasattr(win_manifest, "manifest") and hasattr(win_manifest, "segment_heap_asked"), + "nx2/win_manifest.py has manifest() and segment_heap_asked()") + if fails: + return + plain = win_manifest.manifest(False) + check(plain == win_manifest.MANIFEST, "without the segment heap: the manifest as it was") + check(b"heapType" not in plain, "without the segment heap: no heapType") + on = win_manifest.manifest(True) + s = settings(on) + heap = [t for tag, t in s if tag == WS2020 + "heapType"] + check(heap == ["SegmentHeap"], "with it: one heapType SegmentHeap, 2020 namespace, inside windowsSettings") + check(not any(tag.endswith("}heapType") and tag != WS2020 + "heapType" for tag, _ in s), "no heapType in another namespace") + check((WS2019 + "activeCodePage", "UTF-8") in s and (WS2016 + "longPathAware", "true") in s, + "with it: the UTF-8 code page and longPathAware kept") + check(on.replace(win_manifest.SEGMENT_HEAP, b"") == plain, "with it: nothing else changed") + for env, want in (({}, True), ({"HOOKSHOT_SEGMENT_HEAP": ""}, True), ({"HOOKSHOT_SEGMENT_HEAP": "0"}, False), + ({"HOOKSHOT_SEGMENT_HEAP": " 0 "}, False), ({"HOOKSHOT_SEGMENT_HEAP": "1"}, True), + ({"HOOKSHOT_SEGMENT_HEAP": " 1 "}, True)): + check(win_manifest.segment_heap_asked(env) == want, "HOOKSHOT_SEGMENT_HEAP %r: %s" % (env.get("HOOKSHOT_SEGMENT_HEAP"), + "on" if want else "off")) + with tempfile.TemporaryDirectory(prefix="segment-heap-") as tmp: + res = os.path.join(tmp, "m.res") + was = os.environ.pop("HOOKSHOT_SEGMENT_HEAP", None) + try: + win_manifest.write(res) + check(open(res, "rb").read() == win_manifest.res_bytes(on), "write(), the variable unset: the .res with heapType (on by default)") + os.environ["HOOKSHOT_SEGMENT_HEAP"] = "0" + win_manifest.write(res) + check(open(res, "rb").read() == win_manifest.res_bytes(), "write(), HOOKSHOT_SEGMENT_HEAP=0: the .res as it was") + check(open(win_manifest.write(res, False), "rb").read() == win_manifest.res_bytes(), "write(path, False): as it was") + finally: + os.environ.pop("HOOKSHOT_SEGMENT_HEAP", None) + if was is not None: + os.environ["HOOKSHOT_SEGMENT_HEAP"] = was + build = (ROOT / "nx2" / "build.py").read_text(encoding="utf-8") + start = build.index('exe = os.path.join(out, "Triforce.exe")') + check("win_manifest.write(" in build[start:build.index("capture_output=True", start)], "nx2/build.py links write()'s .res") + docs = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`HOOKSHOT_SEGMENT_HEAP=0`" in docs, "docs/development.md has HOOKSHOT_SEGMENT_HEAP") + + +def windows_cc(given): + if given: + return given + if os.name == "nt": + return shutil.which("clang") + mingw = os.environ.get("HOOKSHOT_MINGW", "") + cc = os.path.join(mingw, "bin", "x86_64-w64-mingw32-clang") + return cc if mingw and os.path.exists(cc) else None + + +def check_probe(cc_arg): + cc = windows_cc(cc_arg) + if not cc: + print("SKIP probe: no compiler for Windows here (clang on Windows, or HOOKSHOT_MINGW=)") + return + with tempfile.TemporaryDirectory(prefix="segment-heap-") as tmp: + tmp = Path(tmp) + src = tmp / "probe.c" + src.write_text(PROBE, encoding="utf-8") + target = [] if os.name == "nt" or "mingw32" in os.path.basename(cc) else ["--target=x86_64-w64-mingw32"] + flags = target + ["-std=c11", "-O0", "-Wall", "-Wextra", "-Werror", "-fuse-ld=lld"] + arms = [] + for name, on, want in (("segment", True, "0xDDEEDDEE"), ("nt", False, "0xFFEEFFEE")): + res = win_manifest.write(str(tmp / (name + ".res")), on) + exe = tmp / ("probe_%s.exe" % name) + subprocess.run([cc] + flags + [str(src), res, "-o", str(exe)], check=True) + arms.append((name, exe, want)) + if os.name != "nt": + print("probe built for Windows; it runs on Windows only") + return + for name, exe, want in arms: + got = subprocess.run([str(exe)], capture_output=True, text=True, check=True).stdout.strip() + check(got == want, "probe with the %s .res: process heap signature %s (want %s)" % (name, got, want)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cc", default="") + args = parser.parse_args() + check_manifest() + if not fails: + check_probe(args.cc) + if fails: + raise SystemExit("FAIL: %d check(s)" % len(fails)) + print("PASS: the segment heap at the build's choice") + + +if __name__ == "__main__": + main() diff --git a/tests/test_settle_gate.py b/tests/test_settle_gate.py new file mode 100644 index 0000000..a1dc7eb --- /dev/null +++ b/tests/test_settle_gate.py @@ -0,0 +1,298 @@ +"""A read looks at the stopped queues only after a new CPU signal (gpu/vk/settle_gate.h, vk_replay.cpp; no game, no GPU). + +(Triforce, 11-22, the performance audit's SYNC-29, after Source A's.) A CPU read that lets the stopped queues whose waits are +met go on takes the order's lock whenever a queue is stopped: the readback's look (RX_VK_RB_SETTLE, off) and the two looks made +for a range a texture may be in. RX_VK_SETTLE_GATE=1 gates the first, =2 the others too; unset, every look as before. + +Compiles a driver against the header and +- checks the switch's text, the levels the lab knobs raise, which site is gated at which level, and the decision's table; +- runs a model of the order: three queues that stop at waits on four events, holders of the order's lock that replay signals + of the stream, look (schedule) and end their burst before they let the lock go, the CPU's signals (queued, then counted, as + two steps that fall anywhere between a holder's steps), a holder that only takes the pending signals in without looking + (the submit's gate wait), and plain stores into an event's memory. 400,000 random steps, and at every quiet point (no + signal half made) a read asks the header's decide(): whenever the gate skips the look, a look would have let nothing on. + With no gate (level 0) nothing is ever skipped; +- runs the same model with each of three mistakes and requires that it then FINDS a skipped look that would have let work + on: the signal counted before it is queued; the count published as it is at the look's end; a wait on an event read from + memory not marked. +And reads vk_replay.cpp: the signal counted after it is queued; the count noted in burst_begin and published in burst_end +after finish_recording; update_blocked marks a wait read from memory; the three sites go through sync_settle_at with their +site, which asks the header and counts; the holders the model stands on (submit_job looks and ends its burst, submit_q +updates after it); the switches table. + + python -I -B tests/test_settle_gate.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +DRIVER = r''' +#include "settle_gate.h" +#include +#include + +namespace sg = mkwvk::settle_gate; +static int checks, fails; +#define CHECK(what, cond) do { checks++; if (!(cond)) fails++, printf("FAIL %s (line %d)\n", what, __LINE__); } while (0) + +enum Mistake { kNone, kCountFirst, kPublishAtEnd, kNoMemoryMark }; + +struct Model { + Mistake mistake = kNone; + uint64_t rng = 0x9E3779B97F4A7C15ull; + uint64_t rnd(uint64_t n) { rng ^= rng << 13, rng ^= rng >> 7, rng ^= rng << 17; return rng % n; } + // the events: a value the replay knows (set) or the game's memory + // (the first two are signalled, by the stream and the CPU; the last two only ever stored into: read from memory for good) + static constexpr int kEvents = 4, kSignalled = 2, kQueues = 3; + uint32_t value[kEvents] = {}, memory[kEvents] = {}; + bool set[kEvents] = {}; + std::vector> pending; // (the CPU's signals queued) + uint64_t signals = 0, settled = 0, burst_signals = 0; + bool wait_in_memory = false; + struct Queue { bool waiting = false; int event = 0; uint32_t want = 0; } q[kQueues]; + // a CPU signal half made: 0 none; else its event + 1 + int flight = 0; + uint32_t flight_value = 0; + int64_t skips = 0, looks = 0, skipped_met = 0, quiet_reads = 0, resumed = 0; + + uint32_t have(int e) const { return set[e] ? value[e] : memory[e]; } + bool met(const Queue& w) const { return w.waiting && (int32_t)(have(w.event) - w.want) >= 0; } + void apply() { // (ev_cpu_apply) + for (const auto& s : pending) value[s.first] = s.second, set[s.first] = true; + pending.clear(); + } + void signal_step() { // (event_signal_queued, a step at a time) + if (!flight) { + if (rnd(3)) return; + flight = (int)rnd(kSignalled) + 1, flight_value = have(flight - 1) + 1 + (uint32_t)rnd(2); + if (mistake == kCountFirst) signals++; + else pending.emplace_back(flight - 1, flight_value); + return; + } + if (mistake == kCountFirst) pending.emplace_back(flight - 1, flight_value); + else signals++; + flight = 0; + } + /// None, one or two steps of the CPU's signals: another thread's, anywhere between a holder's steps + void signal_steps() { + for (uint64_t n = rnd(3); n; n--) signal_step(); + } + void schedule() { + apply(); + for (bool moved = true; moved;) { + moved = false; + for (Queue& w : q) + if (met(w)) { + w.waiting = false, moved = true, resumed++; + if (rnd(2)) stream_signal(); // (what it replays may signal and let another on) + } + } + } + void stream_signal() { + const int e = (int)rnd(kSignalled); + value[e] = have(e) + (uint32_t)rnd(3), set[e] = true; + } + void update_blocked() { + bool in_memory = false; + for (const Queue& w : q) + if (w.waiting && !set[w.event]) in_memory = true; + wait_in_memory = mistake == kNoMemoryMark ? false : in_memory; + } + bool blocked() const { return q[0].waiting || q[1].waiting || q[2].waiting; } + /// A holder of the order's lock: its burst begins, it replays (signals of the stream, a wait it stops at), looks, ends its + /// burst and updates. The CPU's signal steps fall between its steps + void section(bool replays) { + burst_signals = signals; // (burst_begin) + signal_steps(); + if (replays) { + if (rnd(2)) stream_signal(); + const int k = (int)rnd(kQueues); + if (!q[k].waiting && rnd(2)) { // (NOP_WAIT_EVENT: the pending signals taken in, then the wait) + apply(); + Queue w{true, rnd(8) ? (int)rnd(kSignalled) : kSignalled + (int)rnd(kEvents - kSignalled), 0}; // (one in eight on memory) + w.want = have(w.event) + 1 + (uint32_t)rnd(2); + if (!met(w)) q[k] = w; + } + signal_steps(); + } + schedule(); + signal_steps(); + settled = mistake == kPublishAtEnd ? signals : burst_signals; // (burst_end, after finish_recording) + update_blocked(); + } + /// A read at a quiet point: the header's decision at this level; a skipped look that would have let work on is the fault + void read(int level) { + const int what = sg::decide(level, sg::kReadback, blocked(), signals, settled, wait_in_memory); + if (what == sg::kNothing) return; + if (what == sg::kSkip) { + skips++; + if (!flight) { + quiet_reads++; + Model after = *this; + after.apply(); + for (const Queue& w : after.q) skipped_met += after.met(w); + } + return; + } + looks++; + section(false); + } + void run(int level, int steps) { + for (int i = 0; i < steps; i++) { + switch (rnd(8)) { + case 0: case 1: section(true); break; + case 2: signal_step(); break; + case 3: apply(); break; // (the submit's gate wait: signals taken in, no look, nothing published) + case 4: memory[rnd(kEvents)] += (uint32_t)rnd(3); break; // (a plain store into an event's memory) + default: read(level); break; + } + } + } +}; + +int main() { + // the switch and the levels + CHECK("unset", sg::level(nullptr) == 0); + CHECK("empty", sg::level("") == 0); + CHECK("0", sg::level("0") == 0); + CHECK("1", sg::level("1") == 1); + CHECK("2", sg::level("2") == 2); + CHECK("3", sg::level("3") == 0); + CHECK("a word", sg::level("on") == 0); + CHECK("10", sg::level("10") == 0); + CHECK("no knob", sg::level_now(0, false, false) == 0 && sg::level_now(1, false, false) == 1 && sg::level_now(2, false, false) == 2); + CHECK("settlegate", sg::level_now(0, true, false) == 1 && sg::level_now(2, true, false) == 2); + CHECK("texgate", sg::level_now(0, false, true) == 2 && sg::level_now(1, true, true) == 2); + CHECK("level 0 gates nothing", !sg::gated(0, sg::kReadback) && !sg::gated(0, sg::kTexture)); + CHECK("level 1 gates the readback's look", sg::gated(1, sg::kReadback) && !sg::gated(1, sg::kTexture)); + CHECK("level 2 gates both", sg::gated(2, sg::kReadback) && sg::gated(2, sg::kTexture)); + // the decision: nothing stopped, nothing to do at any level; ungated, a look whenever a queue is stopped + for (int level = 0; level <= 2; level++) + for (int site = 0; site < sg::kSites; site++) + for (int bits = 0; bits < 8; bits++) { + const bool blocked = bits & 1, fresh = bits & 2, mem = bits & 4; + const int d = sg::decide(level, site, blocked, fresh ? 8 : 7, 7, mem); + const int want = !blocked ? sg::kNothing : (sg::gated(level, site) && !fresh && !mem) ? sg::kSkip : sg::kLook; + CHECK("the decision's table", d == want); + } + CHECK("a count behind as well as ahead is a new signal", sg::look(true, 3, 9, false) && !sg::look(true, 9, 9, false) && !sg::look(false, 3, 9, true)); + + // the model + { + Model m; + m.run(1, 400000); + printf("gate on: %lld looks, %lld skipped (%lld at a quiet point), %lld waits let on\n", (long long)m.looks, (long long)m.skips, + (long long)m.quiet_reads, (long long)m.resumed); + CHECK("the gate skipped a look that would have let work on", m.skipped_met == 0); + CHECK("the model skipped looks and let work on", m.quiet_reads > 5000 && m.looks > 5000 && m.resumed > 5000); + } + { + Model m; + m.run(0, 100000); + CHECK("no gate: nothing skipped", m.skips == 0 && m.looks > 1000); + } + static const struct { Mistake mistake; const char* what; } kMistakes[] = { + {kCountFirst, "a signal counted before it is queued was not found"}, + {kPublishAtEnd, "the count published as it is at the look's end was not found"}, + {kNoMemoryMark, "a wait on an event read from memory, not marked, was not found"}, + }; + for (const auto& k : kMistakes) { + Model m; + m.mistake = k.mistake; + m.run(1, 400000); + CHECK(k.what, m.skipped_met > 0); + } + if (fails) printf("settle gate: FAILED (%d of %d checks)\n", fails, checks); + else printf("settle gate: all %d checks passed\n", checks); + return fails != 0; +} +''' + + +def check(ok, what): + if not ok: + raise SystemExit("FAIL: " + what) + + +def function(text, head): + at = text.index(head) + return text[at:text.index("\n}\n", at)] + + +def check_source(): + replay = (ROOT / "gpu/vk/vk_replay.cpp").read_text(encoding="utf-8") + # every CPU signal is counted, after it is queued (and nothing else queues one) + check(replay.count("ev_cpu_pending.emplace_back(") == 1 and replay.count("ev_cpu_signals.fetch_add(") == 1, + "a CPU signal is queued somewhere it is not counted, or counted twice") + sig = function(replay, "void event_signal_queued(uint64_t gpu, uint32_t value) {") + queued = sig.index("ev_cpu_pending.emplace_back(gpu, value);") + counted = sig.index("ev_cpu_signals.fetch_add(1, std::memory_order_release);") + check(queued < sig.index("ev_cpu_any.store(true, std::memory_order_release);") < counted < sig.index("return;"), + "event_signal_queued counts the signal before it is queued, or not on the submitting thread's path") + # the look notes the count at its start and publishes it after its recording is finished + begin = function(replay, "void burst_begin(uint64_t value) {") + check("burst_signals = ev_cpu_signals.load(std::memory_order_acquire);" in begin, "burst_begin does not note the signals' count") + end = function(replay, "void burst_end() {") + check("ev_cpu_settled.store(burst_signals, std::memory_order_release);" in end and + end.index("if (rec_cur) finish_recording(burst_value);") < end.index("ev_cpu_settled.store(") and + end.index("if (!burst_open) return;") < end.index("ev_cpu_settled.store("), + "burst_end does not publish the noted count after finish_recording") + check(replay.count("ev_cpu_settled.store(") == 1 and "ev_cpu_settled.store(ev_cpu_signals" not in replay, + "the settled count is published somewhere else, or as the count is at the look's end") + upd = function(replay, "void update_blocked() {") + check("if (q.waiting && ev_values.find(wait_at(q.p).gpu) == ev_values.end()) in_memory = true;" in upd and + "ev_wait_in_memory.store(in_memory, std::memory_order_release);" in upd, + "update_blocked does not mark a stopped queue that waits on an event read from guest memory") + # the sites + at = function(replay, "void sync_settle_at(int site) {") + check("settle_gate::decide(settle_gate_level(), site, blocked_n.load() != 0, ev_cpu_signals.load(std::memory_order_acquire)," in at and + "ev_cpu_settled.load(std::memory_order_acquire), ev_wait_in_memory.load(std::memory_order_acquire));" in at, + "sync_settle_at does not ask the header with the counts and the mark") + check(at.index("if (what == settle_gate::kNothing) return;") < at.index("if (what == settle_gate::kSkip) {") < at.index("sync_settle();") and + "st_settle_skips[site].fetch_add(" in at and "st_settle_looks[site].fetch_add(" in at, "sync_settle_at does not look, skip and count as decided") + lvl = function(replay, "int settle_gate_level() {") + check('settle_gate::level(getenv("RX_VK_SETTLE_GATE"));' in lvl and + 'return settle_gate::level_now(env, lab_ab::on("settlegate"), lab_ab::on("texgate"));' in lvl, + "RX_VK_SETTLE_GATE and its two lab knobs are not read") + inv = function(replay, "void invalidate(uint64_t gpu, uint64_t size) {") + check(inv.count("rb_fix(2)) sync_settle_at(settle_gate::kReadback), w = true;") == 1, "the readback's look is not the gated site, behind RX_VK_RB_SETTLE") + check(inv.count("unforced ? sync_settle_at(settle_gate::kTexture) : wait_queue_empty();") == 2, "the two texture sites are not the gated site") + check("sync_settle()" not in inv.replace("sync_settle() as it was", ""), "invalidate settles past the gate somewhere") + check("st_settle_looks[settle_gate::kReadback].exchange(0)" in inv and "st_settle_looks[settle_gate::kTexture].exchange(0)" in inv, + "the looks and skips are not in the two log lines") + # what the model stands on: a holder that replays looks and ends its burst before it lets the lock go + job = function(replay, "void submit_job(std::vector& handles, uint64_t value, uint64_t queue) {") + check(" schedule(); // (what it signalled may let the stopped ones on)\n burst_end();" in job, "submit_job no longer looks and ends its burst after its replay") + check(" submit_job(h, value, queue);\n update_blocked();" in replay, "submit_q no longer updates the stopped queues after its job") + settle = function(replay, "void sync_settle() {") + check(settle.index("burst_begin(0);") < settle.index("schedule();") < settle.index("burst_end();") < settle.index("update_blocked();"), + "sync_settle is no longer a look inside a burst") + dev = (ROOT / "docs/development.md").read_text(encoding="utf-8") + row = [line for line in dev.splitlines() if line.startswith("| `RX_VK_SETTLE_GATE=1`, `=2` |")] + check(len(row) == 1 and "off by default" in row[0] and "`settlegate`" in row[0] and "`texgate`" in row[0], + "docs/development.md: the switches table does not have RX_VK_SETTLE_GATE's row") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + check_source() + with tempfile.TemporaryDirectory(prefix="settle-gate-") as tmp: + src = Path(tmp) / "driver.cpp" + src.write_text(DRIVER, encoding="utf-8") + exe = Path(tmp) / "driver.exe" + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I", str(ROOT / "gpu/vk"), str(src), "-o", str(exe)], + check=True) + out = subprocess.run([str(exe)], capture_output=True, text=True) + print(out.stdout, end="") + check(out.returncode == 0, "the header's checks or the model failed") + print("PASS: a read looks at the stopped queues only after a new CPU signal, where RX_VK_SETTLE_GATE says so") + + +if __name__ == "__main__": + main() diff --git a/tests/test_shader_stamp_cache.py b/tests/test_shader_stamp_cache.py index 6d96d48..b53ad4d 100644 --- a/tests/test_shader_stamp_cache.py +++ b/tests/test_shader_stamp_cache.py @@ -21,7 +21,9 @@ BUILD_SCRIPT = ROOT / "nx2" / "build.py" def production_code(path): tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - names = {"tree_headers", "vk_sources", "flag_text", "object_key", "strip_comments", "translation_text", "translation_stamp"} + # ((11-21, the performance audit's LIFT-10) vk_pgo_flags: the renderer's own files' profile flags, which the jobs ask for) + names = {"tree_headers", "vk_sources", "flag_text", "object_key", "strip_comments", "translation_text", "translation_stamp", + "vk_pgo_flags"} functions = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in names] assert {n.name for n in functions} == names main = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "main") @@ -89,8 +91,9 @@ class ShaderStampCacheTests(unittest.TestCase): path.write_bytes(text.encode()) self.code = production_code(BUILD_SCRIPT) - def jobs(self, linux, version="fixture", release=False, ngx="", mingw="llvm-mingw", fsr=False): + def jobs(self, linux, version="fixture", release=False, ngx="", mingw="llvm-mingw", fsr=False, vk_pgo=""): namespace = { + "VK_PGO": vk_pgo, # ((11-21, LIFT-10) RX_VK_PGO: "", "gen" or a profile's path) "os": os, "hashlib": hashlib, "GPU": str(self.root / "gpu"), "POSIX": str(self.root / "host/posix"), "HOST": str(self.root / "host"), "LINUX": linux, "CPU": ["-march=x86-64"], @@ -208,6 +211,37 @@ class ShaderStampCacheTests(unittest.TestCase): {"gpu/vk/vk_dlss.cpp"} | ({"gpu/vk/vk_ngx_msvc_rt.cpp"} if mingw and not linux else set())) self.assertEqual(self.changed(plain, {n: ngx[n] for n in plain}), {"gpu/vk/vk_dlss.cpp"}) + def test_a_renderer_profile_reaches_only_the_renderers_own_files(self): + # ((11-21, the performance audit's LIFT-10) RX_VK_PGO=: the back end's own files are built with the profile; the + # vendored translator, ImGui, zstd and the fonts are not; the same profile at another path is no new key, another + # profile is; and the shader stamp, which hashes the translator's text and not the flags, does not move) + one, moved, other = self.root / "a.profdata", self.root / "elsewhere" / "a.profdata", self.root / "b.profdata" + moved.parent.mkdir() + one.write_bytes(b"one profile"), moved.write_bytes(b"one profile"), other.write_bytes(b"another profile") + own = {"gpu/vk/vk_shader.cpp", "gpu/vk/vk_bank.cpp", "gpu/vk/vk_draw.cpp"} + for linux in (False, True): + with self.subTest(linux=linux): + plain = self.jobs(linux) + profiled = self.jobs(linux, vk_pgo=str(one)) + changed = self.changed(plain, profiled) + self.assertTrue(own <= changed, changed) + for name in changed: + # (the back end's own: gpu/vk's files and the two texture decoders it compiles with them) + self.assertTrue(name.startswith("gpu/") and "/eden/" not in name and "/sirit/" not in name, name) + self.assertTrue(any(f.startswith("-fprofile-instr-use=") for f in profiled[name]["flags"]), name) + self.assertEqual(self.changed(profiled, self.jobs(linux, vk_pgo=str(moved))), set()) + self.assertEqual(self.changed(profiled, self.jobs(linux, vk_pgo=str(other))), changed) + stamp = lambda jobs: [f for f in jobs["gpu/vk/vk_shader.cpp"]["flags"] if "SHADER_STAMP" in f or "TRANSLATION" in f] + self.assertEqual(stamp(plain), stamp(profiled)) + self.assertTrue(stamp(plain)) + (self.root / "host/pgo_rt.c").write_bytes(b"int pgo;") # (the counters' writer, which a counting DLL compiles) + counting = self.jobs(False, vk_pgo="gen") if not linux else None + if counting is not None: # (Windows: the counters' writer joins the DLL) + self.assertIn("host/pgo_rt.c", counting) + self.assertIn('-DPGO_ENV="RX_VK_PGO_FILE"', counting["host/pgo_rt.c"]["flags"]) + self.assertIn("-fprofile-instr-generate", counting["gpu/vk/vk_draw.cpp"]["flags"]) + self.assertNotIn("-fprofile-instr-generate", counting["gpu/vk/eden/translate.cpp"]["flags"]) + def test_fsr_reaches_only_vk_fsr4(self): for linux in (False, True): with self.subTest(linux=linux): diff --git a/tests/test_shadow_prev.py b/tests/test_shadow_prev.py index 48912bc..1d8eeb6 100644 --- a/tests/test_shadow_prev.py +++ b/tests/test_shadow_prev.py @@ -85,7 +85,8 @@ def check_source(): "if (!dropped.empty() && dropped[c->slots[i]]) continue;", "if (s == shadow.end() || !s->second.b) continue;", "cap_ready(*c, c->slots[i]);", "if (sh_on && !sh_any++) sh_looks.assign(c->copied.size(), shadow_prev::Look{});", - "if (memcmp(s->second.b.get(), (const uint8_t*)c->map + (uint64_t)c->slots[i] * 4096, 4096) != 0) {", + # ((Triforce, 11-22, MEM-39) the capture's bytes from the staging when it holds the page: tests/test_tex_same_staged.py) + "cap_page_bytes(*c, c->slots[i]);", "if (memcmp(s->second.b.get(), cap_b, 4096) != 0) {", "if (sh_on)", "sh_looks[i] = shadow_prev::look(false, s->second.b.get(), s->second.prev, cap_page_bytes(*c, c->slots[i]), sh_prev,", "sh_vers);", "continue;", "same_pages.push_back({p, gens.page_tex(p)});", "if (sh_on) sh_looks[i] = shadow_prev::look(true, s->second.b.get(), s->second.prev, nullptr, sh_prev, sh_vers);", @@ -99,7 +100,7 @@ def check_source(): # (the shadow's loop: the pages of this capture, for one taken twice, only when a look was kept) "std::sort(same_set.begin(), same_set.end());", "if (sh_any) {", "sh_pages.push_back(p);", "std::sort(sh_pages.begin(), sh_pages.end());", - "for (; i < c->copied.size(); i++) {", "if (!in_tex_merged((uint64_t)p << 12)) continue;", + "for (; i < c->copied.size(); i++) {", "if (!in_tex.in((uint64_t)p << 12)) continue;", # ((Triforce, 11-22, MEM-42) the loop's walk) "if (shadow_n.load(std::memory_order_relaxed) >= kShadowMax && !shadow.count(p)) {", # (left: before the slot is taken, as the slot is now) "if (sh_any && sh_looks[i].v != shadow_prev::NONE) {", "const auto s = shadow.find(p);", diff --git a/tests/test_shadow_slot.py b/tests/test_shadow_slot.py new file mode 100644 index 0000000..b4bcceb --- /dev/null +++ b/tests/test_shadow_slot.py @@ -0,0 +1,112 @@ +"""A character's shadow map reserves the storage of the next step of sides up (host/shadow_slot.h, host/nvn_native.c +shadow_slot_layout and the builder's two storage answers, host/nn_applet.c hos_rt_pool_pinned; no game). + +(Triforce, 11-21, the performance audit's TEX-85, after Source A's.) The game sizes each character's shadow map from the +camera's distance every frame and places a frame's render targets first fit in its target heap: a map a few pixels +larger moved every target made after it, and each moved one is a new image to the renderer. + +This compiles shadow_slot.h alone and checks the rule: 385 reserves 768, 1277 reserves 1536, 1537 and 1920 reserve 1920; +384 and less, 1921 and more, a map that is not square, another format, two levels, a 3D texture and step 0 reserve their +own (0); the step is 384 when not set, 0 on a pinned heap, and the switch's value when set, on a pinned heap too. + +And reads the source: both of the builder's answers take the larger of the texture's own and the slot's, after the +no-builder guard; the slot's layout is nvn_tex_layout of a copy at the slot's side, the builder untouched; the step asks +whether the heap is pinned and is not kept until the heap has been sized; the heap says pinned for HOS_RT_POOL_MB and +for the 5120-wide cap; the switch is in the switches table. + + python -I -B tests/test_shadow_slot.py [--cc clang] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +HOST = ROOT / "host" +RUN_ALL_ARGS = ["--cc", "clang"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +PROGRAM = r""" +#include +#include +#include "shadow_slot.h" +static int fails; +#define CHECK(x) do { if (!(x)) { printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +static uint32_t side(uint32_t w, uint32_t h) { return hos_shadow_slot_side(1, 0x34, 1, 1, w, h, 384); } +int main(void) { + CHECK(side(385, 385) == 768 && side(397, 397) == 768 && side(439, 439) == 768 && side(768, 768) == 768); + CHECK(side(769, 769) == 1152 && side(1277, 1277) == 1536 && side(1537, 1537) == 1920 && side(1920, 1920) == 1920); + CHECK(side(384, 384) == 0 && side(256, 256) == 0 && side(1921, 1921) == 0 && side(4096, 4096) == 0); + CHECK(side(500, 501) == 0); /* (not square) */ + CHECK(hos_shadow_slot_side(1, 0x32, 1, 1, 500, 500, 384) == 0); /* (another format) */ + CHECK(hos_shadow_slot_side(1, 0x34, 2, 1, 500, 500, 384) == 0); /* (two levels) */ + CHECK(hos_shadow_slot_side(0, 0x34, 1, 1, 500, 500, 384) == 0); /* (not two-dimensional) */ + CHECK(hos_shadow_slot_side(1, 0x34, 1, 2, 500, 500, 384) == 0); /* (deep) */ + CHECK(hos_shadow_slot_side(1, 0x34, 1, 1, 500, 500, 0) == 0); /* (the switch off) */ + CHECK(hos_shadow_slot_side(1, 0x34, 1, 0, 500, 500, 384) == 768); /* (depth 0 or 1: a plain 2D texture) */ + CHECK(hos_shadow_slot_side(1, 0x34, 1, 1, 500, 500, 256) == 512 && hos_shadow_slot_side(1, 0x34, 1, 1, 1900, 1900, 512) == 1920); + /* the step */ + CHECK(hos_shadow_slot_step(NULL, 0) == 384 && hos_shadow_slot_step(NULL, 1) == 0); + CHECK(hos_shadow_slot_step("0", 0) == 0 && hos_shadow_slot_step("384", 1) == 384 && hos_shadow_slot_step("256", 0) == 256); + CHECK(hos_shadow_slot_step("off", 0) == 0 && hos_shadow_slot_step("-1", 0) == 0 && hos_shadow_slot_step("", 1) == 384); + return fails ? 1 : 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cc", default="clang") + args = parser.parse_args() + nvn = (HOST / "nvn_native.c").read_text(encoding="utf-8") + check('#include "shadow_slot.h"' in nvn, "nvn_native.c does not use the rule") + slot = block(nvn, "static int shadow_slot_layout(const nvn_texdesc *d, nvn_texlayout *out) {") + check('const char *e = getenv("HOS_SHADOW_BUCKET");' in slot and "const int pinned = hos_rt_pool_pinned();" in slot and + "now = (int)hos_shadow_slot_step(e, pinned > 0);" in slot and "if (e || pinned >= 0) {" in slot, + "the step is not the rule's of the switch and the pinned heap, or is kept before the heap has been sized") + check("hos_shadow_slot_side(d->target == NVN_TEXTURE_TARGET_2D, d->format, d->levels, d->depth, d->width," in slot and + "nvn_texdesc slot = *d;" in slot and "slot.width = slot.height = side;" in slot and "nvn_tex_layout(&slot, out);" in slot, + "the slot's layout is not nvn_tex_layout of a copy at the slot's side") + check("d->width =" not in slot and "d->height =" not in slot, "the texture's own description is changed") + size = block(nvn, "NV(nvnTextureBuilderGetStorageSize) {") + check(size.index('no_builder(c, "storage size")') < size.index("shadow_slot_layout(") and + "if (L.bpb && shadow_slot_layout(&CTB(P(0))->d, &slot) && slot.size > L.size) L.size = slot.size;" in size, + "the storage size is not the larger of the texture's own and its slot's, after the no-builder guard") + align = block(nvn, "NV(nvnTextureBuilderGetStorageAlignment) {") + check(align.index('no_builder(c, "storage alignment")') < align.index("shadow_slot_layout(") and + "if (L.bpb && shadow_slot_layout(&CTB(P(0))->d, &slot) && slot.align > L.align) L.align = slot.align;" in align, + "the storage alignment is not the larger of the texture's own and its slot's") + applet = (HOST / "nn_applet.c").read_text(encoding="utf-8") + pool = block(applet, "uint64_t hos_rt_pool_size(uint64_t game) {") + check("__atomic_store_n(&rt_pool_pinned, e != NULL || v != asked || (is_five_k && scaled > five_k)," in pool and + "static int rt_pool_pinned = -1;" in applet and "int hos_rt_pool_pinned(void) {" in applet, + "the target heap does not say whether it is pinned (HOS_RT_POOL_MB, the 5120-wide cap), or not that it is not known yet") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`HOS_SHADOW_BUCKET=`" in dev, "docs/development.md: HOS_SHADOW_BUCKET is not in the switches table") + + with tempfile.TemporaryDirectory(prefix="shadow-slot-") as tmp: + c, exe = Path(tmp) / "slot.c", Path(tmp) / "slot.exe" + c.write_text(PROGRAM, encoding="utf-8") + subprocess.run([args.cc, "-std=c11", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(HOST), str(c), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True, timeout=60) + check(r.returncode == 0, "the rule as made:\n" + r.stdout) + print("PASS: a character's shadow map reserves the storage of the next step of sides up; its own layout is untouched") + + +if __name__ == "__main__": + main() diff --git a/tests/test_short_sleep_timer.py b/tests/test_short_sleep_timer.py new file mode 100644 index 0000000..6d06a24 --- /dev/null +++ b/tests/test_short_sleep_timer.py @@ -0,0 +1,436 @@ +"""The game's short sleeps wait on the timer on PCs with 6 or fewer logical processors (host/short_sleep.h, host/ +hos_thread.c process_threads, sleep_away, sleep_for and hos_sleep_logged; no game, no threads). + +(Triforce, 11-21, the performance audit's OS-31, after Source A's.) The game's idle job workers sleep 20 us and look +again, about 7,600 times a second. A sleep under a millisecond is yielded away until its time, so each sleeper keeps a +processor busy; with few processors that is time the frame's threads need. + +This compiles short_sleep.h alone and checks the rule: not set, the game's own short sleeps are on the timer with 1 to 6 +processors and yield with 7, 16 or an unknown count, and the host's own always yield; set on, every short sleep is on the +timer; set to 0, off, no or false, every one yields, whatever the processors; a sleep of a millisecond or more never +yields. + +And reads the source: the processors are the process's affinity mask and only with one processor group; the choice is +the rule's, of the switch's value (not its presence) and those processors, made once and logged; the game's SleepThread +goes through sleep_away as the game's own and the host's sleeps as the host's; the yield is the rule's; the switch is in +the switches table. + +(Triforce, 11-22, the performance audit's OS-32, after Source A's.) Off Windows the processors are not counted. Source +A puts the game's own short sleeps on the timer there at any count when the switch is not set; that mode was never +measured in the game, so here it is built and not the default: HOS_SHORT_SLEEP_TIMER=game asks for it on any system, +and short_sleep.h's SHORT_SLEEP_ELSEWHERE says what a build that is not Windows' does with the switch not set (all +yield, as before). Checked: the rule with the system (short_sleep_timer_os), with either value of that default; and +the lines of sleep_for that choose, cut out and compiled once as Windows' and once as another system's, over stand-ins +for the environment and the processors: the other system's build gives SHORT_SLEEP_ELSEWHERE when not set whatever +the processors, the game's own on the timer for "game", all on the timer for 1 and all yield for 0, and Windows' +build goes by the processors as before. + +(Triforce, 11-22, the performance audit's OS-33, after Source A's.) A cap on the game threads that yield through a short +sleep at once, HOS_SHORT_SLEEP_POLLERS, off unless set. The rule: not set, 0 or off no limit; a number that many; auto a +quarter of the processors above 6 (8: 2, 16: 4, 64: 16, not known: none); none with HOS_SHORT_SLEEP_TIMER set or where +the sleeps do not all yield. The gate, called in a scripted order: the cap's worth are let in, the next is sent to the +timer and its count given back, MainThread, the Presentation Thread and the host's sleeps are never counted, and the +count is 0 again once all have left. Three broken gates (no limit, a serial thread counted, a count not given back by +the one sent to the timer) must each fail those checks. + +These sleeps end by the clock: nothing wakes them, so there is no wake-up to lose. What the cap could break is a count +that is never given back (every later sleep then waits on the timer), more threads yielding than the cap, or a sleep +that does not return. So the yield branch is cut out of hos_thread.c's sleep_for and run on real threads: 16 game +threads, one named MainThread and one host thread each sleep 20 us thousands of times against a cap of 4; never more +than 4 counted threads yield at once, the serial and the host thread never take the timer, every sleep returns, the +timer's count is the sleeps sent there, and the count of yielding threads is 0 at the end. The same cut without its +leave must fail. + + python -I -B tests/test_short_sleep_timer.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +HOST = ROOT / "host" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +PROGRAM = r""" +#include +#include +#include "short_sleep.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +int main() { + const long long us20 = 20000, ms = 1000000; + // not set: by the processors + for (int t : {1, 2, 4, 6}) CHECK(short_sleep_timer(nullptr, t) == SHORT_SLEEP_GUEST_TIMER); + for (int t : {0, 7, 8, 16, 24, 64}) CHECK(short_sleep_timer(nullptr, t) == SHORT_SLEEP_YIELD); // (0: not known) + CHECK(SHORT_SLEEP_TIMER_THREADS == 6); + // set: on for all, or off for all, whatever the processors + for (int t : {0, 4, 16}) { + for (const char* on : {"1", "2", "on", "yes", "timer"}) CHECK(short_sleep_timer(on, t) == SHORT_SLEEP_ALL_TIMER); + for (const char* off : {"0", "off", "no", "false", ""}) CHECK(short_sleep_timer(off, t) == SHORT_SLEEP_YIELD); + } + // who yields: 4 processors, the game's 20 us on the timer and the host's yielded + const int few = short_sleep_timer(nullptr, 4), many = short_sleep_timer(nullptr, 16); + CHECK(!short_sleep_yields(us20, few, 1) && short_sleep_yields(us20, few, 0)); + CHECK(short_sleep_yields(us20, many, 1) && short_sleep_yields(us20, many, 0)); // (16: as before, both yield) + CHECK(!short_sleep_yields(us20, SHORT_SLEEP_ALL_TIMER, 1) && !short_sleep_yields(us20, SHORT_SLEEP_ALL_TIMER, 0)); + CHECK(short_sleep_yields(us20, SHORT_SLEEP_YIELD, 1) && short_sleep_yields(us20, SHORT_SLEEP_YIELD, 0)); + // a millisecond and more: the timer, always + for (int how : {SHORT_SLEEP_YIELD, SHORT_SLEEP_GUEST_TIMER, SHORT_SLEEP_ALL_TIMER}) + for (int guest : {0, 1}) CHECK(!short_sleep_yields(ms, how, guest) && !short_sleep_yields(5 * ms, how, guest)); + CHECK(short_sleep_yields(ms - 1, SHORT_SLEEP_YIELD, 1) && short_sleep_yields(ms - 1, SHORT_SLEEP_GUEST_TIMER, 0)); // (just under: still short) + // ((Triforce, 11-22, OS-32) the rule with the system. Windows: as short_sleep_timer, whatever the other systems' default + for (int elsewhere : {SHORT_SLEEP_YIELD, SHORT_SLEEP_GUEST_TIMER}) + for (int t : {0, 1, 4, 6, 7, 8, 16, 64}) { + CHECK(short_sleep_timer_os(nullptr, t, 1, elsewhere) == short_sleep_timer(nullptr, t)); + for (const char* v : {"1", "0", "off", "on", ""}) { + CHECK(short_sleep_timer_os(v, t, 1, elsewhere) == short_sleep_timer(v, t)); + CHECK(short_sleep_timer_os(v, t, 0, elsewhere) == short_sleep_timer(v, t)); // (a value set decides anywhere) + } + // not Windows, not set: the default given, at any processor count (they are not counted there: 0) + CHECK(short_sleep_timer_os(nullptr, t, 0, elsewhere) == elsewhere); + // "game": the game's own on the timer, the host's yield, on any system at any count + CHECK(short_sleep_timer_os("game", t, 0, elsewhere) == SHORT_SLEEP_GUEST_TIMER); + CHECK(short_sleep_timer_os("game", t, 1, elsewhere) == SHORT_SLEEP_GUEST_TIMER); + } + // Source A's default off Windows, the day the constant says so: the game's 20 us on the timer on a Steam Deck's 8 and on 24 + CHECK(!short_sleep_yields(us20, short_sleep_timer_os(nullptr, 0, 0, SHORT_SLEEP_GUEST_TIMER), 1)); + CHECK(short_sleep_yields(us20, short_sleep_timer_os(nullptr, 0, 0, SHORT_SLEEP_GUEST_TIMER), 0)); + // and what is built now: not the default (never measured in the game) + CHECK(SHORT_SLEEP_ELSEWHERE == SHORT_SLEEP_YIELD); + CHECK(short_sleep_yields(us20, short_sleep_timer_os(nullptr, 0, 0, SHORT_SLEEP_ELSEWHERE), 1)); + return fails ? 1 : 0; +} +""" + +# ((Triforce, 11-22, OS-32) sleep_for's lines that choose, compiled as Windows' build (T_WINDOWS defined) and as another +# system's: the source's own #if, its _WIN32 spelled T_WINDOWS so that a Windows compiler can build both +CHOICE = r""" +#include +#include +#include +#include "short_sleep.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +static const char* t_value; +static int t_threads; +static const char* t_getenv(const char* name) { + if (std::strcmp(name, "HOS_SHORT_SLEEP_TIMER")) std::printf("FAIL: getenv(%s)\n", name), fails++; + return t_value; +} +static int process_threads() { return t_threads; } +#define getenv t_getenv +static int choose(const char* value, int threads_there) { + t_value = value, t_threads = threads_there; + { // (the cut declares how: OS-33 keeps it with the cap in one word) +@CUT@ + return how; + } +} +int main() { +#ifdef T_WINDOWS + // Windows: by the processors, as before + CHECK(choose(nullptr, 4) == SHORT_SLEEP_GUEST_TIMER && choose(nullptr, 6) == SHORT_SLEEP_GUEST_TIMER); + CHECK(choose(nullptr, 8) == SHORT_SLEEP_YIELD && choose(nullptr, 16) == SHORT_SLEEP_YIELD && choose(nullptr, 0) == SHORT_SLEEP_YIELD); + CHECK(choose("game", 16) == SHORT_SLEEP_GUEST_TIMER && choose("1", 16) == SHORT_SLEEP_ALL_TIMER && choose("0", 4) == SHORT_SLEEP_YIELD); +#else + // another system: the processors are not known (0); not set gives the header's default at any count + for (int t : {0, 4, 8, 24}) CHECK(choose(nullptr, t) == SHORT_SLEEP_ELSEWHERE); + for (int t : {0, 4, 8, 24}) { + CHECK(choose("game", t) == SHORT_SLEEP_GUEST_TIMER); // (Source A's mode, asked for) + CHECK(choose("1", t) == SHORT_SLEEP_ALL_TIMER && choose("0", t) == SHORT_SLEEP_YIELD && choose("off", t) == SHORT_SLEEP_YIELD); + } +#endif + return fails ? 1 : 0; +} +""" + + +# ((Triforce, OS-33) the cap's rule and its gate in a scripted order) +POLLERS = r""" +#include +#include +#include "short_sleep.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +int main() { + const int Y = SHORT_SLEEP_YIELD; + // off unless asked for + for (int t : {0, 4, 8, 16, 24, 64}) CHECK(short_sleep_pollers(nullptr, nullptr, Y, t) == 0); + for (const char* off : {"0", "off", "no", "false", ""}) CHECK(short_sleep_pollers(off, nullptr, Y, 16) == 0); + // auto: a quarter of the processors above 6 + CHECK(short_sleep_pollers("auto", nullptr, Y, 8) == 2 && short_sleep_pollers("auto", nullptr, Y, 16) == 4); + CHECK(short_sleep_pollers("auto", nullptr, Y, 24) == 6 && short_sleep_pollers("auto", nullptr, Y, 64) == 16); + CHECK(short_sleep_pollers("auto", nullptr, Y, 7) == 1 && short_sleep_pollers("auto", nullptr, Y, 12) == 3); + CHECK(short_sleep_pollers("auto", nullptr, Y, 0) == 0 && short_sleep_pollers("auto", nullptr, Y, 6) == 0); // (not known; few) + CHECK(short_sleep_pollers("on", nullptr, Y, 16) == 4 && short_sleep_pollers("yes", nullptr, Y, 16) == 4); + // a number: that many, whatever the processors, 1024 at most + CHECK(short_sleep_pollers("2", nullptr, Y, 16) == 2 && short_sleep_pollers("2", nullptr, Y, 0) == 2); + CHECK(short_sleep_pollers("1", nullptr, Y, 64) == 1 && short_sleep_pollers("12", nullptr, Y, 8) == 12); + CHECK(short_sleep_pollers("1024", nullptr, Y, 8) == 1024 && short_sleep_pollers("5000", nullptr, Y, 8) == 1024); + CHECK(short_sleep_pollers("99999999999999999999", nullptr, Y, 8) == 1024); + // any HOS_SHORT_SLEEP_TIMER value, or sleeps that do not all yield: no cap + for (const char* timer : {"0", "1", "off", ""}) CHECK(short_sleep_pollers("auto", timer, Y, 16) == 0 && short_sleep_pollers("2", timer, Y, 16) == 0); + CHECK(short_sleep_pollers("2", nullptr, SHORT_SLEEP_GUEST_TIMER, 4) == 0 && short_sleep_pollers("auto", nullptr, SHORT_SLEEP_ALL_TIMER, 16) == 0); + // the word + for (int how : {SHORT_SLEEP_YIELD, SHORT_SLEEP_GUEST_TIMER, SHORT_SLEEP_ALL_TIMER}) + for (int cap : {0, 1, 4, 16, 1024}) { + const int w = short_sleep_word(how, cap); + CHECK(w >= 0 && short_sleep_how(w) == how && short_sleep_cap(w) == cap); + } + CHECK(short_sleep_word(SHORT_SLEEP_YIELD, 0) == SHORT_SLEEP_YIELD); // (no cap: the word is what it was) + // who is never counted + CHECK(short_sleep_serial_name("MainThread") && short_sleep_serial_name("Presentation Thread") && short_sleep_serial_name("sead::MainThread")); + CHECK(!short_sleep_serial_name("WorkerThread0") && !short_sleep_serial_name("Presentation Thread 2") && !short_sleep_serial_name("")); + CHECK(!short_sleep_serial_name("Thread_80012340") && !short_sleep_serial_name(nullptr)); + // the gate, cap 2: two in, the third to the timer with its count given back + int n = 0, a = 9, b = 9, c = 9, s = 9, h = 9; + CHECK(short_sleep_poll_enter(&n, 2, 1, 0, &a) == 1 && a == 1 && n == 1); + CHECK(short_sleep_poll_enter(&n, 2, 1, 0, &b) == 1 && b == 1 && n == 2); + CHECK(short_sleep_poll_enter(&n, 2, 1, 0, &c) == 0 && c == 0 && n == 2); + CHECK(short_sleep_poll_enter(&n, 2, 1, 1, &s) == 1 && s == 0 && n == 2); // (a serial thread: yields, not counted, cap full or not) + CHECK(short_sleep_poll_enter(&n, 2, 0, 0, &h) == 1 && h == 0 && n == 2); // (the host's own sleep: the same) + short_sleep_poll_leave(&n, c), short_sleep_poll_leave(&n, s), short_sleep_poll_leave(&n, h); + CHECK(n == 2); // (who took nothing gives nothing back) + short_sleep_poll_leave(&n, a); + CHECK(n == 1 && short_sleep_poll_enter(&n, 2, 1, 0, &c) == 1 && c == 1 && n == 2); // (a place freed: the next is let in) + short_sleep_poll_leave(&n, b), short_sleep_poll_leave(&n, c); + CHECK(n == 0); + for (int i = 0; i < 100; i++) CHECK(short_sleep_poll_enter(&n, 0, 1, 0, &a) == 1 && a == 0 && n == 0); // (no cap: nothing counted) + for (int i = 0; i < 100; i++) { // (sent to the timer again and again: nothing leaks) + CHECK(short_sleep_poll_enter(&n, 1, 1, 0, &a) == 1 && short_sleep_poll_enter(&n, 1, 1, 0, &b) == 0); + short_sleep_poll_leave(&n, a), short_sleep_poll_leave(&n, b); + } + CHECK(n == 0); + return fails ? 1 : 0; +} +""" + +# (each: what the broken gate does, the header's text, what replaces it) +GATE_MUTANTS = [ + ("no limit", "if (__atomic_add_fetch(pollers, 1, __ATOMIC_RELAXED) <= cap) {", "if (__atomic_add_fetch(pollers, 1, __ATOMIC_RELAXED) > 0) {"), + ("a serial thread counted", "if (cap <= 0 || !guest || serial) return 1;", "if (cap <= 0 || !guest) return 1;"), + ("the host's sleeps counted", "if (cap <= 0 || !guest || serial) return 1;", "if (cap <= 0 || serial) return 1;"), + ("a count not given back by the one sent to the timer", " __atomic_sub_fetch(pollers, 1, __ATOMIC_RELAXED);\n return 0;", " return 0;"), +] + +# ((Triforce, OS-33) hos_thread.c's yield branch on real threads. The stand-ins: the clock is real, a yield is a real yield and +# notes that this thread is yielding, the timer is only counted) +STORM = r""" +#include +#include +#include +#include +#include +#include +#include "short_sleep.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +struct HosThread { int serial; }; +static thread_local HosThread* cur; +static int hos_thread_serial(HosThread* t) { return t->serial; } +static int short_pollers; +static int64_t short_capped; +static std::atomic inside{0}, most{0}; +// in_yield: 0 this sleep has not yielded yet, 1 it is yielding, 2 its time came (the look at the clock that ends the loop, which +// is before the branch gives its count back: a thread is "yielding" here for no longer than the branch holds its count) +static thread_local int in_yield, counted; +static thread_local int64_t until_now; +static int64_t clock_ns() { + return std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); +} +static int64_t hos_now_ns() { + const int64_t now = clock_ns(); + if (in_yield == 1 && now >= until_now) { + in_yield = 2; + if (counted) --inside; + } + return now; +} +static int SwitchToThread() { + if (!in_yield) { + in_yield = 1; + if (counted) { + const int now = ++inside; + int was = most.load(); + while (now > was && !most.compare_exchange_weak(was, now)) {} + } + } + std::this_thread::yield(); + return 1; +} +static void YieldProcessor() {} +// 1: the sleep yielded to its time; 0: it was sent to the timer +static thread_local int fell; +static void sleep_short(int64_t ns, int guest, int word) { + const int64_t until = clock_ns() + ns; + until_now = until, fell = 0; +/* CUT */ + fell = 1; +} +struct Tally { long yielded = 0, timer = 0; }; +static void sleeper(int serial, int guest, int word, int rounds, Tally* out) { + HosThread me{serial}; + cur = guest ? &me : nullptr; + counted = guest && !serial; + for (int i = 0; i < rounds; i++) { + in_yield = 0; + sleep_short(20000, guest, word); + CHECK(in_yield != 1); // (a sleep that yielded went on to its time) + fell ? out->timer++ : out->yielded++; + } +} +static void storm(int cap, int rounds) { + const int word = short_sleep_word(SHORT_SLEEP_YIELD, cap); + Tally game[16], serial, host; + std::vector threads; + short_pollers = 0, short_capped = 0, inside = 0, most = 0; + for (int i = 0; i < 16; i++) threads.emplace_back(sleeper, 0, 1, word, rounds, &game[i]); + threads.emplace_back(sleeper, 1, 1, word, rounds, &serial); + threads.emplace_back(sleeper, 0, 0, word, rounds, &host); + for (auto& t : threads) t.join(); // (every sleep returned) + long yielded = 0, timer = 0; + for (const Tally& g : game) yielded += g.yielded, timer += g.timer; + CHECK(yielded + timer == 16L * rounds); + CHECK(serial.timer == 0 && serial.yielded == rounds); // (the frame's threads always yield) + CHECK(host.timer == 0 && host.yielded == rounds); // (and the host's own sleeps) + CHECK(short_pollers == 0); // (every count given back) + CHECK(short_capped == timer); // (the log's count is the sleeps sent to the timer) + CHECK(inside.load() == 0); + if (cap) CHECK(most.load() <= cap); // (never more than the cap's worth yielding at once) + else CHECK(timer == 0); // (no cap: none sent to the timer, as before) + std::printf("cap %d: %ld sleeps yielded, %ld took the timer, at most %d game threads yielding at once\n", cap, yielded, timer, most.load()); +} +int main() { + storm(4, 2000); + storm(1, 500); + storm(0, 300); + return fails ? 1 : 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + thread = (HOST / "hos_thread.c").read_text(encoding="utf-8") + + check('#include "short_sleep.h"' in thread, "hos_thread.c does not use the rule") + procs = block(thread, "static int process_threads(void) {") + check("GetActiveProcessorGroupCount() == 1 && GetProcessAffinityMask(GetCurrentProcess(), &proc, &sys)" in procs and + "threads = __builtin_popcountll((uint64_t)proc);" in procs and "#if defined(_WIN32)" in procs, + "the processors are not the process's affinity mask, or are counted with more than one processor group") + body = block(thread, "static void sleep_for(int64_t ns, int guest) {") + # ((Triforce, 11-22, OS-32) the rule with the system: Windows by the processors, any other build by the header's default + first, last = " const int threads = process_threads();\n", \ + ' const int how = short_sleep_timer_os(getenv("HOS_SHORT_SLEEP_TIMER"), threads, windows, SHORT_SLEEP_ELSEWHERE);\n' + check(body.count(first) == 1 and body.count(last) == 1 and body.index(first) < body.index(last), + "the choice is not the rule's of the switch's value, the process's processors and the system") + choice = body[body.index(first):body.index(last) + len(last)] + check("#if defined(_WIN32)\n const int windows = 1;\n#else\n const int windows = 0;\n#endif\n" in choice, + "sleep_for does not tell the rule whether this is Windows' build") + check("short_sleep_timer(getenv(" not in thread, "sleep_for still chooses without the system: off Windows the switch not " + "set could not be given a default of its own") + check('getenv("HOS_SHORT_SLEEP_TIMER") != NULL' not in thread, "the switch is still read by its presence: =0 would turn the timer on") + check("if (InterlockedCompareExchange(&how_now, word, -1) == -1) {" in body and 'hos_log("sleep: sleeps under 1 ms: %s' in body, + "the choice is not made once, or leaves no line") + # ((Triforce, OS-33) the cap: the rule's, in the same word, and its gate around the yielding) + check('const int cap = short_sleep_pollers(getenv("HOS_SHORT_SLEEP_POLLERS"), getenv("HOS_SHORT_SLEEP_TIMER"), how, threads);' in body and + "word = short_sleep_word(how, cap);" in body and "const int how = short_sleep_how(word);" in body, + "the cap is not the rule's of its switch, the timer's switch and the processors, kept with the choice") + start, end = body.index(" const int how = short_sleep_how(word);"), body.index(" if (!tried) {") + cut = body[start:end] + check("if (short_sleep_poll_enter(&short_pollers, short_sleep_cap(word), guest, cur ? hos_thread_serial(cur) : 0, &took)) {" in cut and + cut.index("short_sleep_poll_enter(") < cut.index("while (hos_now_ns() < until)") < cut.index("short_sleep_poll_leave(&short_pollers, took);") + < cut.index("return;") < cut.index("__atomic_add_fetch(&short_capped, 1, __ATOMIC_RELAXED);"), + "the yielding is not inside the cap's gate, or a sleeper over the cap does not go on to the timer") + check("int serial;" in block(thread, "struct HosThread {") and + "__atomic_store_n(&t->serial, short_sleep_serial_name(t->name), __ATOMIC_RELAXED);" in block(thread, "void hos_thread_set_name(HosThread *t, const char *name) {") and + "__atomic_store_n(&t->serial, short_sleep_serial_name(t->name), __ATOMIC_RELAXED);" in block(thread, "Ctx *hos_boot_main_thread(uint64_t stack_size) {"), + "a thread is not marked as one the frame waits on where it is named") + check("if (short_sleep_yields(ns, how, guest)) {" in body, "the yield is not the rule's: the game's and the host's sleeps are not told apart") + away = block(thread, "static void sleep_away(int64_t ns, int guest) {") + check("sleep_for(ns, guest);" in away and away.index("hos_vcore_away()") < away.index("sleep_for(ns, guest);") < away.index("hos_vcore_back()"), + "sleep_away does not pass on whose sleep it is, or keeps the core") + check("void hos_sleep_ns(int64_t ns) { sleep_away(ns, 0); }" in thread and "static void sleep_ns(int64_t ns) { sleep_for(ns, 0); }" in thread, + "the host's own sleeps are not the host's (they would wait on the timer on a small PC)") + logged = block(thread, "void hos_sleep_logged(int64_t ns, uint64_t lr) {") + check(logged.count("sleep_away(ns, 1);") == 2 and "hos_sleep_ns(ns)" not in logged, + "the game's SleepThread is not the game's own sleep on both of its paths") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`HOS_SHORT_SLEEP_TIMER=1`, `=0`" in dev, "docs/development.md: HOS_SHORT_SLEEP_TIMER is not in the switches table") + check("| `HOS_SHORT_SLEEP_POLLERS=`, `=auto` |" in dev, "docs/development.md: HOS_SHORT_SLEEP_POLLERS is not in the switches table") + + check("`HOS_SHORT_SLEEP_TIMER=game`" in dev and "`SHORT_SLEEP_ELSEWHERE`" in dev, + "docs/development.md: the HOS_SHORT_SLEEP_TIMER row does not say what =game does, or what other systems do when it is not set") + + with tempfile.TemporaryDirectory(prefix="short-sleep-") as tmp: + src, exe = Path(tmp) / "sleep.cpp", Path(tmp) / "sleep.exe" + src.write_text(PROGRAM, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(HOST), str(src), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + check(r.returncode == 0, "the short sleeps' rule as made:\n" + r.stdout) + # ((Triforce, 11-22, OS-32) the choosing lines themselves, as Windows' build and as another system's + src.write_text(CHOICE.replace("@CUT@", choice.replace("_WIN32", "T_WINDOWS")), encoding="utf-8") + for system, define in (("Windows", ["-DT_WINDOWS"]), ("another system", [])): + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(HOST)] + define + + [str(src), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + check(r.returncode == 0, "sleep_for's choice built for %s:\n%s" % (system, r.stdout)) + + # ((Triforce, OS-33) the cap's rule and gate; then each broken gate, a changed copy of the header, must fail them) + def pollers(name, include): + src, exe = Path(tmp) / (name + ".cpp"), Path(tmp) / (name + ".exe") + src.write_text(POLLERS, encoding="utf-8") + # (a broken gate leaves a parameter unused: not what it is to fail for) + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-Wno-unused-parameter", "-I" + str(include), + str(src), "-o", str(exe)], check=True) + return subprocess.run([str(exe)], capture_output=True, text=True) + r = pollers("pollers", HOST) + check(r.returncode == 0, "the cap on yielding sleepers as made:\n" + r.stdout) + header = (HOST / "short_sleep.h").read_text(encoding="utf-8") + for i, (what, old, new) in enumerate(GATE_MUTANTS): + check(header.count(old) == 1, "short_sleep.h: %r is not there once (update the test with the header)" % old) + folder = Path(tmp) / ("gate%d" % i) + folder.mkdir() + (folder / "short_sleep.h").write_text(header.replace(old, new), encoding="utf-8") + r = pollers("gate%d" % i, folder) + check(r.returncode != 0, "a gate with %s still passed the checks" % what) + + def storm(name, text): + src, exe = Path(tmp) / (name + ".cpp"), Path(tmp) / (name + ".exe") + src.write_text(STORM.replace("/* CUT */", text), encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O2", "-Wall", "-Wextra", "-Werror", "-Wno-unused-function", "-pthread", + "-I" + str(HOST), str(src), "-o", str(exe)], check=True) + return subprocess.run([str(exe)], capture_output=True, text=True, timeout=300) + r = storm("storm", cut) + check(r.returncode == 0, "hos_thread.c's yield branch on threads:\n" + r.stdout) + print(r.stdout.strip()) + leave = " short_sleep_poll_leave(&short_pollers, took);\n" + check(cut.count(leave) == 1, "the leave is not in the cut once") + r = storm("storm-leak", cut.replace(leave, " (void)took;\n")) + check(r.returncode != 0 and "short_pollers == 0" in r.stdout, "a yield branch that never gives its count back still passed") + print("PASS: the game's short sleeps wait on the timer with 6 or fewer processors, the host's yield, and the switch reads its value; " + "off Windows, not set: the header's default (all yield), and =game puts the game's own on the timer anywhere") + print("PASS: the cap on yielding sleepers is off unless set, lets the cap's worth in and sends the next to the timer, on threads too") + + +if __name__ == "__main__": + main() diff --git a/tests/test_sim_cores.py b/tests/test_sim_cores.py new file mode 100644 index 0000000..255259a --- /dev/null +++ b/tests/test_sim_cores.py @@ -0,0 +1,310 @@ +"""The lab's low-end PC simulation (the performance audit's OS-49, after Source B's; no game, no GPU): HOS_SIM_CPU holds the +process to fewer cores (host/hos_sim_cpu.h) and the renderer sizes its pools by its threads (gpu/vk/cpu_count.h); +RX_VK_SIM_GPU_SLOW adds GPU time in proportion to the work measured (gpu/vk/sim_gpu.h, vk_replay.cpp sim_burn). + +The rules, compiled and run: the switch's forms and the values refused; the mask on a 12-core, 24-thread map (6/12 takes the +last six), the floor of one core left out, the die with the largest L3 first, the fastest class first on a hybrid CPU, and the +core kept for the replay (host/hos_thread.c pick_reserved_core, cut out and run) inside every mask; the threads the +renderer takes, and cpu_count() under the variable; the factor's forms, the debt's rule and bounds, the plan net of the burns +in flight, the loop count and its calibration, and a run of the rule that reaches the factor asked within 10%. Each rule +taken out in turn (a mutation) fails. Then the sources: no hardware_concurrency() in gpu/ outside cpu_count.h and +vk_shader.cpp (a stamp file); the host applies the switch before any thread, filters the replay's core by it, sizes the +spinners by it and clears the variable when it does not apply it; the timestamps are two a recording without the switch and +three with it, the burn between the middle one and the end, between full barriers, in the replay's own recordings only, the +collection carrying what it measured; the shader's header was made from it; the docs rows. + + python -I -B tests/test_sim_cores.py [--cxx clang++] (the host's header is compiled as C by the same driver: -x c) +""" +import argparse +import os +import re +import subprocess +import tempfile +from pathlib import Path + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +HOST = ROOT / "host" + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def in_order(text, parts, where): + at = 0 + for part in parts: + found = text.find(part, at) + if found < 0: + raise SystemExit("%s: %r is missing, or not after the part before it (update the test with the code)" % (where, part)) + at = found + len(part) + + +HOST_TEST = r''' +#include +#include +#include "hos_sim_cpu.h" +#define CHECK(x) do { if (!(x)) { fprintf(stderr, "FAIL:%d %s\n", __LINE__, #x); exit(1); } } while (0) +%PICK% +static int ncores; +static uint64_t masks[64]; +static unsigned char classes[64]; +static void add(int n, int per, int first, unsigned char cls) { + for (int i = 0; i < n; i++) masks[ncores] = ((1ull << per) - 1) << (first + i * per), classes[ncores++] = cls; +} +int main(void) { + HosSimCpu s = {0, 0}; + CHECK(hos_sim_cpu_parse("6/12", &s) && s.cores == 6 && s.threads == 12); + CHECK(hos_sim_cpu_parse("4", &s) && s.cores == 4 && s.threads == 4); + CHECK(hos_sim_cpu_parse("1/1", &s) && s.cores == 1 && s.threads == 1); + CHECK(!hos_sim_cpu_parse(NULL, &s) && !hos_sim_cpu_parse("", &s) && !hos_sim_cpu_parse("x", &s) && !hos_sim_cpu_parse("6/", &s)); + CHECK(!hos_sim_cpu_parse("6/4", &s) && !hos_sim_cpu_parse("0/2", &s) && !hos_sim_cpu_parse("6/12x", &s) && !hos_sim_cpu_parse("300", &s)); + CHECK(!hos_sim_cpu_parse("-2/4", &s) && !hos_sim_cpu_parse("6/12/24", &s)); + /* (a 12-core, 24-thread processor of one kind: 6/12 takes the last six cores, and the replay's core is the last) */ + ncores = 0, add(12, 2, 0, 0); + uint64_t m = hos_sim_cpu_mask(masks, classes, ncores, 0, 6); + CHECK(m == 0xFFF000ull && hos_sim_cpu_count(m) == 12); + CHECK(pick_reserved_core(masks, classes, ncores) & m); + /* (one core always stays out; 0 and more than there are: not possible) */ + CHECK(hos_sim_cpu_mask(masks, classes, ncores, 0, 12) == 0 && hos_sim_cpu_mask(masks, classes, ncores, 0, 0) == 0); + CHECK(hos_sim_cpu_mask(masks, classes, ncores, 0, 11) == 0xFFFFFCull && hos_sim_cpu_mask(masks, classes, ncores, 0, 1) == 0xC00000ull); + /* (a two-die X3D part: the die with the largest L3 (the first eight cores here) first, its last cores) */ + ncores = 0, add(16, 2, 0, 0); + const uint64_t die = 0xFFFFull; + m = hos_sim_cpu_mask(masks, classes, ncores, die, 6); + CHECK(m == 0xFFF0ull && (m & ~die) == 0); + m = hos_sim_cpu_mask(masks, classes, ncores, die, 10); /* (more than the die has: its eight, then the last of the other) */ + CHECK((m & die) == die && hos_sim_cpu_count(m) == 20 && (m & 0xF0000000ull) == 0xF0000000ull); + /* (a hybrid processor: 8 performance cores with two threads, 16 efficiency cores after them: the performance ones) */ + ncores = 0, add(8, 2, 0, 1), add(16, 1, 16, 0); + m = hos_sim_cpu_mask(masks, classes, ncores, 0, 6); + CHECK(m == 0xFFF0ull && hos_sim_cpu_count(m) == 12); + CHECK(pick_reserved_core(masks, classes, ncores) == 0xC000ull && (pick_reserved_core(masks, classes, ncores) & m)); + m = hos_sim_cpu_mask(masks, classes, ncores, 0, 10); /* (all eight, then the last efficiency cores) */ + CHECK((m & 0xFFFFull) == 0xFFFFull && (m & 0xC0000000ull) == 0xC0000000ull && hos_sim_cpu_count(m) == 18); + puts("PASS host: the switch's forms, the mask (6/12 of 12/24, the one-core floor, the die first, the fastest class first), the replay's core inside"); + return 0; +} +''' + +VK_TEST = r''' +#include +#include +#include +#include "cpu_count.h" +#include "sim_gpu.h" +#define CHECK(x) do { if (!(x)) { std::fprintf(stderr, "FAIL:%d %s\n", __LINE__, #x); std::exit(1); } } while (0) +int main(int argc, char** argv) { + using cpu_count_rule::sim_threads; + CHECK(sim_threads("6/12", 24) == 12 && sim_threads("4", 24) == 4 && sim_threads("6/12", 0) == 12); + CHECK(sim_threads(nullptr, 24) == 0 && sim_threads("", 24) == 0 && sim_threads("6/4", 24) == 0 && sim_threads("x", 24) == 0); + CHECK(sim_threads("12/24", 24) == 0 && sim_threads("16/32", 24) == 0 && sim_threads("6/12z", 24) == 0); + // (cpu_count() as the variable says: the second argument is what this process was started with) + if (argc > 1) CHECK(cpu_count() == (unsigned)std::atoi(argv[1])); + using namespace sim_gpu; + CHECK(factor_of(nullptr) == 0 && factor_of("") == 0 && factor_of("1") == 0 && factor_of("0.5") == 0 && factor_of("11") == 0); + CHECK(factor_of("2") == 2.0 && factor_of("2.5") == 2.5 && factor_of("10") == 10.0 && factor_of("2x") == 0); + const int64_t ms = 1000000; + // (the debt: the work's (factor - 1) share added, what the burn took taken off, held between its bounds) + CHECK(held(owed(0, 6 * ms, 0, 2.0)) == 6 * ms && held(owed(6 * ms, 2 * ms, 5 * ms, 3.0)) == 5 * ms); + CHECK(held(owed(0, 200 * ms, 0, 2.0)) == kDebtMost && held(owed(0, 0, 90 * ms, 2.0)) == kDebtLeast); + // (the plan: owed less in flight, 1 to 16 ms) + CHECK(plan_ns(6 * ms, 0) == 6 * ms && plan_ns(6 * ms, 5500000) == 0 && plan_ns(40 * ms, 4 * ms) == 16 * ms && plan_ns(-5 * ms, 0) == 0); + CHECK(loops_for(0, 2.0) == 0 && loops_for(10 * ms, 2.0) == 5000000u && loops_for(1, 1e9) == 1u && loops_for(16 * ms, 0.0) > 0u); + CHECK(std::fabs(calibrated(4.0, 2 * ms, 1000000) - 3.6) < 1e-9 && calibrated(4.0, 0, 100) == 4.0 && calibrated(0, 300, 100) == 3.0); + // (a run: 4 recordings a frame of 1.6 ms of work each, factor 2, a loop truly 1.5 ns against the first guess of 4; + // the burns are planned at recording ends and measured 3 recordings later, as the ring does) + int64_t debt = 0, in_flight = 0, plan[3] = {0, 0, 0}; + uint32_t loops[3] = {0, 0, 0}; + double npl = kFirstNsPerLoop, work_all = 0, added_all = 0; + for (int r = 0; r < 4000; r++) { + const int s = r % 3; + if (r >= 3) { // (the slot's last use collected: its work and what its burn took) + const int64_t work = 1600000, added = (int64_t)(loops[s] * 1.5); + in_flight -= plan[s]; + debt = held(owed(debt, work, added, 2.0)); + if (plan[s]) npl = calibrated(npl, added, loops[s]); + if (r > 1000) work_all += work, added_all += added; + } + const int64_t p = plan_ns(debt, in_flight); + plan[s] = p, loops[s] = loops_for(p, npl), in_flight += p; + } + const double reached = (work_all + added_all) / work_all; + if (!(reached > 1.8 && reached < 2.2) || std::fabs(npl - 1.5) > 0.15) { + std::fprintf(stderr, "FAIL: the run reached x%.3f, a loop %.3f ns\n", reached, npl); + return 1; + } + std::printf("PASS vk: the threads the renderer takes, cpu_count(), the factor, the debt and its bounds, the plan, the loops, " + "a run reaching x%.2f of x2\n", reached); + return 0; +} +''' + +HOST_MUTATIONS = [ + ("one core stays out", "if (n > 256 || want < 1 || want > n - 1) return 0;", "if (n > 256 || want < 1 || want > n) return 0;"), + ("the die first", " if (on_i != on_best) {\n if (on_i) best = i;\n continue;\n }\n", ""), + ("the fastest class first", " if (classes[i] != classes[best]) {\n if (classes[i] > classes[best]) best = i;\n continue;\n }\n", ""), + ("the last cores first", "if (masks[i] > masks[best]) best = i;", "if (masks[i] < masks[best]) best = i;"), + ("threads not under cores", "if (*end || c < 1 || c > 256 || t < c || t > 256) return 0;", "if (*end || c < 1 || c > 256 || t > 256) return 0;"), +] +VK_MUTATIONS = [ + ("cpu_count: not fewer, not taken", "return hw && (unsigned)t >= hw ? 0 : (unsigned)t;", "return (unsigned)t;"), + ("the factor over 1", "if (end == e || *end || !(f > 1.0) || f > 10.0) return 0;", "if (end == e || *end || f > 10.0) return 0;"), + ("the burn taken off the debt", "return (double)debt + (double)work_ns * (factor - 1.0) - (double)added_ns;", + "return (double)debt + (double)work_ns * (factor - 1.0);"), + ("the debt's bounds", "inline int64_t held(double owed_ns) { return (int64_t)std::clamp(owed_ns, (double)kDebtLeast, (double)kDebtMost); }", + "inline int64_t held(double owed_ns) { return (int64_t)owed_ns; }"), + ("the burns in flight", "const int64_t o = debt - in_flight;", "const int64_t o = debt + 0 * in_flight;"), + ("the calibration", "return ns_per_loop > 0 ? 0.8 * ns_per_loop + 0.2 * m : m;", "return ns_per_loop + 0 * m;"), +] + + +def check_source(): + # (no count of the PC's threads in gpu/ but through cpu_count(), and vk_shader.cpp's, a stamp file) + for p in sorted((ROOT / "gpu").rglob("*")): + if p.suffix not in (".cpp", ".h") or "third_party" in p.parts: + continue + if p.name in ("cpu_count.h", "vk_shader.cpp"): + continue + if "hardware_concurrency()" in p.read_text(encoding="utf-8", errors="replace"): + raise SystemExit("%s: counts the PC's threads with hardware_concurrency(), not cpu_count() (cpu_count.h)" % p.relative_to(ROOT)) + shader = (VK / "vk_shader.cpp").read_text(encoding="utf-8") + if "cpu_count" in shader: + raise SystemExit("vk_shader.cpp: cpu_count() there would move the shader cache's stamp") + replay = (VK / "vk_replay.cpp").read_text(encoding="utf-8") + draw = (VK / "vk_draw.cpp").read_text(encoding="utf-8") + for name, text in (("vk_replay.cpp", replay), ("vk_draw.cpp", draw)): + if '#include "cpu_count.h"' not in text: + raise SystemExit("%s does not include cpu_count.h" % name) + # (the host: before any thread, the replay's core inside, the spinners by it, the variable cleared when not applied) + main = (HOST / "hos_main.c").read_text(encoding="utf-8") + in_order(main, ["occlusion_switch();", "hos_sim_cpu_apply();", "const char *image = NULL;", "rx_prepare_shaders()"], + "hos_main.c: HOS_SIM_CPU applied before the game and the preparation start threads") + thread = (HOST / "hos_thread.c").read_text(encoding="utf-8") + in_order(block(thread, "static uint64_t last_fastest_core(int *cores) {"), + ["(!sim_cpu_mask || ((uint64_t)e->Processor.GroupMask[0].Mask & sim_cpu_mask))", "return pick_reserved_core(masks, classes, n);"], + "hos_thread.c last_fastest_core: the replay's core inside the mask") + apply = block(thread, "void hos_sim_cpu_apply(void) {") + in_order(apply, ["#if defined(_WIN32)", "if (hos_sim_cpu_parse(e, &s)) {", + "mask = hos_sim_cpu_mask(masks, classes, n, hos_x3d_cpus(), s.cores);", + "if (mask && !SetProcessAffinityMask(GetCurrentProcess(), (DWORD_PTR)mask)) mask = 0;", "if (!mask) {", + 'SetEnvironmentVariableA("HOS_SIM_CPU", NULL);', '_putenv("HOS_SIM_CPU=");', "return;", "sim_cpu_mask = mask;"], + "hos_thread.c hos_sim_cpu_apply") + if "vc_spin_most(sim_cpu_mask ? hos_sim_cpu_count(sim_cpu_mask) : (int)si.dwNumberOfProcessors)" not in block(thread, "static int vc_spin_max(void) {"): + raise SystemExit("hos_thread.c vc_spin_max: not sized by the mask's processors") + # (the GPU: two timestamps without the switch, three with it; the burn; the collection) + if "static const int n = sim_gpu_factor() ? 3 : 2;" not in block(replay, "int ts_count() {"): + raise SystemExit("vk_replay.cpp ts_count: not two a recording without the switch and three with it") + if 'sim_gpu::factor_of(getenv("RX_VK_SIM_GPU_SLOW"))' not in block(replay, "double sim_gpu_factor() {"): + raise SystemExit("vk_replay.cpp: RX_VK_SIM_GPU_SLOW is not read through sim_gpu::factor_of") + begin = block(replay, "void gpu_time_begin(Recording& r) {") + in_order(begin, ["qi.queryCount = ts_count() * kRecordings;", "qi.queryCount = 2 * kRecordings;", + "vkCmdResetQueryPool(r.cmd, ts_pool, ts_count() * r.slot, ts_count());", + "vkCmdWriteTimestamp2(r.cmd, VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT, ts_pool, ts_count() * r.slot);"], "vk_replay.cpp gpu_time_begin") + end = block(replay, "void gpu_time_end(Recording& r, bool uploads) {") + in_order(end, ["if (!ts_pool) return;", "if (ts_count() == 3) sim_burn(r);", + "vkCmdWriteTimestamp2(r.cmd, VK_PIPELINE_STAGE_2_BOTTOM_OF_PIPE_BIT, ts_pool, ts_count() * r.slot + ts_count() - 1);"], + "vk_replay.cpp gpu_time_end: the burn before the end's timestamp") + burn = block(replay, "void sim_burn(Recording& r) {") + in_order(burn, ["vkCmdWriteTimestamp2(r.cmd, VK_PIPELINE_STAGE_2_BOTTOM_OF_PIPE_BIT, ts_pool, 3 * r.slot + 1);", + "if (r.segment || !sim_ready()) return;", "const int64_t ns = sim_gpu::plan_ns(slow.debt, slow.in_flight);", + "const uint32_t loops = sim_gpu::loops_for(ns, slow.ns_per_loop);", "end_pass(r.cmd);", + "full_barrier(r.cmd,", "vkCmdDispatch(r.cmd, 1, 1, 1);", "full_barrier(r.cmd,", "slow.in_flight += ns"], + "vk_replay.cpp sim_burn") + collect = block(replay, "void sim_collect(const Recording& r, const uint64_t* t) {") + in_order(collect, ["slow.in_flight -= plan", "if (!t || t[1] < t[0] || t[2] < t[1]) return;", + "const double o = sim_gpu::owed(slow.debt, work, added, sim_gpu_factor());", "slow.debt = sim_gpu::held(o);", + "if (plan) slow.ns_per_loop = sim_gpu::calibrated(slow.ns_per_loop, added, slow.loops[r.slot]);", + "std::chrono::seconds(10)"], "vk_replay.cpp sim_collect") + gather = block(replay, "void gpu_time_collect(Recording& r) {") + in_order(gather, ["const int tn = ts_count();", "vkGetQueryPoolResults(dev.device, ts_pool, tn * r.slot, tn, sizeof t, t, 8, VK_QUERY_RESULT_64_BIT)", + "if (tn == 3) sim_collect(r, got ? t : nullptr);", "const int64_t ns = (int64_t)((t[tn - 1] - t[0])"], + "vk_replay.cpp gpu_time_collect: the GPU time from the start to the end, the burn inside it") + # (the shader's header made from it) + comp = (VK / "shaders" / "sim_gpu_burn.comp").read_text(encoding="utf-8") + spv = (VK / "shaders" / "sim_gpu_burn_comp_spv.h").read_text(encoding="utf-8") + if "--vn SIM_GPU_BURN_COMP_SPV -o sim_gpu_burn_comp_spv.h sim_gpu_burn.comp" not in comp or "const uint32_t SIM_GPU_BURN_COMP_SPV[] = {" not in spv \ + or "0x07230203" not in spv: + raise SystemExit("gpu/vk/shaders: the burn's SPIR-V header is not the one sim_gpu_burn.comp names") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + for name in ("HOS_SIM_CPU", "RX_VK_SIM_GPU_SLOW"): + if name not in dev: + raise SystemExit("docs/development.md: %s is not in the switches table" % name) + + +def build(tool, d, name, src, includes, std, ext): + path, exe = d / (name + ext), d / (name + ".exe") + path.write_text(src, encoding="utf-8") + cmd = [tool] + (["-x", "c"] if ext == ".c" else []) + [std, "-O1", "-Wall", "-Wextra", "-Werror", "-Wno-unused-function"] + ["-I" + str(i) for i in includes] + [str(path), "-o", str(exe)] + r = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + return (exe, "") if r.returncode == 0 else (None, r.stdout) + + +def run(exe, args=(), env=None): + return subprocess.run([str(exe)] + list(args), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=120, env=env) + + +def mutate(d, tool, label, src, header_dir, header_name, mutations, std, ext, extra_inc): + text = (header_dir / header_name).read_text(encoding="utf-8") + for i, (what, old, new) in enumerate(mutations): + if text.count(old) != 1: + raise SystemExit("mutation %r: its text is not once in %s (update the test with the header)" % (what, header_name)) + inc = d / ("%s-mutant%d" % (label, i)) + inc.mkdir() + (inc / header_name).write_text(text.replace(old, new), encoding="utf-8") + exe, _ = build(tool, d, "%s-mutant%d" % (label, i), src, [inc] + extra_inc, std, ext) + if exe and run(exe).returncode == 0: + raise SystemExit("mutation %r: the test still passed without it" % what) + return len(mutations) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--cxx", default=os.environ.get("CXX", "g++")) + a = ap.parse_args() + check_source() + print("PASS sources: one thread count in gpu/, the host's switch before any thread with the replay's core and the spinners " + "inside it, two timestamps without the GPU switch and three with it, the burn and its collection, the shader, the rows") + pick = block((HOST / "hos_thread.c").read_text(encoding="utf-8"), "static uint64_t pick_reserved_core(") + host_src = HOST_TEST.replace("%PICK%", pick) + with tempfile.TemporaryDirectory(prefix="sim-cores-") as tmp: + d = Path(tmp) + exe, out = build(a.cxx, d, "host", host_src, [HOST], "-std=gnu11", ".c") + if not exe: + raise SystemExit("host: the build failed:\n" + out) + r = run(exe) + if r.returncode: + raise SystemExit("the host test failed:\n" + r.stdout) + print(r.stdout.strip()) + exe, out = build(a.cxx, d, "vk", VK_TEST, [VK], "-std=c++20", ".cpp") + if not exe: + raise SystemExit("vk: the build failed:\n" + out) + r = run(exe) + if r.returncode: + raise SystemExit("the vk test failed:\n" + r.stdout) + print(r.stdout.strip()) + if os.name == "nt": # (cpu_count() reads the variable on Windows alone, as the host applies it there alone) + env = dict(os.environ, HOS_SIM_CPU="1/1") + if run(exe, ["1"], env).returncode: + raise SystemExit("cpu_count(): HOS_SIM_CPU=1/1 did not give 1") + env = dict(os.environ) + env.pop("HOS_SIM_CPU", None) + if run(exe, [str(os.cpu_count() or 0)], env).returncode: + raise SystemExit("cpu_count(): without HOS_SIM_CPU it is not the PC's count") + print("PASS cpu_count(): 1 under HOS_SIM_CPU=1/1, the PC's %d without it" % (os.cpu_count() or 0)) + n = mutate(d, a.cxx, "host", host_src, HOST, "hos_sim_cpu.h", HOST_MUTATIONS, "-std=gnu11", ".c", []) + n += mutate(d, a.cxx, "vkc", VK_TEST, VK, "cpu_count.h", VK_MUTATIONS[:1], "-std=c++20", ".cpp", [VK]) + n += mutate(d, a.cxx, "vkg", VK_TEST, VK, "sim_gpu.h", VK_MUTATIONS[1:], "-std=c++20", ".cpp", [VK]) + print("PASS mutations: %d rules, each taken out, fail their tests" % n) + + +if __name__ == "__main__": + main() diff --git a/tests/test_slot_reset_ahead.py b/tests/test_slot_reset_ahead.py new file mode 100644 index 0000000..0f8399a --- /dev/null +++ b/tests/test_slot_reset_ahead.py @@ -0,0 +1,137 @@ +"""A recording slot's descriptor pools are reset before the recording's first packet (gpu/vk/vk_draw.cpp +slot_reset_locked, draw_slot_reset, draw_begin_recording; vk_replay.cpp begin_recording; no game, no GPU). + +(Triforce, 11-21, the performance audit's REPLAY-04, after Source A's.) With the recorder thread a descriptor pool's +reset, a pool's destruction and a buffer's are boundaries: the replay waits there until the recorder has run every packet +before them. Made in draw_begin_recording, they came after the recording's first packets, so every recording began with +a wait. + +Reads the source for the order: begin_recording resets the slot right after its two command pool resets and before its +command buffers are begun and anything is recorded into them; draw_slot_reset makes the reset only for a slot that has a +pool, under ParLock, and marks it; draw_begin_recording makes a new slot's pool, skips a reset already made (and clears +the mark), and otherwise resets as before, so a helper thread's segment, which never comes through begin_recording, +still resets its slot; one function holds the reset itself; the switch turns the early reset off. + +And runs a model of a slot through recordings: with the reset ahead each recording resets its pool exactly once, before +its first packet; a segment's slot once, in draw_begin_recording; with the switch off once, after the packets. + + python -I -B tests/test_slot_reset_ahead.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +# The same order as the source, with the calls written down as they happen +MODEL = r""" +#include +#include +#include +struct Res { bool pool = false, reset_ahead = false; }; +static std::vector res; +static std::string seen; +static bool own_on = true; +static void slot_reset_locked(Res&) { seen += "R"; } +static void draw_slot_reset(int slot) { + if (!own_on) return; + if ((size_t)slot >= res.size()) return; + Res& r = res[slot]; + if (!r.pool) return; + slot_reset_locked(r); + r.reset_ahead = true; +} +static void draw_begin_recording(int slot) { + if ((size_t)slot >= res.size()) res.resize((size_t)slot + 1); + Res& r = res[slot]; + if (!r.pool) r.pool = true, seen += "M"; + else if (r.reset_ahead) r.reset_ahead = false; + else slot_reset_locked(r); +} +static void begin_recording(int slot) { + seen += "c"; // the command pools' resets + draw_slot_reset(slot); + seen += "p"; // the recording's first packets + draw_begin_recording(slot); + seen += "."; +} +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s (%s)\n", __LINE__, #x, seen.c_str()); fails++; } } while (0) +int main() { + begin_recording(0), begin_recording(0), begin_recording(0); + CHECK(seen == "cpM.cRp.cRp."); // (a new slot's pool made; then each reset once, before the packets) + seen.clear(); + draw_begin_recording(1), draw_begin_recording(1); // (a helper thread's segment: never through begin_recording) + CHECK(seen == "MR"); + seen.clear(); + begin_recording(1); // (the same slot later used by the replay thread itself) + CHECK(seen == "cRp." && !res[1].reset_ahead); + seen.clear(); + own_on = false; + begin_recording(0), begin_recording(0); + CHECK(seen == "cpR.cpR."); // (the switch: after the packets, as before) + return fails ? 1 : 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + draw = (VK / "vk_draw.cpp").read_text(encoding="utf-8") + replay = (VK / "vk_replay.cpp").read_text(encoding="utf-8") + + reset = block(draw, "void slot_reset_locked(RecordingRes& r) {") + check("vkResetDescriptorPool(dev.device, r.pool, 0);" in reset and "vkDestroyDescriptorPool(dev.device, p, nullptr);" in reset and + "destroy_buffer(r.streams[i].buffer, r.streams[i].alloc);" in reset, "the slot's reset is not all in one function") + check(draw.count("vkResetDescriptorPool(dev.device, r.pool, 0);") == 1, "a slot's pool is reset in a place this test does not know") + ahead = block(draw, "void draw_slot_reset(int slot) {") + at = [ahead.index(p) for p in ("if (!on) return;", "ParLock lk;", "if ((size_t)slot >= res.size()) return;", "if (!r.pool) return;", + "slot_reset_locked(r);", "r.reset_ahead = true;")] + check(at == sorted(at), "draw_slot_reset does not reset only a slot that has a pool, under ParLock, and mark it") + check('getenv("RX_VK_RECORDER_OWN") && atoi(getenv("RX_VK_RECORDER_OWN")) == 0' in ahead, "the switch is not RX_VK_RECORDER_OWN") + begin = block(draw, "void draw_begin_recording(int slot, VkCommandBuffer cmd, uint64_t value) {") + at = [begin.index(p) for p in ("if (!r.pool) {", "r.pool = make_pool(r.shift);", "} else if (r.reset_ahead) {", "r.reset_ahead = false;", + "} else {", "slot_reset_locked(r);")] + check(at == sorted(at), "draw_begin_recording does not skip a reset already made, or no longer resets a slot that did not have one") + rec = block(replay, "Recording& begin_recording(uint64_t value) {") + at = [rec.index(p) for p in ("vkResetCommandPool(dev.device, r.pool, 0);", "vkResetCommandPool(dev.device, r.upload_pool, 0);", + "draw_slot_reset(r.slot);", "vkBeginCommandBuffer(r.cmd, &bi);", "occ_slot_begin(r);", "gpu_time_begin(r);", + "draw_begin_recording(r.slot, r.cmd, value);")] + check(at == sorted(at), "the slot is not reset right after its command pools' resets and before the recording's first packet") + check(replay.count("draw_slot_reset(") == 1, "a segment's start, or another place, resets a slot ahead") + check("void draw_slot_reset(int slot);" in (VK / "vk_draw.h").read_text(encoding="utf-8"), "vk_draw.h does not declare it") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`RX_VK_RECORDER_OWN=0`" in dev, "docs/development.md: RX_VK_RECORDER_OWN=0 is not in the switches table") + + with tempfile.TemporaryDirectory(prefix="slot-reset-") as tmp: + c, exe = Path(tmp) / "model.cpp", Path(tmp) / "model.exe" + c.write_text(MODEL, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", str(c), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + check(r.returncode == 0, "the order's model:\n" + r.stdout) + print("PASS: a recording slot's descriptor pools are reset before the recording's first packet, once a recording") + + +if __name__ == "__main__": + main() diff --git a/tests/test_slot_retire.py b/tests/test_slot_retire.py new file mode 100644 index 0000000..86779f9 --- /dev/null +++ b/tests/test_slot_retire.py @@ -0,0 +1,108 @@ +"""A depth target another size replaced at its start is let go 16 presents after its last use (gpu/vk/slot_retire.h, +vk_texture.cpp texture_sweep; no game, no GPU). + +(Triforce, 11-21, the performance audit's TEX-76, after Source B's.) The game draws each character's shadow map at a +fixed address, sized from the camera every frame; each size is a texture of its own, and the sizes before it, drawn +over, waited 120 presents unused before the sweep took them: a pile of dead maps at every slot. + +This compiles slot_retire.h alone and checks the rule: a 768 map replaced by a 1024 one and unused 16 presents is +retired, at 15 it is not; nor one replaced by the same size, a colour target, another format's, one of two levels or +layers or a 3D one, one the GPU never drew, one whose newer target was written before its last use; a wait of 0 retires +nothing; the switch's values. + +And reads the source: the sweep's two gates that hold a young texture back let a retired one through, and only those; +the rule is asked with the texture's own fields and the others at its start; a retired one is counted; the switch is in +the switches table. + + python -I -B tests/test_slot_retire.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +PROGRAM = r""" +#include +#include +#include "slot_retire.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +using namespace slot_retire; +int main() { + CHECK(kWait == 16); + CHECK(wait(nullptr) == 16 && wait("") == 16 && wait("16") == 16 && wait("4") == 4 && wait("120") == 120); + CHECK(wait("0") == 0 && wait("-1") == 0 && wait("off") == 0); + // a 768 map last used in present 1000, and the 1024 one drawn over its start in frame 1001 + const Target old{true, true, 1, 1, 1, 0x34, 768, 768, 1000, 1000}; + const Target newer{true, true, 1, 1, 1, 0x34, 1024, 1024, 1040, 1001}; + CHECK(retired_by(old, newer, 16, 16) && retired_by(old, newer, 40, 16)); + CHECK(!retired_by(old, newer, 15, 16) && !retired_by(old, newer, 0, 16)); + CHECK(!retired_by(old, newer, 1000, 0)); // (the switch at 0: never) + { Target n = newer; n.width = n.height = 768; CHECK(!retired_by(old, n, 16, 16)); } // (the same size: drawn into again) + { Target n = newer; n.height = 768; CHECK(retired_by(old, n, 16, 16)); } // (another width alone is another size) + { Target n = newer; n.nvn_format = 0x32; CHECK(!retired_by(old, n, 16, 16)); } // (another format at that start) + { Target n = newer; n.last_written = 1000; CHECK(!retired_by(old, n, 16, 16)); } // (not written after its last use) + { Target n = newer; n.last_written = 999; CHECK(!retired_by(old, n, 16, 16)); } + { Target o = old; o.depth = false; CHECK(!retired_by(o, newer, 16, 16)); } // (a colour target) + { Target o = old; o.gpu_written = false; CHECK(!retired_by(o, newer, 16, 16)); } // (the GPU never drew it) + { Target o = old; o.levels = 2; CHECK(!retired_by(o, newer, 16, 16)); } + { Target o = old; o.layers = 2; CHECK(!retired_by(o, newer, 16, 16)); } + { Target o = old; o.deep = 2; CHECK(!retired_by(o, newer, 16, 16)); } + { Target o = old; o.deep = 0; CHECK(retired_by(o, newer, 16, 16)); } // (0 or 1: a plain 2D one) + return fails ? 1 : 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + tex = (VK / "vk_texture.cpp").read_text(encoding="utf-8") + start = tex.index("void texture_sweep(") + start = tex.index("void texture_sweep(", start + 1) if tex.count("void texture_sweep(") > 1 else start + sweep = tex[start:] + check('#include "slot_retire.h"' in tex, "vk_texture.cpp does not use the rule") + check('static const uint64_t retire_wait = slot_retire::wait(getenv("RX_VK_SLOT_RETIRE"));' in sweep, "the switch is not the rule's") + check("slot_retire::Target{t->gpu_written, (t->aspect & VK_IMAGE_ASPECT_DEPTH_BIT) != 0, t->levels, t->layers, t->depth," in sweep and + "t->nvn_format, t->width, t->height, t->last_frame, t->last_written};" in sweep, + "the rule is not asked with the texture's own fields") + asks = sweep[sweep.index("const auto slot_retired = [&](const Texture* tex) {"):] + asks = asks[:asks.index(" };") + 6] + at = [asks.index(p) for p in ("if (!retire_wait || !tex->gpu_written || frame_no - tex->last_frame < retire_wait) return false;", + "auto [a, e] = by_gpu.equal_range(tex->gpu);", + "slot_retire::retired_by(old, slot_target(o->second), frame_no - tex->last_frame, retire_wait)")] + check(at == sorted(at), "a retire does not look at the other targets at the texture's start") + check("if (age60(frame_no - last) < need_unused && !shifted && !slot_retired(tex)) continue;" in sweep, + "the hold's first pass still holds a retired depth target back for the sweep's own wait") + check("const bool retired = young && !shifted && !gone_early && slot_retired(tex);" in sweep and + "if (young && !shifted && !gone_early && !retired) continue;" in sweep, + "the sweep's gate still holds a retired depth target back, or lets other young textures through") + gate = sweep.index("if (young && !shifted && !gone_early && !retired) continue;") + check(gate < sweep.index("const bool overwritten = shifted || drawn_over(tex) || stale;") < sweep.index(" if (sweep && !overwritten && !gone_early) continue;\n"), + "a retired target is no longer taken as drawn over: it would be written back, or kept by a sweep") + check("st_slot_retired" in sweep and "let go %llu presents after their last use" in sweep, "the retired targets are not counted in the log") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`RX_VK_SLOT_RETIRE=`" in dev, "docs/development.md: RX_VK_SLOT_RETIRE is not in the switches table") + + with tempfile.TemporaryDirectory(prefix="slot-retire-") as tmp: + c, exe = Path(tmp) / "retire.cpp", Path(tmp) / "retire.exe" + c.write_text(PROGRAM, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(VK), str(c), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + check(r.returncode == 0, "the rule as made:\n" + r.stdout) + print("PASS: a depth target another size replaced at its start is let go 16 presents after its last use") + + +if __name__ == "__main__": + main() diff --git a/tests/test_split_before_draw.py b/tests/test_split_before_draw.py new file mode 100644 index 0000000..efa096f --- /dev/null +++ b/tests/test_split_before_draw.py @@ -0,0 +1,137 @@ +"""A submission's part ends only right before a plain draw, on Intel's GPUs (gpu/vk/barrier_rules.h; no game, no GPU). + +(Triforce, 11-22, the performance audit's REPLAY-14, after Source A's.) On an Arc B580 a command buffer that begins with +mesh draws drew its later plain draws as exploded triangles. Source A ends a part only right before a plain draw; here +a helper thread's segment can begin with mesh draws too, so the dry pass makes its points the same way. + +Compiles a small driver against the header and checks: + split_before_draw unset: on for vendor 0x8086 (Intel), off for NVIDIA, AMD and any other; 0 off and 1 on with any + vendor; an empty value is unset; + cut_ok only a plain draw with rendering on and no rasterizer discard: not a mesh draw or a dispatch (not + plain), not with rendering off, not with the rasterizer discarding. + +And reads gpu/vk/vk_replay.cpp: the op loop's top asks cut_ok with the three plain draw ops and the state before the op, +on the replay's own thread only; maybe_split at the count sets part_end_wanted and returns where the rule is on, and +splits only when the loop's top calls it; the dry pass sets dry_point_wanted at its count where the rule is on and makes +the point at the plain draw with the state before it; both flags are cleared with a new submission or dry pass; and the +docs row. + + python -I -B tests/test_split_before_draw.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +DRIVER = r''' +#include +#include +#include "barrier_rules.h" +#include +#include + +using namespace mkwvk::barrier_rules; +static int checks, fails; +static void want(const char* what, bool got, bool wanted) { + checks++; + if (got != wanted) fails++, printf("FAIL %s: %d, want %d\n", what, (int)got, (int)wanted); +} + +int main() { + const uint32_t intel = 0x8086, nvidia = 0x10DE, amd = 0x1002, other = 0x5143; + want("unset, Intel", split_before_draw(nullptr, intel), true); + for (const uint32_t v : {nvidia, amd, other}) { + want("unset, another vendor", split_before_draw(nullptr, v), false); + want("empty is unset, another vendor", split_before_draw("", v), false); + want("1, another vendor", split_before_draw("1", v), true); + want("0, another vendor", split_before_draw("0", v), false); + } + want("empty is unset, Intel", split_before_draw("", intel), true); + want("0, Intel", split_before_draw("0", intel), false); + want("1, Intel", split_before_draw("1", intel), true); + // cut_ok(plain draw, rendering on, rasterizer discard) + want("a plain draw that draws", cut_ok(true, true, false), true); + want("a mesh draw or a dispatch", cut_ok(false, true, false), false); + want("a plain draw with rendering off", cut_ok(true, false, false), false); + want("a plain draw with the rasterizer discarding", cut_ok(true, true, true), false); + want("nothing at all", cut_ok(false, false, true), false); + if (fails) printf("split before draw: FAILED (%d of %d checks)\n", fails, checks); + else printf("split before draw: all %d checks passed\n", checks); + return fails != 0; +} +''' + + +def block(src, start): + at = src.index(start) + return src[at:src.index("\n}\n", at)] + + +def check_source(): + replay = (ROOT / "gpu/vk/vk_replay.cpp").read_text(encoding="utf-8") + loop = block(replay, "void replay(VkCommandBuffer& cmd, NvnState& s, const uint8_t* p, const uint8_t* end = nullptr) {") + top = (" if (!worker && (part_end_wanted || dry_point_wanted) &&\n" + " barrier_rules::cut_ok(h.op == NOP_DRAW || h.op == NOP_DRAW_IDX || h.op == NOP_DRAW_INDIRECT, " + "s.render_enabled, s.raster_discard)) {\n") + if top not in loop or loop.index(top) > loop.index("switch (h.op) {"): + raise SystemExit("replay: the op loop's top does not ask cut_ok with the three plain draws and the state before the op, " + "on the replay's own thread, before the op is replayed") + cut = loop[loop.index(top):loop.index("switch (h.op) {")] + order = [cut.find(n) for n in ( + "if (dry && dry_point_wanted) {", "dry_point_wanted = false;", + "if (!seg_early_hook || seg_early_hook->wants(dry_work, p)) {", + "par_points.push_back({p, s, ub_updates.size(), par_conds.size(), dry_work});", + "} else if (!dry && part_end_wanted) {", "part_end_now = true;", "maybe_split(cmd);", "part_end_now = false;")] + if min(order) < 0 or order != sorted(order): + raise SystemExit("replay: the cut before a plain draw does not make the dry pass's point there, or the serial replay's split") + count = (" if (++dry_work % par_point_every() == 0) {\n" + " // ((Triforce, 11-22, REPLAY-14) on Intel the point is made at the next plain draw: the loop's top)\n" + " if (split_before_draw_on()) dry_point_wanted = true;\n") + if count not in loop or "else if (!seg_early_hook || seg_early_hook->wants(dry_work, p + h.size)) {" not in loop: + raise SystemExit("replay: the dry pass's count does not hand its point to the next plain draw where the rule is on, and " + "make it at once where it is off") + if "if (!worker) maybe_split(cmd);" not in loop: + raise SystemExit("replay: the serial replay no longer counts its parts after each draw and dispatch") + split = block(replay, "void maybe_split(VkCommandBuffer& cmd) {") + order = [split.find(n) for n in ( + "if (tail_on()) return;", "if (part_end_now) {", "part_end_wanted = false;", + "if (!rec_cur || rec_parts + 1 >= kMaxParts || dump_on()) return;", "} else {", "if (part_end_wanted) return;", + "if (!rec_cur || every <= 0 || ++part_work < every || rec_parts + 1 >= kMaxParts || dump_on()) return;", + "if (split_before_draw_on()) {", "part_end_wanted = true;", "end_pass(cmd);", "submit_recording(*rec_cur, false, 0, true);", + "part_work = 0;")] + if min(order) < 0 or order != sorted(order): + raise SystemExit("maybe_split: at the count it does not hold the part's end for the next plain draw where the rule is on, " + "or the held cut does not go through the same split") + if replay.count("part_work = 0, part_end_wanted = false;") != 2: + raise SystemExit("a new submission does not clear part_end_wanted with part_work") + if replay.count("dry_work = 0, dry_point_wanted = false;") != 2: + raise SystemExit("a new dry pass does not clear dry_point_wanted with dry_work") + rules = block(replay, "bool split_rules(int which) {") + if 'barrier_rules::split_before_draw(getenv("RX_VK_SPLIT_BEFORE_DRAW"), dev.props.vendorID);' not in rules: + raise SystemExit("split_rules does not read RX_VK_SPLIT_BEFORE_DRAW with the vendor, once") + docs = (ROOT / "docs/development.md").read_text(encoding="utf-8") + if "| `RX_VK_SPLIT_BEFORE_DRAW=0` or `1` |" not in docs: + raise SystemExit("docs/development.md: the switches table has no row for RX_VK_SPLIT_BEFORE_DRAW") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="g++") + args = parser.parse_args() + check_source() + with tempfile.TemporaryDirectory(prefix="split-draw-") as tmp: + cpp, exe = Path(tmp) / "split_draw_test.cpp", Path(tmp) / "split-draw.exe" + cpp.write_text(DRIVER, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-Wall", "-Wextra", "-Werror", "-I" + str(ROOT / "gpu/vk"), str(cpp), "-o", str(exe)], + check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + print(r.stdout.rstrip()) + if r.returncode: + raise SystemExit("FAIL: the split-before-draw rule's checks failed") + + +if __name__ == "__main__": + main() diff --git a/tests/test_split_ub_flush.py b/tests/test_split_ub_flush.py new file mode 100644 index 0000000..45c5704 --- /dev/null +++ b/tests/test_split_ub_flush.py @@ -0,0 +1,107 @@ +"""No uniform updates written at a part's end on Intel's GPUs without mirrors (gpu/vk/barrier_rules.h; no game, no GPU). + +(Triforce, 11-22, the performance audit's REPLAY-14, after Source A's.) A split submission wrote its uniform updates +into the game's memory at every part's end. Without mirrors that memory is what later parts' other reads and the game's +CPU see, mid-submission, where an unsplit submission writes them once at its end: white blocks and flashes on Intel's +integrated GPUs. There only the submission's last part writes them now. + +Compiles a small driver against the header and checks the vendor and mirror table with the switch unset and forced: + unset Intel without mirrors: off; Intel with mirrors (an Arc card): on; NVIDIA and AMD, with or without mirrors: on; + 0 off with any vendor, mirrors or not; 1: on with any vendor, mirrors or not; empty: unset. + +And reads gpu/vk/vk_replay.cpp: maybe_split writes the updates and moves ub_first_of_recording only where the rule is on, +in the place they were; split_rules reads the switch once with the vendor and pools_mirrored(); a whole recording's +end still writes them without asking (the last part's write); and the docs row. + + python -I -B tests/test_split_ub_flush.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +DRIVER = r''' +#include +#include +#include "barrier_rules.h" +#include +#include + +using namespace mkwvk::barrier_rules; +static int checks, fails; +static void want(const char* what, bool got, bool wanted) { + checks++; + if (got != wanted) fails++, printf("FAIL %s: %d, want %d\n", what, (int)got, (int)wanted); +} + +int main() { + const uint32_t intel = 0x8086, nvidia = 0x10DE, amd = 0x1002; + want("unset, Intel without mirrors", split_ub_flush(nullptr, intel, false), false); + want("unset, Intel with mirrors", split_ub_flush(nullptr, intel, true), true); + want("empty is unset, Intel without mirrors", split_ub_flush("", intel, false), false); + for (const uint32_t v : {nvidia, amd}) + for (int mirrors = 0; mirrors < 2; mirrors++) { + want("unset, another vendor: as before", split_ub_flush(nullptr, v, mirrors != 0), true); + want("empty, another vendor: as before", split_ub_flush("", v, mirrors != 0), true); + } + for (const uint32_t v : {intel, nvidia, amd}) + for (int mirrors = 0; mirrors < 2; mirrors++) { + want("0: off with any vendor", split_ub_flush("0", v, mirrors != 0), false); + want("1: on with any vendor", split_ub_flush("1", v, mirrors != 0), true); + } + if (fails) printf("split ub flush: FAILED (%d of %d checks)\n", fails, checks); + else printf("split ub flush: all %d checks passed\n", checks); + return fails != 0; +} +''' + + +def block(src, start): + at = src.index(start) + return src[at:src.index("\n}\n", at)] + + +def check_source(): + replay = (ROOT / "gpu/vk/vk_replay.cpp").read_text(encoding="utf-8") + split = block(replay, "void maybe_split(VkCommandBuffer& cmd) {") + order = [split.find(n) for n in ( + "const uint64_t ended = work_between();", "if (split_ub_flush_on()) flush_ub_updates(cmd);", "texture_end_recording(cmd);", + "memory_end_submission(cmd);", "submit_recording(*rec_cur, false, 0, true);", + "if (split_ub_flush_on()) ub_first_of_recording = ub_updates.size();", "Recording& r = begin_recording(rec_cur_value);")] + if min(order) < 0 or order != sorted(order): + raise SystemExit("maybe_split does not write a part's uniform updates, and move its first one, only where the rule is on") + if split.count("flush_ub_updates(cmd);") != 1 or split.count("ub_first_of_recording = ub_updates.size();") != 1: + raise SystemExit("maybe_split writes the uniform updates, or moves their first, in a place that does not ask the rule") + rules = block(replay, "bool split_rules(int which) {") + if 'barrier_rules::split_ub_flush(getenv("RX_VK_SPLIT_UBFLUSH"), dev.props.vendorID, pools_mirrored());' not in rules or \ + "static const bool ub_flush = [] {" not in rules: + raise SystemExit("split_rules does not read RX_VK_SPLIT_UBFLUSH with the vendor and the mirrors, once") + # a whole recording's end writes them whatever the rule: the submission's last part + if replay.count(" flush_ub_updates(r.cmd);\n") < 2: + raise SystemExit("a recording's end no longer writes its uniform updates: with the rule off nothing would write them") + docs = (ROOT / "docs/development.md").read_text(encoding="utf-8") + if "| `RX_VK_SPLIT_UBFLUSH=0` or `1` |" not in docs: + raise SystemExit("docs/development.md: the switches table has no row for RX_VK_SPLIT_UBFLUSH") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="g++") + args = parser.parse_args() + check_source() + with tempfile.TemporaryDirectory(prefix="split-ub-") as tmp: + cpp, exe = Path(tmp) / "split_ub_test.cpp", Path(tmp) / "split-ub.exe" + cpp.write_text(DRIVER, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-Wall", "-Wextra", "-Werror", "-I" + str(ROOT / "gpu/vk"), str(cpp), "-o", str(exe)], + check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + print(r.stdout.rstrip()) + if r.returncode: + raise SystemExit("FAIL: the uniform flush rule's checks failed") + + +if __name__ == "__main__": + main() diff --git a/tests/test_staging_burst.py b/tests/test_staging_burst.py new file mode 100644 index 0000000..88b3bd9 --- /dev/null +++ b/tests/test_staging_burst.py @@ -0,0 +1,275 @@ +"""The idle staging arenas stay through a load's burst (gpu/vk/staging_keep.h, vk_texture.cpp staging and texture_frame; +no game, no GPU). + +(Triforce, 11-22, the performance audit's TEX-28, after Source A's.) An arena is taken again once the GPU is done with +the submission that read it, known a present later, so frames that stage in a row use two arenas in turn. Where 64 MB +are kept (the driver allows 6.5 GB or less) the one that had its turn was freed at every second present and made again +the present after. An idle arena staged into in the last 2 seconds may now stay while the arenas add up to 256 MB, +unless VRAM is over 90% of the allowance. + +This cuts the arenas, staging() and texture_frame's give-back loop out of vk_texture.cpp and runs them with stand-ins +(no memory is really taken), a process a case since the switch and what is kept are read once: + - a card allowed 6,000 MB, 20 frames that stage 32 MB each: 2 arenas made, not 11, and every make counted; 128 MB + kept through the burst; 64 MB again once 2 seconds have passed since the last staging; + - the same with RX_VK_STAGING_BURST_S=0, and with VRAM short (pool_pressed): 11 made, as before; + - one frame that stages 6 arenas' worth: the idle ones are trimmed to 256 MB at the next present, and an arena over + 64 MB goes at once, burst or not; + - a card allowed 31,000 MB (256 MB kept anyway), a card that reports nothing, and a 6,000 MB card with VRAM short: + 300 frames of mixed staging with rests give the same makes, frees and sizes present by present with the rule and + with =0. +And the header's rules by their values, and the source: the loop reads the switch once and asks the header, the +condition is the one it was but for the allowance, staging() counts every arena it makes, the memory line says so, the +switch is in the switches table. + + python -I -B tests/test_staging_burst.py [--cxx clang++] +""" +import argparse +import os +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +LOOP_HEAD = " static const uint64_t keep = []() -> uint64_t {\n" +LOOP_END = " if (frame_no % 600 == 0) { // (the process's memory now and then: what the host side holds)\n" + +FIXTURE = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include "staging_keep.h" + +using VkBuffer = void*; +using VmaAllocation = void*; +#define VK_NULL_HANDLE nullptr +enum { VK_BUFFER_USAGE_TRANSFER_SRC_BIT = 1, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT = 32 }; +static struct { struct { struct { uint64_t nonCoherentAtomSize = 64; } limits; } props; } dev; +static uint64_t rec_value = 0; +static bool pool_pressed = false; +static uint64_t card_mb = 0; +static int makes, frees; +static uint64_t live_bytes; +// (no memory is taken: nothing here writes through an arena's pointer) +static bool create_buffer(uint64_t size, int, bool, VkBuffer& b, VmaAllocation& a, void** m) { + static char mem[16]; + uint64_t* tag = new uint64_t(size); + b = tag, a = tag, *m = mem; + makes++, live_bytes += size; + return true; +} +static void destroy_buffer(VkBuffer, VmaAllocation a) { + frees++, live_bytes -= *(uint64_t*)a; + delete (uint64_t*)a; +} +static bool vram_usage(uint64_t* used, uint64_t* allowed) { + *used = 0, *allowed = card_mb << 20; + return card_mb != 0; +} +static void logf(const char*, ...) {} +// ((Triforce, 11-22) MEM-141's swap hold, merged into the loop: off, as it always is on Windows +#include "linux_pressure.h" +static bool memory_swap_hold() { return false; } +static void memory_swap_valve(int, uint64_t) {} + +/* ARENAS */ +/* STAGING */ + +static uint64_t give_back(uint64_t completed) { +/* LOOP */ + return kept; +} + +static int fails; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +constexpr uint64_t MB = 1ull << 20; + +static uint64_t frame_at, kept_bytes; +// one frame: its recording stages, then its present learns that the GPU is done with the frame before +static void frame(std::initializer_list stages_mb) { + rec_value = ++frame_at; + for (uint64_t mb : stages_mb) CHECK(staging(mb * MB).data != nullptr); + completed_value = frame_at - 1; + kept_bytes = give_back(frame_at - 1); +} +static void age(int ms) { // (time passes: every arena was last staged into that much longer ago) + for (Arena& a : arenas) a.used_at -= std::chrono::milliseconds(ms); +} +static void say() { + std::printf("%llu: made %d freed %d kept %llu MB in %zu arenas, counted %lld\n", (unsigned long long)frame_at, makes, frees, + (unsigned long long)(kept_bytes / MB), arenas.size(), (long long)st_arena_makes.load()); +} + +int main(int argc, char** argv) { + if (argc < 3) return 2; + card_mb = (uint64_t)std::atoll(argv[1]); + const char* what = argv[2]; + pool_pressed = argc > 3 && argv[3][0] == 'p'; + if (!std::strcmp(what, "rule")) { + using namespace staging_keep; + CHECK(burst_s(nullptr) == 2 && burst_s("") == 2 && burst_s("2") == 2 && burst_s("5") == 5 && burst_s("99999") == 3600); + CHECK(burst_s("0") == 0 && burst_s("-3") == 0 && burst_s("off") == 0); + CHECK(recent(2, false, 0) && recent(2, false, 1999) && !recent(2, false, 2000) && !recent(2, false, 60000)); + CHECK(!recent(2, true, 0) && !recent(0, false, 0) && !recent(2, false, -1) && recent(1, false, 999) && !recent(1, false, 1000)); + CHECK(allowance(64 * MB, true) == 256 * MB && allowance(64 * MB, false) == 64 * MB); + CHECK(allowance(256 * MB, true) == 256 * MB && allowance(256 * MB, false) == 256 * MB); // (a card over 6.5 GB: no change) + CHECK(allowance(512 * MB, true) == 512 * MB && allowance(0, false) == 0 && allowance(0, true) == 256 * MB); + } else if (!std::strcmp(what, "row")) { + // 20 frames that stage 32 MB each, then a present with nothing staged, then 2 seconds later another + for (int i = 0; i < 20; i++) frame({32}); + say(); + frame({}); + say(); + age(2000); + frame({}); + say(); + } else if (!std::strcmp(what, "big")) { + frame({60, 60, 60, 60, 60, 60}); // (60 + 60 do not fit one arena: six of them, all read by this frame) + say(); + frame({}); // (the GPU is done with them: idle, and staged into a moment ago) + say(); + frame({100}); // (an arena of 100 MB) + say(); + frame({}); // (idle: over 64 MB, it goes at once) + say(); + age(2000); + frame({}); + say(); + } else if (!std::strcmp(what, "mix")) { + uint32_t r = 12345; + const auto next = [&] { return (r = r * 1664525u + 1013904223u) >> 8; }; + for (int i = 0; i < 300; i++) { + const uint32_t n = next() % 6; + rec_value = ++frame_at; + for (uint32_t k = 0; k < n; k++) CHECK(staging((1 + next() % 60) * MB).data != nullptr); + if (next() % 7 == 0) age(700 * (int)(1 + next() % 4)); + completed_value = frame_at - 1; + kept_bytes = give_back(frame_at - 1); + say(); + } + } else { + return 2; + } + CHECK(makes == (int)st_arena_makes.load()); // (every make counted) + CHECK(st_arena_make_ns.load() >= 0); + uint64_t held = 0; + for (const Arena& a : arenas) held += a.size; + CHECK(held == live_bytes && (size_t)(makes - frees) == arenas.size()); + return fails != 0; +} +''' + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def check_source(tex): + check('#include "staging_keep.h"' in tex, "vk_texture.cpp does not include staging_keep.h") + staging = block(tex, "Staging staging(uint64_t n, uint64_t align = 256) {") + made = staging.index("const double ms = std::chrono::duration(std::chrono::steady_clock::now() - t_make).count();") + check(staging.index("if (!small) return {};") < made < + staging.index("st_arena_makes.fetch_add(1, std::memory_order_relaxed);") < + staging.index("st_arena_make_ns.fetch_add((int64_t)(ms * 1e6), std::memory_order_relaxed);") < staging.index("arenas.push_back(a);"), + "staging() does not count every arena it makes, and its time, before it keeps the arena") + check(staging.count("st_arena_makes") == 1, "staging() counts an arena somewhere else too") + loop = tex[tex.index(LOOP_HEAD):tex.index(LOOP_END)] + at = [loop.index(n) for n in ( + 'static const int64_t burst_s = staging_keep::burst_s(getenv("RX_VK_STAGING_BURST_S"));', + "const bool idle = it->value <= completed && it->value != rec_value;", + # ((Triforce, 11-22) MEM-141's swap hold merged in: a shorter idle while it is on, and no burst's room then + "const bool stale = idle_now && (swap_hold_began || std::chrono::steady_clock::now() - it->used_at > std::chrono::seconds(idle_now));", + "swap_hold_on ? keep : staging_keep::allowance(keep, staging_keep::recent(burst_s, pool_pressed, since_used_ms));", + "if (idle && (it->size > kArena || kept + it->size > allowance || stale)) {", + "destroy_buffer(it->buffer, it->allocation);")] + check(at == sorted(at), "the give-back loop no longer reads the switch once and asks staging_keep.h for what may stay") + check("kept + it->size > keep" not in loop, "the give-back loop still tests what is kept without the burst's allowance") + check('"vk: memory: working set %llu MB (peak %llu), private %llu MB, staging %llu MB in %zu arenas, %lld made so "' in tex and + "(long long)st_arena_makes.load(std::memory_order_relaxed)," in tex, + "the memory line no longer says how many arenas were made") + docs = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("| `RX_VK_STAGING_BURST_S=` |" in docs and "`gpu/vk/staging_keep.h`" in docs, + "docs/development.md: RX_VK_STAGING_BURST_S is not in the switches table") + arenas = tex[tex.index("struct Arena {"):tex.index("// (GPU writes into guest memory by buffers")] + check("std::atomic st_arena_makes{0}, st_arena_make_ns{0};" in arenas, "the arenas' counters are not beside the arenas") + return arenas, staging, loop + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + check((VK / "staging_keep.h").is_file(), "gpu/vk/staging_keep.h is not there") + arenas, staging, loop = check_source((VK / "vk_texture.cpp").read_text(encoding="utf-8")) + env0 = {k: v for k, v in os.environ.items() if not k.startswith("RX_VK_STAGING_")} + with tempfile.TemporaryDirectory(prefix="staging-burst-") as tmp: + cpp, exe = Path(tmp) / "burst.cpp", Path(tmp) / "burst.exe" + cpp.write_text(FIXTURE.replace("/* ARENAS */", arenas).replace("/* STAGING */", staging).replace("/* LOOP */", loop), + encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(VK), str(cpp), "-o", str(exe)], + check=True) + + def run(card, what, burst=None, pressed=False): + env = dict(env0) + if burst is not None: + env["RX_VK_STAGING_BURST_S"] = burst + r = subprocess.run([str(exe), str(card), what] + (["pressed"] if pressed else []), capture_output=True, text=True, env=env) + check(r.returncode == 0, "%s on a card allowed %d MB (RX_VK_STAGING_BURST_S %s%s):\n%s" + % (what, card, burst or "unset", ", VRAM short" if pressed else "", r.stdout + r.stderr)) + return r.stdout.splitlines() + + run(6000, "rule") + # frames that stage in a row, where 64 MB are kept + check(run(6000, "row") == ["20: made 2 freed 0 kept 128 MB in 2 arenas, counted 2", + "21: made 2 freed 0 kept 128 MB in 2 arenas, counted 2", + "22: made 2 freed 1 kept 64 MB in 1 arenas, counted 2"], + "20 frames staging in a row on a card allowed 6,000 MB: not 2 arenas made, or not 64 MB again 2 s later:\n" + + "\n".join(run(6000, "row"))) + before = ["20: made 11 freed 9 kept 128 MB in 2 arenas, counted 11", + "21: made 11 freed 10 kept 64 MB in 1 arenas, counted 11", + "22: made 11 freed 10 kept 64 MB in 1 arenas, counted 11"] + check(run(6000, "row", burst="0") == before, "RX_VK_STAGING_BURST_S=0 is not the loop as it was:\n" + "\n".join(run(6000, "row", burst="0"))) + check(run(6000, "row", pressed=True) == before, "with VRAM short the arenas are not given back as before:\n" + + "\n".join(run(6000, "row", pressed=True))) + check(run(6000, "row", burst="1")[1] == "21: made 2 freed 0 kept 128 MB in 2 arenas, counted 2", "RX_VK_STAGING_BURST_S=1 is no burst rule") + # a frame that stages six arenas' worth, and an arena over 64 MB + check(run(6000, "big") == ["1: made 6 freed 0 kept 384 MB in 6 arenas, counted 6", + "2: made 6 freed 2 kept 256 MB in 4 arenas, counted 6", + "3: made 7 freed 2 kept 356 MB in 5 arenas, counted 7", + "4: made 7 freed 3 kept 256 MB in 4 arenas, counted 7", + "5: made 7 freed 6 kept 64 MB in 1 arenas, counted 7"], + "a burst's idle arenas are not held to 256 MB, or an arena over 64 MB stays:\n" + "\n".join(run(6000, "big"))) + check(run(6000, "big", burst="0")[1] == "2: made 6 freed 5 kept 64 MB in 1 arenas, counted 6", "=0 keeps more than 64 MB idle on a small card") + # where nothing may change: 256 MB kept anyway, or VRAM short + for card, pressed, why in ((31000, False, "a card allowed 31,000 MB"), (0, False, "a card that reports nothing"), + (6657, False, "a card allowed just over 6.5 GB"), (6000, True, "a small card with VRAM short")): + a, b = run(card, "mix", pressed=pressed), run(card, "mix", burst="0", pressed=pressed) + check(len(a) == 300 and a == b, why + ": the rule changes what is made or given back there") + check(run(6000, "mix") != run(6000, "mix", burst="0"), "the mixed frames do not exercise the rule on a small card") + small, small0 = run(6000, "mix")[-1], run(6000, "mix", burst="0")[-1] + made = lambda line: int(line.split("made ")[1].split()[0]) + check(made(small) < made(small0), "on a small card the rule does not make fewer arenas over the mixed frames: %s against %s" % (small, small0)) + print("a card allowed 6,000 MB, 300 mixed frames: %d arenas made with the rule, %d without" % (made(small), made(small0))) + print("PASS: the idle staging arenas stay through a load's burst, and nothing changes where 256 MB are kept") + + +if __name__ == "__main__": + main() diff --git a/tests/test_stall_host.py b/tests/test_stall_host.py new file mode 100644 index 0000000..a127ebe --- /dev/null +++ b/tests/test_stall_host.py @@ -0,0 +1,387 @@ +"""The host's part of a long frame, a line beside the renderer's stall line (host/stall_host.h, host/mkw_glue.c +hos_stall_host_presented; no game). + +(Triforce, 11-22, the performance audit's OS-51, after Source B's.) The stall line splits a frame of 50 ms or more on the +replay's side and calls the rest outside the replay and the present. The host counts where it spends that rest (file +reads, writes, flushes, save commits, discards, zero-fills, decommits, give-back passes, the waits for a virtual core), +and a present that comes the stall line's threshold or more after the last one logs it with the process's page faults +and CPU time. + +- stall_host.h compiled and run: the threshold from the two switches (RX_VK_STALL_LOG's, 50 ms unset, off at 0; + HOS_STALL_HOST=0 off); a cause's adds and its take, also from eight threads at once (every add counted, the longest the + longest); the line for a frame with causes, for one with none ("none of the host's causes": ruled out), with lines + left out before it; the bound (3 a second, 1500 a session). +- hos_stall_host_presented cut out of mkw_glue.c and run on a scripted clock: no line for a short frame, and what a + short frame counted is not carried into a later long one; a long frame's line has what was counted since the present + before it, the page faults and CPU time of that frame only (Windows' 32-bit fault count wrapping included) and the + core waits; a run of long frames is bounded; with HOS_STALL_HOST=0 or RX_VK_STALL_LOG=0 nothing is counted, nothing + is asked of the system and nothing is logged; a threshold of 100 ms is kept. +- on Windows, process_counters and process_work compiled against Windows itself and run: the fault count rises when + fresh memory is touched, the CPU time when the processor is used. +- the sources read: each counted call is between its begin and end (the read's and the discard's end before the + virtual core is taken back, so the wait for the core is counted once, as a core wait), the virtual cores count their + waits, the present makes the look, and the switch is in the switches table. + + python -I -B tests/test_stall_host.py [--cc clang] +""" +import argparse +import os +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +RUN_ALL_ARGS = ["--cc", "clang"] + +ROOT = Path(__file__).resolve().parents[1] +HOST = ROOT / "host" + + +def fail(message): + raise SystemExit("FAIL: " + message) + + +def block(source, marker): + start = source.index(marker) + end = source.index("{", start) + 1 + depth = 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[start:end] + + +HEADER = r''' +#include +#include +#include "stall_host.h" +static int fails; +#define CHECK(x) do { if (!(x)) { printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +StallHostCause hos_stall_host[STALL_HOST_CAUSES]; +int64_t hos_stall_host_us = -1; +int64_t hos_now_ns(void) { return 1; } +static StallHostCause shared; +static void *adder(void *arg) { + const int64_t me = (int64_t)(intptr_t)arg; + for (int i = 1; i <= 20000; i++) stall_host_add(&shared, i + me, 4096); + return NULL; +} +static void same(const char *got, const char *want) { + if (strcmp(got, want)) { printf("FAIL: the line is\n %s\nnot\n %s\n", got, want); fails++; } +} +int main(void) { + /* the threshold */ + CHECK(stall_host_threshold_us(NULL, NULL) == 50000); /* the stall line's 50 ms */ + CHECK(stall_host_threshold_us("100", NULL) == 100000 && stall_host_threshold_us("12.5", NULL) == 12500); + CHECK(stall_host_threshold_us("0", NULL) == 0); /* no stall line: none of this */ + CHECK(stall_host_threshold_us("abc", NULL) == 0 && stall_host_threshold_us("-5", NULL) == 0); /* (as the renderer reads it) */ + CHECK(stall_host_threshold_us(NULL, "0") == 0 && stall_host_threshold_us("80", "0") == 0); /* HOS_STALL_HOST=0 */ + CHECK(stall_host_threshold_us(NULL, "1") == 50000 && stall_host_threshold_us("80", "1") == 80000); + /* a cause */ + StallHostCause c = {0, 0, 0, 0}; + stall_host_add(&c, 3000000, 1 << 20), stall_host_add(&c, 41000000, 11 << 20), stall_host_add(&c, 500000, 0); + stall_host_add(&c, -7, 0); /* (a clock that went back counts as no time) */ + CHECK(c.ns == 44500000 && c.count == 4 && c.bytes == (12 << 20) && c.longest == 41000000); + StallHostCause t = stall_host_take(&c); + CHECK(t.ns == 44500000 && t.count == 4 && t.bytes == (12 << 20) && t.longest == 41000000); + CHECK(c.ns == 0 && c.count == 0 && c.bytes == 0 && c.longest == 0); + t = stall_host_take(&c); + CHECK(t.count == 0 && t.ns == 0); + /* eight threads adding at once: nothing lost, the longest is the longest */ + pthread_t th[8]; + for (intptr_t i = 0; i < 8; i++) pthread_create(&th[i], NULL, adder, (void *)(i * 100000)); + for (int i = 0; i < 8; i++) pthread_join(th[i], NULL); + CHECK(shared.count == 8 * 20000 && shared.bytes == 8LL * 20000 * 4096); + CHECK(shared.ns == 8LL * (20000LL * 20001 / 2) + 20000LL * 100000 * (0 + 1 + 2 + 3 + 4 + 5 + 6 + 7)); + CHECK(shared.longest == 20000 + 700000); + /* the line */ + char line[1024]; + StallHostFrame f; + memset(&f, 0, sizeof f); + f.frame = 4812, f.gap_ns = 143200000; + f.cause[STALL_HOST_READ] = (StallHostCause){52000000, 3, 12 << 20, 41000000}; + f.cause[STALL_HOST_FLUSH] = (StallHostCause){18000000, 1, 0, 18000000}; + f.cause[STALL_HOST_DISCARD] = (StallHostCause){9000000, 2, 180 << 20, 6100000}; + f.cause[STALL_HOST_CORE_WAIT] = (StallHostCause){64000000, 212, 0, 31000000}; + f.page_faults = 41000, f.kernel_ns = 38000000, f.user_ns = 120000000; + CHECK(stall_host_line(&f, line, sizeof line) == (int)strlen(line)); + same(line, "vi: stall, the host's part: frame 4812, 143.2 ms: file reads 3 (12.0 MB, 52.0 ms, longest 41.0), flushes 1 (18.0 ms, " + "longest 18.0); discards 2 (180.0 MB, 9.0 ms, longest 6.1); core waits 64.0 ms (212, longest 31.0); 41000 page faults; " + "kernel 38.0 ms, user 120.0"); + memset(&f, 0, sizeof f); + f.frame = 7, f.gap_ns = 61000000, f.page_faults = 12, f.kernel_ns = 1000000, f.user_ns = 59500000, f.held = 2; + stall_host_line(&f, line, sizeof line); + same(line, "vi: stall, the host's part: frame 7, 61.0 ms: none of the host's causes; 12 page faults; kernel 1.0 ms, user 59.5 " + "(2 such lines left out before this one)"); + memset(&f, 0, sizeof f); + f.frame = 9, f.gap_ns = 250000000; + f.cause[STALL_HOST_WRITE] = (StallHostCause){2000000, 1, 1 << 19, 2000000}; + f.cause[STALL_HOST_COMMIT] = (StallHostCause){30000000, 1, 0, 30000000}; + f.cause[STALL_HOST_ZERO] = (StallHostCause){4000000, 1, 64 << 20, 4000000}; + f.cause[STALL_HOST_DECOMMIT] = (StallHostCause){1000000, 2, 2 << 20, 700000}; + f.cause[STALL_HOST_GIVE_BACK] = (StallHostCause){5000000, 1, 16 << 20, 5000000}; + stall_host_line(&f, line, sizeof line); + same(line, "vi: stall, the host's part: frame 9, 250.0 ms: file writes 1 (0.5 MB, 2.0 ms, longest 2.0), save commits 1 (30.0 ms, " + "longest 30.0), zero-fills 1 (64.0 MB, 4.0 ms, longest 4.0), decommits 2 (2.0 MB, 1.0 ms, longest 0.7), give-backs 1 " + "(16.0 MB, 5.0 ms, longest 5.0); 0 page faults; kernel 0.0 ms, user 0.0"); + char small[40]; /* (a buffer too small: cut, never written past) */ + CHECK(stall_host_line(&f, small, sizeof small) >= (int)sizeof small - 1 && strlen(small) == sizeof small - 1); + /* the bound: three at once, the fourth held; a third of a second later one more; 1500 in all */ + StallHostBound b = STALL_HOST_BOUND; + int64_t now = 1000000000; + CHECK(stall_host_bound_take(&b, now) && stall_host_bound_take(&b, now) && stall_host_bound_take(&b, now)); + CHECK(!stall_host_bound_take(&b, now) && b.held == 1); + now += 340000000; + CHECK(stall_host_bound_take(&b, now) && !stall_host_bound_take(&b, now) && b.held == 2); + int told = 4; + for (int i = 0; i < 4000; i++) told += stall_host_bound_take(&b, now += 1000000000); + CHECK(told == 1500 && b.told == 1500); + if (!fails) printf("PASS: the threshold, a cause from eight threads, the line and the bound\n"); + return fails != 0; +} +''' + +LOOK = r''' +#include +#include "stall_host.h" +static int fails; +#define CHECK(x) do { if (!(x)) { printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +typedef long long LONG64; +StallHostCause hos_stall_host[STALL_HOST_CAUSES]; +int64_t hos_stall_host_us = -1; +volatile LONG64 hos_present_count; +/* the stand-ins: a clock the case moves, the process's counts as the case sets them, the core waits it queues, the log kept */ +static int64_t clock_ns = 5000000000LL; +int64_t hos_now_ns(void) { return clock_ns; } +static uint64_t faults_now; +static int64_t kernel_now, user_now, wait_ns, wait_n, wait_longest; +static int asked, lines; +static char last[1024]; +static void process_work(uint64_t *page_faults, int64_t *kernel_ns, int64_t *user_ns) { + asked++; + *page_faults = faults_now, *kernel_ns = kernel_now, *user_ns = user_now; +} +void hos_vcore_waits_take(int64_t *ns, int64_t *count, int64_t *longest) { + *ns = wait_ns, *count = wait_n, *longest = wait_longest; + wait_ns = wait_n = wait_longest = 0; +} +static void hos_log(const char *fmt, const char *line) { + (void)fmt; + lines++; + snprintf(last, sizeof last, "%s", line); +} +@PRESENTED@ +/* a present ms after the last one */ +static void present(double ms) { + clock_ns += (int64_t)(ms * 1e6); + hos_present_count++; + hos_stall_host_presented(); +} +/* a counted call that takes ms */ +static void call(int cause, double ms, uint64_t bytes) { + const int64_t began = stall_host_begin(); + clock_ns += (int64_t)(ms * 1e6); + stall_host_end(cause, began, bytes); +} +int main(int argc, char **argv) { + const char *c = argc > 1 ? argv[1] : ""; + if (!strcmp(c, "off")) { /* HOS_STALL_HOST=0, or RX_VK_STALL_LOG=0 */ + CHECK(stall_host_begin() == 0); + call(STALL_HOST_READ, 80, 1 << 20); + present(16), present(300), present(300); + CHECK(lines == 0 && asked == 0 && hos_stall_host[STALL_HOST_READ].count == 0); + } else if (!strcmp(c, "threshold")) { /* RX_VK_STALL_LOG=100 */ + present(16), present(80); + CHECK(lines == 0); + present(120); + CHECK(lines == 1 && strstr(last, ", 120.0 ms:")); + } else { + CHECK(stall_host_begin() == clock_ns); + present(16); /* the first present: nothing to measure from */ + CHECK(lines == 0 && asked == 1); + /* a short frame with a flush in it: no line, and its flush is not the next frame's */ + call(STALL_HOST_FLUSH, 9, 0); + wait_ns = 2000000, wait_n = 5, wait_longest = 900000; + faults_now = 0xFFFFFFF0u, kernel_now = 1000000000, user_now = 5000000000LL; + present(7); + CHECK(lines == 0 && asked == 2 && hos_stall_host[STALL_HOST_FLUSH].count == 0); + /* a long frame: three reads, a discard, core waits; the fault count wraps */ + call(STALL_HOST_READ, 41, 6 << 20), call(STALL_HOST_READ, 5, 3 << 20), call(STALL_HOST_READ, 6, 3 << 20); + call(STALL_HOST_DISCARD, 4.5, 180 << 20); + wait_ns = 64000000, wait_n = 212, wait_longest = 31000000; + faults_now = 0x10, kernel_now += 38000000, user_now += 120000000; + present(143.2 - 56.5); + CHECK(lines == 1 && asked == 3); + if (strcmp(last, "vi: stall, the host's part: frame 3, 143.2 ms: file reads 3 (12.0 MB, 52.0 ms, longest 41.0); discards 1 " + "(180.0 MB, 4.5 ms, longest 4.5); core waits 64.0 ms (212, longest 31.0); 32 page faults; kernel 38.0 ms, " + "user 120.0")) { + printf("FAIL: the long frame's line is\n %s\n", last); + fails++; + } + /* exactly at the threshold: a line; just under: none; and nothing counted: ruled out */ + present(50); + CHECK(lines == 2 && strstr(last, "frame 4, 50.0 ms: none of the host's causes; 0 page faults; kernel 0.0 ms, user 0.0")); + present(49.9); + CHECK(lines == 2); + /* long frames in a row: three a second at most, the ones left out said by the next line */ + clock_ns += 5000000000LL; + hos_stall_host_presented(); /* (a 5 s frame: the bound is full again after it) */ + CHECK(lines == 3); + for (int i = 0; i < 6; i++) present(60); + /* (two lines left for the first two; the next three are left out; by the sixth a third of a second has passed) */ + CHECK(lines == 6 && strstr(last, "(3 such lines left out before this one)")); + present(2000); + CHECK(lines == 7 && !strstr(last, "left out")); + } + if (!fails) printf("PASS %s\n", c); + return fails != 0; +} +''' + +WORK = r''' +#include +#include +#include +#include +static int fails; +#define CHECK(x) do { if (!(x)) { printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +@COUNTERS@ +@WORK@ +int main(void) { + uint64_t f0, f1; + int64_t k0, u0, k1, u1; + process_work(&f0, &k0, &u0); + CHECK(f0 > 0 && k0 >= 0 && u0 >= 0); + volatile unsigned char *p = (volatile unsigned char *)VirtualAlloc(NULL, 64 << 20, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); + CHECK(p != NULL); + for (size_t i = 0; p && i < (64u << 20); i += 4096) p[i] = 1; /* 16384 first touches */ + volatile uint64_t x = 1; + const ULONGLONG until = GetTickCount64() + 120; + while (GetTickCount64() < until) x = x * 6364136223846793005ULL + 1; /* 120 ms of the processor */ + process_work(&f1, &k1, &u1); + CHECK((uint32_t)(f1 - f0) >= 4096); /* (a fault a page, or nearly) */ + CHECK(k1 >= k0 && u1 >= u0 && (k1 - k0) + (u1 - u0) > 0); /* (Windows counts in 15.6 ms steps, on a PC that may be busy) */ + printf("this process: %u page faults for 64 MB of first touches, %.0f ms in the kernel and %.0f in its own code\n", + (unsigned)(uint32_t)(f1 - f0), (k1 - k0) / 1e6, (u1 - u0) / 1e6); + if (!fails) printf("PASS: the process's page faults and CPU time as Windows counts them\n"); + return fails != 0; +} +''' + + +def between(text, begin, body, end, what): + """begin, then body, then end, in that order, each there""" + try: + if not text.index(begin) < text.index(body) < text.index(end, text.index(body)): + fail(what) + except ValueError: + fail(what) + + +def read_sources(): + fs = (HOST / "nn_fs.c").read_text(encoding="utf-8") + mem = (HOST / "hos_mem.c").read_text(encoding="utf-8") + thread = (HOST / "hos_thread.c").read_text(encoding="utf-8") + nvn = (HOST / "nvn_native.c").read_text(encoding="utf-8") + glue = (HOST / "mkw_glue.c").read_text(encoding="utf-8") + dev = (ROOT / "docs/development.md").read_text(encoding="utf-8") + for name, text in (("nn_fs.c", fs), ("hos_mem.c", mem), ("mkw_glue.c", glue)): + if '#include "stall_host.h"' not in text: + fail("host/%s does not include stall_host.h" % name) + read = block(fs, "static uint64_t read_at(") + between(read, "const int64_t stall0 = stall_host_begin();", "ReadFile(fh->h,", "stall_host_end(STALL_HOST_READ, stall0, done);", + "nn_fs.c read_at: the read is not counted") + if not read.index("hos_vcore_away();") < read.index("stall_host_begin();") or \ + not read.index("stall_host_end(STALL_HOST_READ") < read.index("if (away) hos_vcore_back();"): + fail("nn_fs.c read_at: the read's time takes in the wait for the virtual core, which the core waits count already") + between(block(fs, "HLE(_ZN2nn2fs9WriteFileENS0_10FileHandleElPKvmRKNS0_11WriteOptionE) {"), "const int64_t stall0 = stall_host_begin();", + "WriteFile(fh->h,", "stall_host_end(STALL_HOST_WRITE, stall0, put);", "nn_fs.c WriteFile: the write is not counted") + between(block(fs, "HLE(_ZN2nn2fs9FlushFileENS0_10FileHandleE) {"), "const int64_t stall0 = stall_host_begin();", "FlushFileBuffers(", + "stall_host_end(STALL_HOST_FLUSH, stall0, 0);", "nn_fs.c FlushFile: the flush is not counted") + between(block(fs, "HLE(_ZN2nn2fs6CommitEPKc) {"), "const int64_t stall0 = stall_host_begin();", "save_tx_stage(&tx_mounts[i].p);", + "stall_host_end(STALL_HOST_COMMIT, stall0, 0);", "nn_fs.c Commit: the save's staging is not counted") + discard = block(mem, "static void discard_pages(") + between(discard, "const int64_t stall0 = stall_host_begin();", "hos_guest_fill(lo, 0, hi - lo)", # (MEM-13: the fill, then its mark) + "stall_host_end(STALL_HOST_ZERO, stall0, hi - lo);", "hos_mem.c discard_pages: the zero-fill under an import is not counted") + between(discard, "const int64_t stall0 = stall_host_begin();", "VirtualAlloc(GP(lo), (SIZE_T)(hi - lo), MEM_RESET, PAGE_READWRITE);", + "stall_host_end(STALL_HOST_DISCARD, stall0, hi - lo);", "hos_mem.c discard_pages: the discard is not counted") + if not discard.index("stall_host_end(STALL_HOST_DISCARD") < discard.index("if (away) hos_vcore_back();"): + fail("hos_mem.c discard_pages: the discard's time takes in the wait for the virtual core") + between(block(mem, "HLE(_ZN2nn2os15FreeMemoryPagesEmm)"), "stall_host_begin()", "MEM_DECOMMIT)", + "stall_host_end(STALL_HOST_DECOMMIT, stall0, hi > lo ? hi - lo : 0);", "hos_mem.c FreeMemoryPages: the decommit is not counted") + between(block(mem, "int64_t hos_mem_give_back(uint64_t max_bytes) {"), "const int64_t stall0 = stall_host_begin();", "gc_give_back(", + "stall_host_end(STALL_HOST_GIVE_BACK, stall0, given);", "hos_mem.c hos_mem_give_back: the pass is not counted") + acquire = block(thread, "static void vc_acquire(HosThread *t) {") + if "__atomic_add_fetch(&vc_gap.ns, waited, __ATOMIC_RELAXED), __atomic_add_fetch(&vc_gap.n, 1, __ATOMIC_RELAXED);" not in acquire or \ + "if (waited > __atomic_load_n(&vc_gap.longest, __ATOMIC_RELAXED)) __atomic_store_n(&vc_gap.longest, waited, __ATOMIC_RELAXED);" not in acquire: + fail("hos_thread.c vc_acquire: a wait for a core is not added to the waits since the last present") + take = block(thread, "void hos_vcore_waits_take(int64_t *ns, int64_t *count, int64_t *longest) {") + if take.count("__atomic_exchange_n(&vc_gap.") != 3: + fail("hos_thread.c hos_vcore_waits_take does not take the three counts and leave them at zero") + present = block(nvn, "NV(nvnQueuePresentTexture) {") + if not present.index("hos_vi_native_presented();") < present.index("hos_stall_host_presented();"): + fail("nvn_native.c: the present does not make the host's look after it is counted and paced") + work = block(glue, "static void process_work(uint64_t *page_faults, int64_t *kernel_ns, int64_t *user_ns) {") + if "*page_faults = m.PageFaultCount;" not in work or "GetProcessTimes(GetCurrentProcess(), &made, &ended, &kernel, &user)" not in work: + fail("mkw_glue.c process_work: not the process's page faults and its kernel and user time") + if "StallHostCause hos_stall_host[STALL_HOST_CAUSES];" not in glue or "int64_t hos_stall_host_us = -1;" not in glue: + fail("mkw_glue.c: the counters or the threshold not read yet are not defined") + if "| `HOS_STALL_HOST=0` |" not in dev: + fail("docs/development.md: HOS_STALL_HOST is not in the switches table") + return glue + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cc", default="clang") + args = parser.parse_args() + try: + glue = read_sources() + presented = block(glue, "void hos_stall_host_presented(void) {") + counters = glue[glue.index("typedef struct {\n DWORD cb, PageFaultCount;"):glue.index("void hos_process_memory(")] + work = block(glue, "static void process_work(uint64_t *page_faults, int64_t *kernel_ns, int64_t *user_ns) {") + except ValueError as e: + fail("a text this test reads is not there (%s)" % e) + compiler = shutil.which(args.cc) + if not compiler: + raise SystemExit("C compiler not found: " + args.cc) + env = {k: v for k, v in os.environ.items() if k not in ("RX_VK_STALL_LOG", "HOS_STALL_HOST")} + with tempfile.TemporaryDirectory(prefix="stall-host-") as temporary: + folder = Path(temporary) + + def build(name, text, extra=()): + (folder / (name + ".c")).write_text(text, encoding="utf-8") + exe = folder / (name + ".exe") + built = subprocess.run([compiler, "-std=gnu17", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(HOST), str(folder / (name + ".c")), + "-o", str(exe), *extra], capture_output=True, text=True, timeout=180) + if built.returncode: + print(built.stdout + built.stderr) + fail("the %s fixture did not build" % name) + return exe + + def run(exe, case="", **switches): + done = subprocess.run([str(exe), case], env=dict(env, **switches), capture_output=True, text=True, timeout=120) + if done.returncode or "PASS" not in done.stdout: + print(done.stdout[-3000:] + done.stderr[-1000:]) + fail("%s %s %s" % (exe.stem, case, switches or "")) + return done.stdout.strip() + + print(run(build("header", HEADER, ["-pthread"]))) + look = build("look", LOOK.replace("@PRESENTED@", presented)) + run(look, "on") + run(look, "on", RX_VK_STALL_LOG="50", HOS_STALL_HOST="1") + run(look, "off", HOS_STALL_HOST="0") + run(look, "off", RX_VK_STALL_LOG="0") + run(look, "threshold", RX_VK_STALL_LOG="100") + print("PASS: the present's look: short frames say nothing and carry nothing over, a long frame's line is that frame's, " + "long frames in a row are bounded, both switches turn all of it off") + if sys.platform == "win32": + print(run(build("work", WORK.replace("@COUNTERS@", counters).replace("@WORK@", work)))) + else: + print("SKIP: the process's counts are Windows' (not this system)") + print("PASS: the host's part of a long frame is counted where it is spent and logged beside the stall line") + + +if __name__ == "__main__": + main() diff --git a/tests/test_steady_period.py b/tests/test_steady_period.py index df62dbe..a59e298 100644 --- a/tests/test_steady_period.py +++ b/tests/test_steady_period.py @@ -126,6 +126,9 @@ static void renderer_present(void) { static int mode_asked = 1; int hos_ini_option(const char *key, int otherwise) { return strcmp(key, "steady_frame_rate") ? otherwise : mode_asked; } static int fps_max(void) { return 240; } +static void movie_pace_decide(void) {} /* ((Triforce, 11-22, PRESENT-30) no movie plays here: paced by the cap) */ +static int pace_fps(void) { return fps_max(); } +void hos_bench_present(int64_t now) { (void)now; } /* (benchmark mode's note of each present: not on here) */ /* ---- host/nn_vi.c's state these functions use, and the rest of hos_vi_native_presented's stand-ins ---- */ static volatile LONG64 presents; static int64_t next_frame_at, window_start, frames_in_window; @@ -399,7 +402,8 @@ def main(): steady = steady[:steady.index("static int64_t steady_period(int64_t base, int64_t now) {")] + block(steady, "static int64_t steady_period(") presented = vi[vi.index("void hos_vi_native_presented(void) {"):] presented = presented[:re.search(r"^}", presented, re.M).end()] - code = "\n".join((region, block(vi, "void hos_vi_native_acquire(int active) {"), steady, presented)) + # (with the 1% low's sort, which the cut-out present calls) + code = "\n".join((region, block(vi, "void hos_vi_native_acquire(int active) {"), steady, block(vi, "static int cmp_i64_desc("), presented)) source = PREFIX.replace("@IDS@", ids.group(0) + "\n" + cap.group(0)) + code + SUFFIX env = {k: v for k, v in os.environ.items() if not k.startswith(("HOS_", "RX_"))} with tempfile.TemporaryDirectory(prefix="steady-period-") as tmp: diff --git a/tests/test_step_steady.py b/tests/test_step_steady.py index aff43a2..9b76b81 100644 --- a/tests/test_step_steady.py +++ b/tests/test_step_steady.py @@ -164,7 +164,7 @@ def check_source(): "the steady step does not start afresh where the average does") check("secs_avg = step_steady(&steady, dt, (double)pace_period_ns / 1e9, timed ? 1.0 / 240.0 : 1.0 / top, 4.0 / 60.0);" in step, "the steady step is not taken from this interval and the paced period within each path's bounds") - check(step.index("const int top = timed ? 60 : fps_max();") < step.index("secs_avg = step_steady("), + check(step.index("const int top = timed ? 60 : pace;") < step.index("secs_avg = step_steady("), # ((Triforce, 11-22, PRESENT-30) pace_fps()) "the bounds are read before they are known") check("const double secs = secs_avg;" in step and "const double ratio = secs_avg * 60.0;" in step, "the step the game gets is not the one made") diff --git a/tests/test_sticky_hot_pieces.py b/tests/test_sticky_hot_pieces.py new file mode 100644 index 0000000..0849851 --- /dev/null +++ b/tests/test_sticky_hot_pieces.py @@ -0,0 +1,391 @@ +"""The hot pieces stay where they were from one lift to the next, and each declares only what it uses +(nx2/split_hot.py), on a made-up code_hot.c (game-free). + +(Triforce, 11-22, the performance audit's LIFT-30, after Source A's.) Every split dealt the hot functions out afresh by +their sizes and gave every piece code_hot.c's whole preamble, so an update that touched a few hot functions could change +most of the eight pieces, the longest files of the build. Now /image/hot_pieces.txt keeps each function's piece, +and a piece keeps a declaration only if it uses the name. + + - A64_HOT_STICKY=0: the pieces are v0.6.0's byte for byte (their digest here was taken with v0.6.0's nx2/split_hot.py + on the same made-up file), and hot_pieces.txt is neither written nor read. + - With no hot_pieces.txt every function lands in the piece v0.6.0 put it in; the file is then written. + - A piece's preamble is the whole one without the declarations it does not use: the lines that are not declarations + stay in every piece, every f_ and hos_ name a piece uses is still declared in it, and no declaration of a name it + does not use is left. Lines that read like declarations but are code stay too: a call inside an inline helper, a + macro's continued line, a declaration whose attribute is on the line before. With clang, every piece compiles + (-fsyntax-only, an undeclared name an error). + - With clang, after the changes below (a grown function, a new declaration, two new functions): the whole code_hot.c + and its eight sticky pieces, each built at -O2 with a driver and stand-ins for the hos_ names and the functions + outside the hot list, run every hot function from the same 100 random register states (edge values among them); + every register after every run must be the same. Leaving the helper's call out of the pieces that do not name it + (as a first draft of the rule did) fails here and above. + - One hot function grown by forty lines: exactly one piece changes. + - One hot function given a call of a function no hot one named before (so the lift adds a declaration): exactly one + piece changes. With A64_HOT_STICKY=0 the same change rewrites every piece: what this is for. + - A new hot function goes to the piece it shares the most calls with; one that calls nothing hot, to the cheapest. + - A function that left the hot list leaves its piece alone; the others stay. + - When keeping the pieces would make the costliest more than 1.15 times a fresh split's, the split is the fresh one; + so it is when hot_pieces.txt was made for another number of pieces, or is not what the splitter writes. + - hot_pieces.txt lies in /image for a gen folder /sa/gen or /gen, and is not in the lift key. + + python -I -B tests/test_sticky_hot_pieces.py [--cc clang] +""" +import argparse +import contextlib +import hashlib +import io +import os +import shutil +import subprocess +import sys +import tempfile + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(ROOT, "nx2")) +import kit # noqa: E402,F401 (a64-kit on the path) +import chunks # noqa: E402 +import split_hot # noqa: E402 + +RUN_ALL_ARGS = ["--cc", "clang"] +PIECES = 8 +# (the eight pieces and their costs with A64_HOT_STICKY=0, as v0.6.0's nx2/split_hot.py made them from made_up()'s file) +V060_DIGEST = "3dae331a1bea79fd38c6b9b4ef71f9f89e0215e7" +# (lines of the preamble that every piece keeps. The last ones read like declarations but are code: a call inside an inline +# helper, a macro's continued line, a declaration whose attribute is on the line before. Leaving one out of a piece changes +# what the helper does there, or what the macro or the attribute takes in) +FIXED = ('#include "a64.h"\n' + "__attribute__((returns_twice)) int recomp_sj_setjmp(void **slot);\n" + "#define MADE_UP_SCALE(c) ((c)->x[9] = hos_fixed_scale_bits)\n" + "extern void hos_probe_note(Ctx *c, uint64_t v);\n" + "static inline void made_up_probe(Ctx *c) {\n" + " c->x[10] += 5;\n" + " hos_probe_note(c, c->x[10]);\n" + "}\n" + "#define MADE_UP_LATE_DECL \\\n" + " void hos_probe_late(Ctx *c);\n" + "__attribute__((cold))\n" + "void hos_probe_cold(void);\n") +HOS = ("extern volatile int hos_fixed_resolution;\n", "extern volatile uint32_t hos_fixed_scale_bits;\n", + "extern void hos_oot_menu_tick(Ctx *c, uint64_t screen);\n", "uint64_t hos_mods_table_size(uint64_t size, uint64_t path);\n", + "extern volatile float hos_text_scale;\n") + + +def check(what, ok): + if not ok: + raise SystemExit("FAIL: sticky hot pieces: " + what) + + +def function(name, lines, calls=(), extra=""): + body = "".join("c->x[%d] = c->x[%d] + %d;\n" % (1 + k % 7, 2 + k % 5, k) for k in range(lines)) + return "void %s(Ctx *__restrict A64_NOESC c) {\n" % name + body + extra + "".join("%s(c);\n" % t for t in calls) + "}\n" + + +def made_up(): + """{name: (lines, calls, extra)}: 240 hot functions of many sizes that call each other and functions outside""" + fns = {} + names = ["f_%08x" % (0x80100000 + 0x40 * i) for i in range(240)] + for i, n in enumerate(names): + lines = 40 + (i * 37) % 260 + (2600 if i % 53 == 7 else 0) + calls = [] + if i % 3 == 0 and i + 1 < len(names): + calls.append(names[i + 1]) + if i % 5 == 0: + calls.append(names[(i * 7 + 3) % len(names)]) + if i % 11 == 0: + calls.append("f_%08x" % (0x90000000 + 0x100 * (i % 13))) # (not hot: declared, defined elsewhere) + extra = "" + if i == 10: + extra = "if (hos_fixed_resolution) c->x[8] &= ~0x18ULL;\n" + if i == 77: + extra = "hos_oot_menu_tick(c, c->x[19]);\nMADE_UP_SCALE(c);\n" + if i == 150: + extra = "c->x[0] = hos_mods_table_size(c->x[0], c->x[1]);\n" + if i % 17 == 4: + extra += "made_up_probe(c);\n" + fns[n] = (lines, tuple(c for c in calls if c != n), extra) + return fns + + +def hot_text(fns): + """code_hot.c as the lift and the patches write it: the fixed lines, the hos_ declarations, the declaration of every + function a hot one names, then the functions in address order""" + named = sorted({c for _, calls, _ in fns.values() for c in calls}) + preamble = FIXED + "".join(HOS) + "".join("void %s(Ctx *A64_NOESC);\n" % n for n in named) + return preamble, preamble + "".join(function(n, *fns[n]) for n in sorted(fns)) + + +def split(work, fns, sticky=True, pieces=PIECES): + """One lift's split of the made-up file: {piece file name: text}, and what the splitter said""" + gen = os.path.join(work, "sa", "gen") + os.makedirs(gen, exist_ok=True) + for old in os.listdir(gen): + if old.startswith("code_hot"): + os.remove(os.path.join(gen, old)) + with open(os.path.join(gen, "code_hot.c"), "w", encoding="utf-8", newline="") as f: + f.write(hot_text(fns)[1]) + if sticky: + os.environ.pop("A64_HOT_STICKY", None) + else: + os.environ["A64_HOT_STICKY"] = "0" + said = io.StringIO() + try: + with contextlib.redirect_stdout(said): + split_hot.split(gen, pieces) + finally: + os.environ.pop("A64_HOT_STICKY", None) + out = {} + for name in sorted(os.listdir(gen)): + if name.startswith("code_hot_") and name.endswith(".c"): + with open(os.path.join(gen, name), encoding="utf-8", newline="") as f: + out[name] = f.read() + with open(os.path.join(gen, chunks.COSTS), encoding="utf-8") as f: + costs = f.read() + return out, said.getvalue(), costs + + +def where(pieces): + """{function: piece file} of a split""" + return {m: name for name, text in pieces.items() for m in split_hot.HEAD.findall(text)} + + +def changed(a, b): + return sorted(n for n in set(a) | set(b) if a.get(n) != b.get(n)) + + +def digest(pieces, costs): + h = hashlib.sha1(usedforsecurity=False) + for name in sorted(pieces): + h.update(name.encode() + bytes(1) + pieces[name].encode() + bytes(1)) + h.update(costs.encode()) + return h.hexdigest() + + +def body_of(text): + marks = list(split_hot.HEAD.finditer(text)) + return text[:marks[0].start()] if marks else text, text[marks[0].start():] if marks else "" + + +STATES = 100 # (random register states a function, in run_both) +A64_H = "#include \ntypedef struct { uint64_t x[32]; } Ctx;\n#define A64_NOESC\n" + + +def driver(fns): + """The C that runs every hot function from the same random states and writes each state's registers after it: the hos_ + names and the functions outside the hot list as stand-ins that leave a mark in the registers""" + outside = sorted({c for _, calls, _ in fns.values() for c in calls} - set(fns)) + hot = sorted(fns) + return "".join([ + "#include \n#include \n#include \n#include \"a64.h\"\n", + "volatile int hos_fixed_resolution;\nvolatile uint32_t hos_fixed_scale_bits;\nvolatile float hos_text_scale;\n", + "void hos_oot_menu_tick(Ctx *c, uint64_t screen) { c->x[3] ^= screen * 0x9E3779B97F4A7C15ull; c->x[4] += 1; }\n", + "uint64_t hos_mods_table_size(uint64_t size, uint64_t path) { return size * 0x100000001b3ull ^ (path << 40) ^ 0xabcdef12345ull; }\n", + "void hos_probe_note(Ctx *c, uint64_t v) { c->x[11] = c->x[11] * 0x5851F42D4C957F2Dull + v + 1; }\n", + "".join("void %s(Ctx *c) { c->x[5] = c->x[5] * 6364136223846793005ull + 0x%sull; c->x[6] ^= c->x[5] >> 17; }\n" + % (n, n[2:]) for n in outside), + "".join("void %s(Ctx *c);\n" % n for n in hot), + "static void (*const hot[])(Ctx *) = {%s};\n" % ", ".join(hot), + "static uint64_t seed = 0x2545F4914F6CDD1Dull;\n" + "static uint64_t next(void) { uint64_t z = (seed += 0x9E3779B97F4A7C15ull);\n" + " z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ull; z = (z ^ (z >> 27)) * 0x94D049BB133111EBull; return z ^ (z >> 31); }\n" + "static const uint64_t edge[] = {0, 1, ~0ull, 1ull << 63, (1ull << 63) - 1, 0xffffffffull, 1ull << 32, 0x18};\n" + "int main(int argc, char **argv) {\n" + " FILE *out = fopen(argv[1], \"wb\");\n" + " if (!out) return 2;\n" + " for (unsigned f = 0; f < sizeof hot / sizeof hot[0]; f++)\n" + " for (int s = 0; s < %d; s++) {\n" % STATES, + " Ctx c;\n" + " for (int r = 0; r < 32; r++) { uint64_t v = next(); c.x[r] = (v & 3) == 0 ? edge[(v >> 2) & 7] : next(); }\n" + " hos_fixed_resolution = (int)(next() & 1);\n" + " hos_fixed_scale_bits = (uint32_t)next();\n" + " hot[f](&c);\n" + " if (fwrite(c.x, sizeof c.x, 1, out) != 1) return 3;\n" + " }\n" + " return fclose(out) != 0;\n" + "}\n"]) + + +def run_both(tmp, cc, gen, fns, label): + """The hot functions built from the whole code_hot.c and built from its pieces, each run from the same random states: + every register after every run must be the same. Returns how many runs were compared""" + root = os.path.join(tmp, "run-" + label) + os.makedirs(root) + with open(os.path.join(root, "a64.h"), "w", newline="\n") as f: + f.write(A64_H) + with open(os.path.join(root, "driver.c"), "w", newline="\n") as f: + f.write(driver(fns)) + shutil.copyfile(os.path.join(gen, "code_hot.c.whole"), os.path.join(root, "code_hot.c")) + pieces = sorted(os.path.join(gen, n) for n in os.listdir(gen) if chunks.HOT_PIECE.fullmatch(n)) + dumps = [] + for kind, srcs in (("whole", [os.path.join(root, "code_hot.c")]), ("pieces", pieces)): + exe = os.path.join(root, kind + ".exe") + r = subprocess.run([cc, "-O2", "-fno-strict-aliasing", "-Werror=implicit-function-declaration", "-I" + root] + srcs + + [os.path.join(root, "driver.c"), "-o", exe], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + check("%s: the %s build compiles:\n%s" % (label, kind, r.stdout[-1500:]), r.returncode == 0) + dump = os.path.join(root, kind + ".bin") + check("%s: the %s build runs" % (label, kind), subprocess.run([exe, dump]).returncode == 0) + with open(dump, "rb") as f: + dumps.append(f.read()) + runs = len(fns) * STATES + check("%s: %d runs written by each build" % (label, runs), len(dumps[0]) == len(dumps[1]) == runs * 32 * 8) + if dumps[0] != dumps[1]: + at = next(i for i in range(len(dumps[0])) if dumps[0][i] != dumps[1][i]) // 8 + check("%s: the pieces compute what the whole file computes: first difference in %s, state %d, x%d" + % (label, sorted(fns)[at // 32 // STATES], at // 32 % STATES, at % 32), False) + return runs + + +def check_all(tmp, cc): + fns = made_up() + preamble, whole = hot_text(fns) + map_file = os.path.join(tmp, "w", "image", "hot_pieces.txt") + work = os.path.join(tmp, "w") + check("the pieces file is /image/hot_pieces.txt for /sa/gen and for /gen", + split_hot.pieces_file(os.path.join(work, "sa", "gen")) == map_file == split_hot.pieces_file(os.path.join(work, "gen"))) + + # A64_HOT_STICKY=0: v0.6.0's pieces, and no pieces file + old, said, costs = split(work, fns, sticky=False) + check("with A64_HOT_STICKY=0 there are eight pieces", sorted(old) == ["code_hot_%d.c" % i for i in range(PIECES)]) + check("with A64_HOT_STICKY=0 the pieces are v0.6.0's byte for byte (digest %s)" % digest(old, costs), digest(old, costs) == V060_DIGEST) + check("with A64_HOT_STICKY=0 every piece has the whole preamble", all(t.startswith(preamble) for t in old.values())) + check("with A64_HOT_STICKY=0 no pieces file is written", not os.path.exists(map_file) and "sticky" not in said and "fresh" not in said) + check("the made-up file is big enough to mean something", all(len(split_hot.HEAD.findall(t)) > 5 for t in old.values())) + + # no pieces file: v0.6.0's piece for every function; the preamble cut to what each piece uses + new, said, _ = split(work, fns) + check("with no pieces file every function lands in v0.6.0's piece", where(new) == where(old) and "fresh (no pieces file)" in said) + check("the pieces file is written", split_hot.read_pieces(map_file) == {f: int(n[len("code_hot_"):-2]) for f, n in where(new).items()}) + kept_lines = 0 + for name, text in new.items(): + head, body = body_of(text) + check("%s holds its functions as before" % name, body == body_of(old[name])[1]) + lines, all_lines = head.splitlines(keepends=True), preamble.splitlines(keepends=True) + it = iter(all_lines) + check("%s: its preamble is the whole one with lines left out" % name, all(line in it for line in lines)) + check("%s: the lines that are not declarations stay" % name, head.startswith(FIXED)) + used = set(split_hot.NAME.findall(body)) | set(split_hot.NAME.findall(FIXED)) + declared = {n for line in lines[len(FIXED.splitlines()):] for n in split_hot.NAME.findall(line)} + whole_declared = {n for line in all_lines[len(FIXED.splitlines()):] for n in split_hot.NAME.findall(line)} + check("%s: every name it uses is still declared" % name, used & whole_declared <= declared) + check("%s: no declaration of a name it does not use is left: %s" % (name, sorted(declared - used)[:4]), declared <= used) + kept_lines += len(lines) + check("the pieces declare less than the whole preamble eight times over", kept_lines < PIECES * len(preamble.splitlines()) // 2) + check("the piece with the macro's name keeps the declaration the macro needs, as every piece does", + all("hos_fixed_scale_bits;" in t for t in new.values())) + check("hos_text_scale, which nothing uses, is declared nowhere", not any("hos_text_scale" in t for t in new.values())) + if cc: # (every piece is C that compiles: no declaration it needs was left out) + inc = os.path.join(tmp, "inc") + os.makedirs(inc) + with open(os.path.join(inc, "a64.h"), "w", newline="\n") as f: + f.write(A64_H) + for name in new: + r = subprocess.run([cc, "-fsyntax-only", "-Wno-everything", "-Werror=implicit-function-declaration", "-I" + inc, + os.path.join(work, "sa", "gen", name)], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + check("%s compiles:\n%s" % (name, r.stdout[-1500:]), r.returncode == 0) + + # one function grown: one piece changes + grown = dict(fns) + victim = sorted(fns)[61] + grown[victim] = (fns[victim][0] + 40,) + fns[victim][1:] + after, said, _ = split(work, grown) + check("a grown function changes exactly one piece (%s)" % changed(new, after), changed(new, after) == [where(new)[victim]] and "sticky (240 functions where they were, 0 new)" in said) + + # a new declaration, used by one function: one piece changes; with the switch at 0, all of them + called = dict(grown) + called[victim] = (grown[victim][0], grown[victim][1] + ("f_9000beef",), grown[victim][2]) + check("the call adds a declaration to code_hot.c's preamble", hot_text(called)[0] != hot_text(grown)[0]) + after2, _, _ = split(work, called) + check("a declaration one function uses changes exactly one piece (%s)" % changed(after, after2), changed(after, after2) == [where(new)[victim]]) + plain_before, _, _ = split(os.path.join(tmp, "plain"), grown, sticky=False) + plain_after, _, _ = split(os.path.join(tmp, "plain"), called, sticky=False) + check("with A64_HOT_STICKY=0 the same declaration changes every piece", len(changed(plain_before, plain_after)) == PIECES) + + # new hot functions: beside what they call; else in the cheapest piece + friend = sorted(fns)[200] + more = dict(called) + more["f_80200000"] = (30, (friend, friend), "") + more["f_80200040"] = (30, (), "") + cost = {n: sum(chunks.fn_cost(len(function(f, *called[f]))) for f in split_hot.HEAD.findall(t)) for n, t in after2.items()} + cost[where(after2)[friend]] += chunks.fn_cost(len(function("f_80200000", *more["f_80200000"]))) # (placed first, by name) + after3, said, _ = split(work, more) + check("a new function goes to the piece it shares the most calls with", where(after3)["f_80200000"] == where(after2)[friend]) + check("a new function that calls nothing hot goes to the cheapest piece", where(after3)["f_80200040"] == min(cost, key=lambda n: (cost[n], n))) + check("the others stay", all(where(after3)[f] == p for f, p in where(after2).items()) and "sticky (240 functions where they were, 2 new)" in said) + check("two new functions change two pieces at most", len(changed(after2, after3)) <= 2) + runs = run_both(tmp, cc, os.path.join(work, "sa", "gen"), more, "sticky") if cc else 0 + + # a function that left the hot list + fewer = dict(more) + gone = sorted(fns)[5] + del fewer[gone] + after4, _, _ = split(work, fewer) + check("a function that left the hot list changes its piece and its callers' alone", + all(where(after4)[f] == p for f, p in where(after3).items() if f != gone) and where(after3)[gone] in changed(after3, after4)) + + # past the slack: the fresh split + huge = dict(fewer) + for k in range(6): + huge["f_8030%04x" % (0x40 * k)] = (9000, (friend,), "") + after5, said, _ = split(work, huge) + fresh, _, _ = split(os.path.join(tmp, "fresh"), huge) + check("past %.2f times a fresh split's costliest piece the split is the fresh one: %s" % (split_hot.STICKY_SLACK, said.splitlines()[0]), + "fresh (kept where they were, the costliest piece is" in said and where(after5) == where(fresh)) + check("the pieces file follows the fresh split", split_hot.read_pieces(map_file) == {f: int(n[len("code_hot_"):-2]) for f, n in where(after5).items()}) + again, said, _ = split(work, huge) + check("the next split keeps the fresh one's pieces", again == after5 and "sticky (" in said) + + # another number of pieces, or a file that is not the splitter's + four, said, _ = split(work, huge, pieces=4) + four_fresh, _, _ = split(os.path.join(tmp, "four"), huge, pieces=4) + check("a pieces file for another number of pieces is not kept to", "another number of pieces" in said and where(four) == where(four_fresh)) + with open(map_file, "w") as f: + f.write("not a pieces file\n") + check("a file that is not the splitter's reads as none", split_hot.read_pieces(map_file) == {}) + junk, said, _ = split(work, huge, pieces=4) + check("and the split is the fresh one", "fresh (no pieces file)" in said and where(junk) == where(four_fresh)) + + # the rule by itself: the slack is measured against the fresh split + bodies = {n: function(n, *more[n]) for n in more} + edges = split_hot.call_edges(bodies) + fresh_bins, _ = split_hot.fresh_split(bodies, PIECES, edges) + one_piece = {n: 0 for n in bodies} + bins, why = split_hot.sticky_split(bodies, PIECES, one_piece, edges, fresh_bins) + check("everything in one piece is past the slack", bins is fresh_bins and why.startswith("fresh (kept where they were")) + check("no pieces file, the fresh split", split_hot.sticky_split(bodies, PIECES, {}, edges, fresh_bins)[0] is fresh_bins) + check("STICKY_SLACK is 1.15", split_hot.STICKY_SLACK == 1.15) + return len(fns), runs + + +def check_sources(): + with open(os.path.join(ROOT, "nx2", "make_game.py"), encoding="utf-8") as f: + make_game = f.read() + check("nx2/make_game.py names hot_pieces.txt (it must stay out of the lift key: any map gives the same functions)", + "hot_pieces.txt" not in make_game) + check("nx2/make_game.py's lift_key no longer takes the A64_ variables: A64_HOT_STICKY is not in the key", + 'if k.startswith(("A64_", "OOT_", "RX_"))' in make_game) + with open(os.path.join(ROOT, "nx2", "split_hot.py"), encoding="utf-8") as f: + src = f.read() + for line in (' return os.environ.get("A64_HOT_STICKY", "1").strip() != "0"\n', + " bins, how = sticky_split(bodies, pieces, read_pieces(pieces_file(gen)), edges, bins)\n", + " head = piece_preamble(preamble, {n for name in names for n in NAME.findall(bodies[name])}) if keep else preamble\n"): + check("nx2/split_hot.py no longer has the line %r" % line.strip(), src.count(line) == 1) + with open(os.path.join(ROOT, "docs", "development.md"), encoding="utf-8") as f: + check("docs/development.md: A64_HOT_STICKY is not in the switches table", "| `A64_HOT_STICKY=0` |" in f.read()) + + +def main(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--cc") + a = p.parse_args() + check_sources() + cc = shutil.which(a.cc) if a.cc else shutil.which("clang") + with tempfile.TemporaryDirectory(prefix="triforce-sticky-hot-") as tmp: + n, runs = check_all(tmp, cc) + print("PASS: %d made-up hot functions: v0.6.0's pieces with A64_HOT_STICKY=0 and with no pieces file; one piece changes for a " + "grown function or a new declaration; new functions beside their callees; the fresh split past the slack; %s" + % (n, "the sticky pieces and the whole file built and run from the same %d random states, every register the same" % runs + if cc else "the compile skipped: no clang")) + + +if __name__ == "__main__": + main() diff --git a/tests/test_sweep_walk.py b/tests/test_sweep_walk.py index 6566600..40681be 100644 --- a/tests/test_sweep_walk.py +++ b/tests/test_sweep_walk.py @@ -44,7 +44,7 @@ def check(tex, dev): "if (!sweep || sweep_all || kv.second->gpu_written || kv.second->let_go) order.push_back({kv.second->last_frame, kv.second.get()});", # (under the budget the loop takes only what was drawn over or let go early) "\n if (sweep && !overwritten && !gone_early) continue;", - "const bool overwritten = shifted || drawn_over(tex);", + "const bool overwritten = shifted || drawn_over(tex) || stale;", "const bool gone_early = let_go_short && tex->let_go && !tex->gpu_written &&", # (drawn over: only a texture the GPU drew) " const auto drawn_over = [&](const Texture* tex) {\n if (!tex->gpu_written) return false;", diff --git a/tests/test_taau_env_lookup.py b/tests/test_taau_env_lookup.py index 48832aa..ac2648e 100644 --- a/tests/test_taau_env_lookup.py +++ b/tests/test_taau_env_lookup.py @@ -19,8 +19,10 @@ def block(source, marker): return source[start:start + re.search(r'^}', source[start:], re.M).end()] -def fixture(source): - return (PREFIX + block(source, 'bool taau_key(uint64_t key) {') + '\n' + +def fixture(source, rule): + # ((Triforce, 11-22, GPUWORK-20) taau_stand_in asks upscale_off.h which dispatches it stands in for: the header itself) + return (PREFIX.replace('/*UPSCALE_OFF_H*/', rule.replace('#pragma once', '')) + + block(source, 'bool taau_key(uint64_t key) {') + '\n' + block(source, 'bool taau_stand_in(VkCommandBuffer cmd,') + '\n' + SUFFIX) @@ -42,15 +44,24 @@ static const char* fixture_getenv(const char* name) { namespace Shader { namespace Backend { namespace SPIRV { static bool mma_software = true; } } } using VkCommandBuffer = int; using VkImageView = int; -constexpr int VK_IMAGE_VIEW_TYPE_2D=2, VK_FORMAT_B10G11R11_UFLOAT_PACK32=7; -enum class ViewUse { Storage }; +constexpr int VK_IMAGE_VIEW_TYPE_2D=2, VK_FORMAT_B10G11R11_UFLOAT_PACK32=7, VK_FORMAT_UNDEFINED=0, VK_FORMAT_E5B9G9R9_UFLOAT_PACK32=9; +enum class ViewUse { Storage, Sample }; struct Desc { unsigned count=1; int format=23; }; -struct Info { std::vector texture_descriptors, image_descriptors; }; +struct Info { std::vector texture_descriptors, image_descriptors; bool uses_cooperative_matrix=false; }; +/*UPSCALE_OFF_H*/ +/* ((Triforce, 11-22, GPUWORK-20) the upscaler Off, and the two gameplay networks by key, as vk_draw.cpp has them) */ +static bool off_now; +static bool upscale_off_on(){return off_now;} +/* ((Triforce, 11-22, DESC-30) the lab's "diagold" phases, vk_draw.cpp: never here, so taau_key answers from its parsed list) */ +static bool dispatch_diag_old(){return false;} +static bool gameplay_network(uint64_t key){return key==0xfd029a317a9a4b94ull || key==0x9dc98871eb7fc101ull;} struct Stage { Info info; }; struct NvnState {}; struct Program { uint64_t key; std::vector stages; }; struct Texture { int format=VK_FORMAT_B10G11R11_UFLOAT_PACK32; }; static Texture texture; +static Texture scene_texture{VK_FORMAT_E5B9G9R9_UFLOAT_PACK32}; /* (the network's input, asked for only for a network taken by its shape) */ +static int sample_view_calls; static struct { bool push_descriptor=false; } dev; static bool vkCmdPushDescriptorSetKHR=true; struct ImageInfo { VkImageView imageView; }; @@ -70,7 +81,8 @@ static uint32_t handle_of(const NvnState&,const Stage&,const Desc&,uint32_t inde ++handle_calls;return index==0?handle_value:0; } static int image_format(int f){return f+100;} -static VkImageView texture_view_of(uint32_t handle,int type,ViewUse use,int format,Texture** out,uint32_t* level){ +static VkImageView texture_view_of(uint32_t handle,int type,ViewUse use,int format,Texture** out,uint32_t* level=nullptr){ + if(use==ViewUse::Sample){ ++sample_view_calls; if(handle==5 && type==VK_IMAGE_VIEW_TYPE_2D && format==VK_FORMAT_UNDEFINED) *out=&scene_texture; (void)level; return 98; } ++view_calls; if(handle!=5 || type!=VK_IMAGE_VIEW_TYPE_2D || use!=ViewUse::Storage || format!=123) return 0; *out=resolve_image?&texture:nullptr;*level=7;return 99; @@ -102,6 +114,50 @@ int main(int argc,char** argv){ else if(!strcmp(argv[1],"disabled"))configured_keys="-",expected=false; else if(!strcmp(argv[1],"custom"))configured_keys="0123456789abcdef",target=0x0123456789abcdefull; else if(!strcmp(argv[1],"prefixes"))configured_keys="abcd,fd029a"; + else if(!strcmp(argv[1],"off")){ + /* ((Triforce, 11-22, GPUWORK-20) the upscaler Off on a card that runs the networks: none is listed, every gameplay + network is stood in for with the plain stretch, anything else is left alone */ + Shader::Backend::SPIRV::mma_software=false; + dev.push_descriptor=true;dw().imgs={{11},{44}}; + NvnState state{};Stage stage;stage.info.texture_descriptors={{1,23}};stage.info.image_descriptors={{1,23}}; + auto run=[&](uint64_t key,bool matrix){Program q{key,{}};stage.info.uses_cooperative_matrix=matrix;q.stages.push_back(stage); + forwarded_temporal=true;forwarded_text.clear();return taau_stand_in(19,state,q);}; + const uint64_t networks[]={0xfd029a317a9a4b94ull,0x9dc98871eb7fc101ull}; + const uint64_t shares[]={0x6676619641320ba7ull,0xf10782f9e58b6a7cull,0x72ba090c70fe2614ull,0x0775215f8da720e6ull}; + for(uint64_t k:networks)check(!run(k,true) && !upscale_calls,"with the upscaler not Off, a card that runs the networks stands in for none"); + for(uint64_t k:shares)check(!run(k,true) && !upscale_calls,"with the upscaler not Off, a share's network runs"); + check(sample_view_calls==0,"with the upscaler not Off, no network's input is looked up"); + off_now=true; + int calls=0; + for(uint64_t k:networks){ + check(run(k,true) && upscale_calls==++calls,"Off: a gameplay network by key is stood in for"); + check(!forwarded_temporal && forwarded_text=="the upscaler Off" && forwarded_key==k,"Off: the stretch is the plain one, no FXAA and no history bound"); + check(hudless_key==k,"Off: the picture is still taken for frame generation"); + } + check(sample_view_calls==0,"Off: a network known by key needs no look at its input"); + for(uint64_t k:shares)check(run(k,true) && upscale_calls==++calls && !forwarded_temporal,"Off: a share's network (by its shape) is stood in for"); + check(sample_view_calls==4,"Off: a network taken by its shape has its input's format looked at"); + check(!run(0x1111111111111111ull,false) && upscale_calls==calls,"Off: a program that is no network is left alone"); + scene_texture.format=VK_FORMAT_B10G11R11_UFLOAT_PACK32; + check(!run(shares[0],true) && upscale_calls==calls,"Off: a network of that shape whose input is not the scene's format is left alone"); + scene_texture.format=VK_FORMAT_E5B9G9R9_UFLOAT_PACK32; + stage.info.texture_descriptors={{1,23},{1,23}};dw().imgs={{11},{12},{44}}; + check(!run(shares[0],true) && upscale_calls==calls,"Off: a network sampling two textures is left alone"); + check(run(networks[1],true) && upscale_calls==calls+1,"Off: a network known by key is taken whatever its shape"); + printf("%s failures=%u eligible=%d\n",argv[1],failures,upscale_calls); + return failures?1:0; + } + else if(!strcmp(argv[1],"off-software")){ + /* (Off where the first network is already stood in for: Off's plain stretch, not the stand-in's FXAA) */ + dev.push_descriptor=true;dw().imgs={{11},{44}};off_now=true; + NvnState state{};Stage stage;stage.info.texture_descriptors={{1,23}};stage.info.image_descriptors={{1,23}}; + Program q{0xfd029a317a9a4b94ull,{}};q.stages.push_back(stage); + check(taau_stand_in(19,state,q) && !forwarded_temporal && forwarded_text=="the upscaler Off","Off wins over the listed stand-in's FXAA"); + off_now=false; + check(taau_stand_in(19,state,q) && forwarded_temporal && forwarded_text=="fd029a317a9a4b94","not Off: the listed stand-in as before"); + printf("%s failures=%u eligible=%d\n",argv[1],failures,upscale_calls); + return failures?1:0; + } else if(strcmp(argv[1],"software"))return 91; check(taau_key(target)==expected && !taau_key(0),"existing key matcher unchanged"); NvnState state{};Program program{target,{}};Stage stage; @@ -154,7 +210,8 @@ def main(): p.add_argument('--emit-only', type=Path) p.add_argument('--cxx-arg', action='append', default=[], help='extra compiler/linker flag (repeatable)') args = p.parse_args() - cpp = fixture((args.source / 'gpu/vk/vk_draw.cpp').read_text(encoding='utf-8')) + cpp = fixture((args.source / 'gpu/vk/vk_draw.cpp').read_text(encoding='utf-8'), + (args.source / 'gpu/vk/upscale_off.h').read_text(encoding='utf-8')) if args.emit_only: with args.emit_only.open('x', encoding='utf-8') as f: f.write(cpp) @@ -171,13 +228,13 @@ def main(): command = [args.cxx, '-std=c++17', '-Wall', '-Wextra', '-Werror', *args.cxx_arg, str(source), '-o', str(exe)] subprocess.run(command, cwd=folder, check=True) failed = False - for case in ('software', 'hardware', 'no-f16', 'empty', 'disabled', 'custom', 'prefixes'): + for case in ('software', 'hardware', 'no-f16', 'empty', 'disabled', 'custom', 'prefixes', 'off', 'off-software'): result = subprocess.run([str(exe), case], cwd=folder) print(case, 'exit', result.returncode, flush=True) failed |= result.returncode != 0 if failed: raise SystemExit(1) - print('PASS: seven key configurations; rejected lookups removed, eligible per-call forwarding preserved') + print('PASS: seven key configurations; rejected lookups removed, eligible per-call forwarding preserved; the upscaler Off') if __name__ == '__main__': diff --git a/tests/test_target_image_pool.py b/tests/test_target_image_pool.py index fad4fd9..fe1370c 100644 --- a/tests/test_target_image_pool.py +++ b/tests/test_target_image_pool.py @@ -47,7 +47,8 @@ class TargetImagePoolTests(unittest.TestCase): self.assertIn(part, rule, part) cap = block(self.tex, "uint64_t pool_cap()") self.assertIn('getenv("RX_VK_TARGET_POOL_MB")', cap) - self.assertIn(": 256", cap) + # ((11-21, the performance audit's TEX-29) with no switch a twelfth of the allowance, 256 to 768: target_pool.h) + self.assertIn("target_pool::default_mb(", cap) self.assertIn("uint64_t pool_key = 0;", self.hdr) def test_a_texture_made_for_a_target_is_one_and_its_image_comes_from_the_targets_blocks(self): @@ -95,11 +96,13 @@ class TargetImagePoolTests(unittest.TestCase): def test_the_trim_is_bounded_a_present_and_gives_back_idle_and_excess_images(self): trim = block(self.tex, "size_t pool_trim()") - self.assertIn("pool_pressed ? 0 : pool_cap()", trim) - self.assertIn("pool_pressed ? 64 : 16", trim) - self.assertIn("frame_no - v[i].since <= 120", trim) + # ((11-21, TEX-29) the bound, the count and the wait are target_pool.h's, the walk the age list's from its front: + # tests/test_target_pool_rules.py has the rules and the order) + self.assertIn("target_pool::cap_for(pool_cap(), pool_tier, tiers, floor_kept)", trim) + self.assertIn("target_pool::trim_most(pool_tier, floor_kept)", trim) + self.assertIn("target_pool::idle_out(frame_no, it->since, idle,", trim) self.assertIn("pool_bytes <= cap", trim) - self.assertIn("pool_destroy(v[i]);", trim) + self.assertIn("pool_destroy(*it);", trim) destroy = block(self.tex, "void pool_destroy(Pooled& p)") self.assertIn("vkDestroyImageView", destroy) self.assertIn("destroy_image(p.image, p.allocation);", destroy) diff --git a/tests/test_target_overlap.cpp b/tests/test_target_overlap.cpp index 0aba1ef..2d9a855 100644 --- a/tests/test_target_overlap.cpp +++ b/tests/test_target_overlap.cpp @@ -21,5 +21,35 @@ int main() { CHECK("less than an eighth is not", !shifted_over(a, a + 8 * 4096, 100, a + 7 * 4096 + 1, a + 20 * 4096, 101)); CHECK("apart is not", !shifted_over(a, a + size, 100, a + size, a + 2 * size, 101)); CHECK("empty ranges are not", !shifted_over(a, a, 100, a - 1, a + size, 101) && !shifted_over(a, a + size, 100, b, b, 101)); + { // ((Triforce, 11-21, TEX-30) the others' parts added up, each byte once + using mkwvk::target_overlap::ShiftedOver; + const uint64_t p = 4096, t = a, n = 16 * p; // (a target of 16 pages) + ShiftedOver s{t, t + n, 100}; + s.add(t - 4 * p, t + p, 101); // (a sixteenth over its head) + CHECK("a sixteenth alone is not an eighth", !s.an_eighth() && s.covered() == p); + CHECK("nor is it one by one", !shifted_over(t, t + n, 100, t - 4 * p, t + p, 101) && !shifted_over(t, t + n, 100, t + 15 * p, t + 20 * p, 101)); + s.add(t + 15 * p, t + 20 * p, 101); // (and a sixteenth over its tail) + CHECK("a sixteenth over the head and one over the tail make an eighth", s.an_eighth() && s.covered() == 2 * p); + ShiftedOver o{t, t + n, 100}; + o.add(t - p, t + p, 101), o.add(t - p / 2, t + p + p / 2, 105); // (two over its head, half a page of them shared) + CHECK("parts that overlap count once", o.covered() == p + p / 2 && !o.an_eighth()); + o.add(t + 15 * p + p / 2, t + 20 * p, 101); // (and half a page over its tail) + CHECK("and a part elsewhere adds to them", o.covered() == 2 * p && o.an_eighth()); + ShiftedOver w{t, t + n, 100}; + w.add(t - p, t + 8 * p, 100), w.add(t - p, t + 8 * p, 40); + CHECK("one written by its last use counts nothing", w.covered() == 0 && !w.an_eighth()); + w.add(t, t + 8 * p, 101); + CHECK("the same start is drawn_over's case", w.covered() == 0); + w.add(t + p, t + 9 * p, 101); + CHECK("a view inside it is not a new target", w.covered() == 0); + w.add(t + n, t + 2 * n, 101); + CHECK("apart is not", w.covered() == 0 && !w.an_eighth()); + ShiftedOver whole{t + p, t + n, 100}; + whole.add(t, t + 2 * n, 101); + CHECK("one covering it whole is", whole.an_eighth() && whole.covered() == n - p); + ShiftedOver empty{t, t, 100}; + empty.add(t - p, t + p, 101); + CHECK("an empty range is not", !empty.an_eighth()); + } std::puts("target overlap: all checks passed"); } diff --git a/tests/test_target_overlap.py b/tests/test_target_overlap.py index 473cce4..4726682 100644 --- a/tests/test_target_overlap.py +++ b/tests/test_target_overlap.py @@ -13,8 +13,14 @@ def check_source(): """The eviction pass still uses the rule, and the switch that turns it off is still read""" src = (ROOT / "gpu/vk/vk_texture.cpp").read_text(encoding="utf-8") for needle in ('#include "target_overlap.h"', "target_overlap::shifted_over(", 'getenv("RX_VK_DEAD_TARGETS")', - "const bool overwritten = shifted || drawn_over(tex);", "return !drawn_over(tex) && !drawn_over_shifted(tex);", - 'getenv("RX_VK_DEAD_MIN_UNUSED")', "if (tex->last_value > completed)"): + "const bool overwritten = shifted || drawn_over(tex) || stale;", "return !drawn_over(tex) && !drawn_over_shifted(tex);", + 'getenv("RX_VK_DEAD_MIN_UNUSED")', "if (tex->last_value > completed)", + # ((Triforce, 11-21, the performance audit's TEX-30) the others' parts added up, in the order of their starts; + # one unused present; the switches back + 'static const bool dead_sum = !(getenv("RX_VK_DEAD_SUM") && !atoi(getenv("RX_VK_DEAD_SUM")));', + "target_overlap::ShiftedOver over{lo, hi, tex->last_frame};", + "over.add((*it)->gpu, (*it)->gpu + (*it)->guest_size, (*it)->last_written);", "if (over.an_eighth()) return true;", + 'getenv("RX_VK_DEAD_MIN_UNUSED") ? (uint64_t)std::max(1, atoi(getenv("RX_VK_DEAD_MIN_UNUSED"))) : 1;'): if needle not in src: raise SystemExit("vk_texture.cpp no longer has: " + needle) diff --git a/tests/test_target_pool_rules.py b/tests/test_target_pool_rules.py new file mode 100644 index 0000000..8d8cb07 --- /dev/null +++ b/tests/test_target_pool_rules.py @@ -0,0 +1,193 @@ +"""The render targets' pool is bound by the allowance, trimmed oldest first, and held in tiers (gpu/vk/target_pool.h, +vk_texture.cpp pool_cap, pool_take, pool_put, pool_trim, texture_frame; no game, no GPU). + +(Triforce, 11-21, the performance audit's TEX-29, after Source A's and Source B's.) The pool gives a dead render target's +image to the next target of its shape. It held a fixed 256 MB, kept every image 120 presents, was trimmed in its hash +table's order and emptied above 90% of the allowance: at 2560 x 1440 it destroyed nearly every image it was given. + +This compiles target_pool.h alone and checks the rules: allowances of 3, 7 and 12 GB give 256, 597 and 768 MB and an +unknown one 256; the tier is 1 above 90% and 2 above 97%; tier 1 halves the bound and tier 2 leaves none, with the tiers +off any tier leaves none, and with a floor kept the bound stays; a shape never taken goes after 8 presents and one taken +before after 120; 16 images a present, 64 under a tier. And it runs a model of the pool with the same rules, an age list +and the shapes' lists as vk_texture.cpp has them: over the bound the oldest goes first whatever its shape; a one-off +shape is gone after 8 presents while a shape that is taken again is still there; a take gives the newest of its shape. + +And reads the source: the bound is the switch or the rule's of the allowance; the trim walks the age list from its front +with the rule's cap, count and wait, and stops at the first image younger than the short wait; a take notes the shape; +the tier is set where the pressure is known; the switches are in the switches table. + + python -I -B tests/test_target_pool_rules.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +PROGRAM = r""" +#include +#include +#include +#include +#include +#include +#include +#include "target_pool.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +using namespace target_pool; + +// the pool as vk_texture.cpp keeps it: one list by age, each shape's entries, the shapes taken lately +struct Pooled { uint64_t key; int id; uint64_t bytes, since; }; +static std::list age; +static std::unordered_map::iterator>> by_key; +static std::unordered_set taken; +static uint64_t bytes_held, now; +static std::vector gone; +static void put(uint64_t key, int id, uint64_t bytes) { + age.push_back({key, id, bytes, now}); + by_key[key].push_back(std::prev(age.end())); + bytes_held += bytes; +} +static int take(uint64_t key) { + auto it = by_key.find(key); + if (it == by_key.end() || it->second.empty()) return -1; + const auto at = it->second.back(); + const int id = at->id; + bytes_held -= at->bytes; + age.erase(at); + it->second.pop_back(); + if (it->second.empty()) by_key.erase(it); + taken.insert(key); + return id; +} +static size_t trim(uint64_t cap, int tier_now, uint64_t idle) { + const uint64_t bound = cap_for(cap, tier_now, true, false); + const size_t most = trim_most(tier_now, false); + size_t n = 0; + for (auto it = age.begin(); it != age.end() && n < most;) { + if (bytes_held <= bound && !idle_out(now, it->since, idle, taken.count(it->key) != 0)) { + if (now - it->since <= idle) break; + ++it; + continue; + } + auto& v = by_key[it->key]; + for (auto m = v.begin(); m != v.end(); ++m) + if (*m == it) { v.erase(m); break; } + if (v.empty()) by_key.erase(it->key); + gone.push_back(it->id), bytes_held -= it->bytes, n++; + it = age.erase(it); + } + return n; +} + +int main() { + const uint64_t MB = 1ull << 20, GB = 1ull << 30; + CHECK(default_mb(3 * GB) == 256 && default_mb(7 * GB) == 597 && default_mb(12 * GB) == 768 && default_mb(0) == 256); + CHECK(default_mb(31419 * MB) == 768 && default_mb(8 * GB) == 682 && default_mb(1 * GB) == 256); + CHECK(tier(true, 89, 100) == 0 && tier(true, 90, 100) == 0 && tier(true, 91, 100) == 1 && tier(true, 97, 100) == 1); + CHECK(tier(true, 98, 100) == 2 && tier(false, 99, 100) == 0 && tier(true, 99, 0) == 0); + CHECK(cap_for(600 * MB, 0, true, false) == 600 * MB && cap_for(600 * MB, 1, true, false) == 300 * MB && cap_for(600 * MB, 2, true, false) == 0); + CHECK(cap_for(600 * MB, 1, false, false) == 0 && cap_for(600 * MB, 2, false, false) == 0 && cap_for(600 * MB, 0, false, false) == 600 * MB); + CHECK(cap_for(600 * MB, 1, true, true) == 600 * MB && cap_for(600 * MB, 2, false, true) == 600 * MB); // (a floor kept: not given back) + CHECK(trim_most(0, false) == 16 && trim_most(1, false) == 64 && trim_most(2, false) == 64 && trim_most(2, true) == 16); + CHECK(kIdle == 8 && kIdleTaken == 120); + CHECK(!idle_out(108, 100, 8, false) && idle_out(109, 100, 8, false)); // (a one-off shape: after 8 presents) + CHECK(!idle_out(220, 100, 8, true) && idle_out(221, 100, 8, true)); // (a shape taken before: after 120) + CHECK(!idle_out(340, 100, 240, true) && idle_out(341, 100, 240, true) && idle_out(341, 100, 240, false)); // (a longer wait asked for) + + // the model: over the bound the oldest goes first, whatever its shape + now = 1; + put(0xA, 1, 100 * MB), put(0xB, 2, 100 * MB), put(0xA, 3, 100 * MB), put(0xC, 4, 100 * MB); + CHECK(trim(256 * MB, 0, 8) == 2 && gone.size() == 2 && gone[0] == 1 && gone[1] == 2 && bytes_held == 200 * MB); + CHECK(by_key.count(0xB) == 0 && by_key[0xA].size() == 1); + // a take gives the newest of its shape + put(0xA, 5, 10 * MB); + CHECK(take(0xA) == 5 && take(0xA) == 3 && take(0xA) == -1 && take(0xD) == -1); + CHECK(bytes_held == 100 * MB && age.size() == 1); + // the waits: shape 0xA is taken again, 0xC and 0xE never were + gone.clear(); + now = 10; + put(0xA, 6, 10 * MB), put(0xE, 7, 10 * MB); + now = 18; + CHECK(trim(256 * MB, 0, 8) == 1 && gone[0] == 4); // (0xC, in since present 1: long past 8; the others are 8 old) + now = 19; + CHECK(trim(256 * MB, 0, 8) == 1 && gone[1] == 7); // (0xE, never taken, at 9 presents; 0xA's waits on) + now = 130; + CHECK(trim(256 * MB, 0, 8) == 0 && age.size() == 1); + now = 131; + CHECK(trim(256 * MB, 0, 8) == 1 && gone[2] == 6 && age.empty() && by_key.empty() && bytes_held == 0); + // a tier: half the bound, then none; 64 a present + gone.clear(); + for (int i = 0; i < 100; i++) put(0x100 + (uint64_t)i, i, 4 * MB); + CHECK(trim(256 * MB, 1, 8) == 64 && bytes_held == 36 * 4 * MB); // (400 MB held, the bound 128: but 64 a present) + CHECK(trim(256 * MB, 1, 8) == 4 && bytes_held == 128 * MB); // (to half the bound) + CHECK(trim(256 * MB, 2, 8) == 32 && bytes_held == 0 && age.empty()); + for (size_t i = 1; i < gone.size(); i++) CHECK(gone[i] == gone[i - 1] + 1); // (always the oldest next) + return fails ? 1 : 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + tex = (VK / "vk_texture.cpp").read_text(encoding="utf-8") + + check('#include "target_pool.h"' in tex and "#include " in tex, "vk_texture.cpp does not use the rules") + cap = block(tex, "uint64_t pool_cap() {") + check('getenv("RX_VK_TARGET_POOL_MB")' in cap and "target_pool::default_mb(known ? allowed : 0)" in cap, + "the bound is not the switch, or the rule's of the allowance") + take = block(tex, "bool pool_take(") + check("const std::list::iterator at = it->second.back();" in take and "pool_age.erase(at);" in take and + "if (pool_taken.insert(key).second) {" in take and "target_pool::kTakenKeys" in take, + "a take does not give the newest of its shape, leaves it in the age list, or does not note the shape as taken") + put = block(tex, "void pool_put(Dead& d) {") + check("pool_age.push_back(Pooled{d.pool_key," in put and "target_pool[d.pool_key].push_back(std::prev(pool_age.end()));" in put, + "an image put in is not at the back of the age list and of its shape's") + trim = block(tex, "size_t pool_trim() {") + at = [trim.index(p) for p in ( + "const uint64_t cap = target_pool::cap_for(pool_cap(), pool_tier, tiers, floor_kept);", + "const size_t most = target_pool::trim_most(pool_tier, floor_kept);", + "for (auto it = pool_age.begin(); it != pool_age.end() && n < most;) {", + "if (pool_bytes <= cap && !target_pool::idle_out(frame_no, it->since, idle, pool_taken.count(it->key) != 0)) {", + "if (frame_no - it->since <= idle) break;", "pool_destroy(*it);", "it = pool_age.erase(it);")] + check(at == sorted(at), "the trim does not walk the age list from its front with the rule's bound, count and wait") + check('getenv("RX_VK_TARGET_POOL_TRIM")' in trim and 'getenv("RX_VK_POOL_IDLE")' in trim and "target_pool::kIdle" in trim, + "the trim's switches are not read") + check("pool_tier = target_pool::tier(known, pressure, allowed);" in tex, "the tier is not set where the pressure is known") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`RX_VK_TARGET_POOL_MB=`, `RX_VK_POOL_IDLE=`, `RX_VK_TARGET_POOL_TRIM=0`" in dev, + "docs/development.md: the pool's switches are not in the table") + + with tempfile.TemporaryDirectory(prefix="target-pool-") as tmp: + c, exe = Path(tmp) / "pool.cpp", Path(tmp) / "pool.exe" + c.write_text(PROGRAM, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(VK), str(c), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + check(r.returncode == 0, "the rules as made:\n" + r.stdout) + print("PASS: the render targets' pool is bound by the allowance, trimmed oldest first, and held in tiers") + + +if __name__ == "__main__": + main() diff --git a/tests/test_tex_bc1.cpp b/tests/test_tex_bc1.cpp index 796da12..c941150 100644 --- a/tests/test_tex_bc1.cpp +++ b/tests/test_tex_bc1.cpp @@ -206,15 +206,65 @@ static void test_astc_authored() { CHECK(astc_opaque::block(plain(8).set(0, 11, 0x66).b, 12, 12)); // (8 x 5 weights of 2 bits: 80 bits) CHECK(!astc_opaque::block(plain(8).set(0, 11, 0x466).b, 12, 12)); // (two planes of 8 x 5: 80 weights, over 64) CHECK(!astc_opaque::block(plain(8).set(10, 1, 1).set(11, 2, 3).b, 8, 8)); // (two planes and four partitions) - // several partitions: one shared mode (bits 23 and 24 zero, the mode in bits 25 to 28), or a mode each (not looked for) + // several partitions: one shared mode (bits 23 and 24 zero, the mode in bits 25 to 28) for (unsigned cem = 0; cem < 16; cem++) for (unsigned parts = 1; parts < 4; parts++) { Block k = plain(15); k.set(11, 2, parts).set(23, 2, 0).set(25, 4, cem); CHECK(astc_opaque::block(k.b, 8, 8) == is_opaque_mode(cem)); - k.set(23, 2, 1 + cem % 3); - CHECK(!astc_opaque::block(k.b, 8, 8)); } + // ((11-21, the performance audit's TEX-50) a mode a partition, authored as the decoder reads it: bits 23 and 24 a number + // from 1 to 3, then a class bit a partition and two mode bits a partition, their first four in bits 25 to 28 and the rest + // right below the weights (plain's weights are 32 bits: at 96 less their count). Every class and mode of every partition: + // opaque only when each partition's mode is) + for (unsigned parts = 2; parts <= 4; parts++) { + const unsigned extra = 3 * parts - 4; + long opaque = 0, all = 0; + for (unsigned base = 1; base <= 3; base++) + for (uint32_t v = 0; v < (1u << (3 * parts)); v++, all++) { + Block k = plain(15); + k.set(11, 2, parts - 1).set(23, 2, base).set(25, 4, v & 15).set(128 - 32 - extra, extra, v >> 4); + bool want = true; + for (unsigned i = 0; i < parts; i++) { + const unsigned cls = base - 1 + ((v >> i) & 1), mode = (cls << 2) | ((v >> (parts + 2 * i)) & 3); + want = want && is_opaque_mode(mode); + } + CHECK(astc_opaque::block(k.b, 8, 8) == want); + opaque += want; + astc_opaque::parts_too = false; // (RX_VK_TEX_BC1_PARTS=0: refused unread, as before) + CHECK(!astc_opaque::block(k.b, 8, 8)); + astc_opaque::parts_too = true; + } + CHECK(opaque > 0 && opaque < all); + } + { // (one partition with alpha among opaque ones: no. Three partitions over base 2, classes 0 1 1: modes 4, 8 and 8) + Block k = plain(15); + k.set(11, 2, 2).set(23, 2, 2).set(25, 4, 0b0110).set(128 - 32 - 5, 5, 0); + CHECK(!astc_opaque::block(k.b, 8, 8)); // (partition 0 is mode 4, luminance and alpha) + k.set(25, 4, 0b0111); // (classes 1 1 1: modes 8, 8 and 8) + CHECK(astc_opaque::block(k.b, 8, 8)); + k.set(128 - 32 - 5, 5, 0b00011); // (the last partition's mode bits 3: mode 11, HDR) + CHECK(!astc_opaque::block(k.b, 8, 8)); + } + { // (the decoder's last error with these bits counted: 8 x 4 weights of 3 bits are 96 bits, block mode 0x57; with 29 bits + // read and 2 more for two partitions 127, which fits; 5 or 8 more for three or four wrap, and the block is an error, + // (0, 0, 0, 0) on the GPU, whatever its modes say) + for (unsigned parts = 1; parts <= 4; parts++) + for (unsigned base = 0; base <= 3; base++) { + Block k = plain(8); + k.set(0, 11, 0x57).set(11, 2, parts - 1); + if (parts > 1) k.set(23, 6, base); // (classes and mode bits all 0: every partition mode (base - 1) * 4, or 0 shared) + for (unsigned i = 29; i < 32; i++) k.set(i, 1, 0); + const unsigned extra = parts > 1 && base ? 3 * parts - 4 : 0; + const bool fits = 96 + (parts == 1 ? 17 : 29) + extra <= 128; + if (extra && fits) k.set(128 - 96 - extra, extra, 0); // (where they do not fit they would lie over bits 23 and 24) + const unsigned mode = parts == 1 ? 8 : base ? (base - 1) << 2 : 0; + CHECK(astc_opaque::block(k.b, 8, 8) == (fits && is_opaque_mode(mode))); + } + Block k = plain(8); + k.set(0, 11, 0x57).set(11, 2, 2).set(23, 6, 1); // (three partitions, a mode each, every mode 0: opaque but for the wrap) + CHECK(!astc_opaque::block(k.b, 8, 8)); + } // a void extent: the constant's alpha, and the decoder's two errors Block v; v.set(0, 9, 0x1FC).set(9, 1, 0).set(10, 2, 3); diff --git a/tests/test_tex_bc1.py b/tests/test_tex_bc1.py index 66e4a66..81bbf53 100644 --- a/tests/test_tex_bc1.py +++ b/tests/test_tex_bc1.py @@ -202,8 +202,10 @@ def check_menu_and_docs(): check(re.search(r"\bint tex_bc1 = 1;", header), "Opaque textures as BC1 is not on by default") check("if (version < 5) {" in function(menu, "void menu_init() {") and "opts.tex_bc1 = 1" in function(menu, "void menu_init() {"), "an options file from before options_version 5 does not take Opaque textures as BC1 on") - check("int tex_bc1_at_start = 0;" in menu and "tex_bc1_at_start = opts.tex_bc1;" in function(menu, "void menu_init() {"), - "menu_init does not keep the Opaque textures as BC1 row the game started with") + # ((11-21, the performance audit's TEX-50) the row's own default until the file is read: a first start, with no file, takes it) + check("int tex_bc1_at_start = Options().tex_bc1;" in menu and "static_assert(Options().tex_bc1 == 1," in menu and + "tex_bc1_at_start = opts.tex_bc1;" in function(menu, "void menu_init() {"), + "the game does not start with the Opaque textures as BC1 row's default, or menu_init does not keep the row it read") rows = menu[menu.index("Row rows[] = {"):] rows = rows[:rows.index("\n};")] row = re.search(r'\{TAB_DEBUG, "Opaque textures as BC1", &opts\.tex_bc1, 0, 1, 1, tx_onoff,.*?hn_tex_bc1, true, true\},', rows, re.S) diff --git a/tests/test_tex_count_blocks.py b/tests/test_tex_count_blocks.py new file mode 100644 index 0000000..2ebf5bc --- /dev/null +++ b/tests/test_tex_count_blocks.py @@ -0,0 +1,88 @@ +"""An image in the game's own BC or ASTC blocks is counted by whole blocks (gpu/vk/tex_count.h, vk_texture.cpp create; +no game, no GPU). + +(Triforce, 11-22, the performance audit's TEX-86, after Source A's.) A texture the GPU samples as the game stores it +was counted by its texels: a byte a texel for BC (BC1 and BC4 are half that) and, for ASTC where the driver samples it, +the 16 bytes of a block for every texel: 64 times an 8 x 8 texture's size. The budget then let textures go for memory +nothing held. + +Compiles tests/tex_count_test.cpp against the header: BC1, BC4 and BC7 and ASTC 4 x 4, 8 x 8, 10 x 5 and 12 x 12 chains +with their 1 x 1 tails, a 3D volume, layers, the levels from a first one on, the 1365 x 1365 x 90 ASTC 8 x 8 array at +53.8 MiB (3,410 as it was counted), and the switch's values. + +And reads the source: create() asks the rule for the first size estimate (bytes_at) and for the budget's count, for an +image in the game's own format only (not one the renderer keeps as BC3 or BC1, x.bc), not a sparse one, before the pad +lifts the count to the allocation; the switch is read once and is in the switches table. + + python -I -B tests/test_tex_count_blocks.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def function(text, head): + start = text.index(head) + return text[start:text.index("\n}\n", start) + 3] + + +def check_source(): + tex = (VK / "vk_texture.cpp").read_text(encoding="utf-8") + check('#include "tex_count.h"' in tex, "vk_texture.cpp does not include tex_count.h") + on = function(tex, "bool count_by_blocks() {") + check("static const bool on = [] {" in on and 'tex_count::by_texels(getenv("RX_VK_COUNT_BY_TEXELS"))' in on and on.count("logf(") == 2, + "count_by_blocks no longer reads RX_VK_COUNT_BY_TEXELS once and says which count is on") + create = function(tex, "Texture* create(const Key& key") + # the first size estimate: asked while x.format is still the game's own format, for every level from `skip` on + native = create.index("const bool native_blocks = (is_bc(x.format) || is_astc(x.format)) && count_by_blocks();") + est = create.index("auto bytes_at = [&](uint32_t skip) -> uint64_t {") + asked = create.index("if (native_blocks)\n return tex_count::bytes(x.width, x.height, x.depth, skip, x.levels, x.layers, " + "(uint32_t)bw, (uint32_t)bh,\n (uint32_t)bpb) * std::max(1, d.samples);") + kept = create.index("x.bc = true") + check(native < est < asked < create.index("x.sparse = np > 0;") < kept, + "bytes_at no longer counts the game's own blocks by tex_count.h before anything is kept as BC3") + check("return px * x.layers * texel * std::max(1, d.samples);" in create[asked:kept], + "bytes_at no longer counts the other formats by their texels") + # the budget's count: after the texels' count, before BC1's own line and the pad + count = [create.index(n) for n in ( + "x.bytes = x.sparse ? 0 : px * x.layers * texel * x.samples;", + "if (!x.sparse && !x.bc && (is_bc(x.format) || is_astc(x.format)) && count_by_blocks())\n" + " x.bytes = tex_count::bytes(x.width, x.height, x.depth, 0, x.levels, x.layers, (uint32_t)bw, (uint32_t)bh, (uint32_t)bpb) *\n" + " x.samples;", + "if (x.bc1) x.bytes = tex_bc1::bytes(x.width, x.height, x.levels, x.layers);", + "if (!x.sparse && !by_estimate) x.pad = std::max(x.bytes, allocation_size(x.allocation)) - x.bytes;", + "total_bytes += x.bytes + x.pad;")] + check(count == sorted(count), "create() no longer counts the game's own blocks by tex_count.h between the texels' count and the pad") + check(create.count("count_by_blocks()") == 2, "the count by blocks is asked somewhere else in create()") + docs = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("| `RX_VK_COUNT_BY_TEXELS=1` |" in docs and "`gpu/vk/tex_count.h`" in docs, + "docs/development.md: RX_VK_COUNT_BY_TEXELS is not in the switches table") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + check_source() + with tempfile.TemporaryDirectory(prefix="tex-count-") as tmp: + exe = Path(tmp) / "tex-count.exe" + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-I" + str(VK), + str(ROOT / "tests" / "tex_count_test.cpp"), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + print(r.stdout.rstrip()) + check(r.returncode == 0, "the count by blocks: its checks failed") + print("PASS: an image in the game's own BC or ASTC blocks is counted by whole blocks") + + +if __name__ == "__main__": + main() diff --git a/tests/test_tex_pressure.py b/tests/test_tex_pressure.py index 137a634..69e4002 100644 --- a/tests/test_tex_pressure.py +++ b/tests/test_tex_pressure.py @@ -100,7 +100,7 @@ def check_source(tex, dev): # (the step is judged once it is known whether a render target drew over the texture, and before anything else of the # pass is done with it: the finalized ones' cap, the let-go count, the choice) in_order("vk_texture.cpp texture_sweep (the step)", sweep, - ["const bool overwritten = shifted || drawn_over(tex);", + ["const bool overwritten = shifted || drawn_over(tex) || stale;", "if (!sweep && !overwritten && tex_pressure::stepped(press.step, live_chosen)) continue;", "if (finalized_kept && keep_finalized(tex)) {", "if (gone_early) let_go_n++", "evict.push_back({tex,", " if (!overwritten) live_chosen++;", "press.next = frame_no + tex_pressure::kNextLook;", diff --git a/tests/test_tex_same_staged.py b/tests/test_tex_same_staged.py new file mode 100644 index 0000000..9212c9f --- /dev/null +++ b/tests/test_tex_same_staged.py @@ -0,0 +1,193 @@ +"""RX_VK_TEX_SAME's compare reads a capture's texture page from the staging, ordinary memory, when the staging holds it, +not from the capture buffer (gpu/vk/vk_memory.cpp take_capture and cap_page_bytes; no game, no GPU). + +(Triforce, 11-22, the performance audit's MEM-39, after Source A's.) A capture buffer asks for cached host memory, but a +driver can hand out write-combined memory, which the CPU reads at about 0.3 GB/s. Such a capture's texture pages are +copied into a staging block first and the shadow's copies read that; the compare of each copied texture page with its +shadow copy still read the buffer. + +This cuts cap_page_bytes out of vk_memory.cpp and runs it on random captures built the way capture_make builds them (a +texture page that is not late: into the staging, and the buffer from there; any other page: the buffer alone): + - the compare's verdict from cap_page_bytes is the verdict from the buffer, page for page (the same bytes); + - a staged page's bytes are read from the staging and not from the buffer; an unstaged one's from the buffer; + - with no staging (a cached buffer, RX_VK_SHADOW_STAGE=0) it is the old pointer exactly. + +And reads the source: the compare reads cap_page_bytes(*c, c->slots[i]) after cap_ready, the buffer only in the lab +knob's phases; the other reads of the capture buffer in take_capture are the three known ones (two of late pages, one of +unstaged pages), each marked; the first uncached buffer is logged; the lab's switch and the switches table. + + python -I -B tests/test_tex_same_staged.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + check(marker in text, "the source has no " + marker) + start = text.index(marker) + end = text.index("{", start + len(marker) - 1) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +HEAD = r""" +#include +#include +#include +#include +#include +struct Capture { + void* map = nullptr; + std::unique_ptr stage; + std::vector sidx; +}; +""" + +TAIL = r""" +static uint64_t rnd_state = 0x2545F4914F6CDD1Dull; +static uint64_t rnd() { + rnd_state ^= rnd_state << 13, rnd_state ^= rnd_state >> 7, rnd_state ^= rnd_state << 17; + return rnd_state; +} +int main() { + int fails = 0; + long staged_n = 0, unstaged_n = 0, same_n = 0, changed_n = 0; + for (int t = 0; t < 400; t++) { + const uint32_t n = 1 + (uint32_t)(rnd() % 48); + std::vector guest((size_t)n * 4096), buffer((size_t)n * 4096), shadow((size_t)n * 4096); + for (auto& b : guest) b = (uint8_t)rnd(); + shadow = guest; + Capture c; + c.map = buffer.data(); + const bool staging = t % 4 != 0; // (one capture in four has a cached buffer: no staging) + std::vector tex(n), late(n); + uint32_t tn = 0; + if (staging) c.sidx.assign(n, -1); + for (uint32_t i = 0; i < n; i++) { + tex[i] = rnd() % 3 != 0, late[i] = rnd() % 7 == 0; + if (staging && tex[i] && !late[i]) c.sidx[i] = (int32_t)tn++; + if (rnd() % 2) shadow[(size_t)i * 4096 + rnd() % 4096] ^= 0x5a; // (half the pages changed since their shadow copy) + } + if (staging && !tn) c.sidx.clear(); // (capture_make: nothing to stage, no staging) + if (!c.sidx.empty()) c.stage.reset(new uint8_t[(size_t)tn * 4096]); + for (uint32_t i = 0; i < n; i++) { // (the copy: the staging first and the buffer from there, else the buffer) + const uint8_t* src = guest.data() + (size_t)i * 4096; + if (!c.sidx.empty() && c.sidx[i] >= 0) { + uint8_t* st = c.stage.get() + (size_t)c.sidx[i] * 4096; + memcpy(st, src, 4096); + memcpy(buffer.data() + (size_t)i * 4096, st, 4096); + } else memcpy(buffer.data() + (size_t)i * 4096, src, 4096); + } + for (uint32_t i = 0; i < n; i++) { + const uint8_t* was = (const uint8_t*)c.map + (uint64_t)i * 4096; // (the read as it was) + const uint8_t* now = cap_page_bytes(c, i); + const uint8_t* sh = shadow.data() + (size_t)i * 4096; + const bool same_was = memcmp(sh, was, 4096) == 0, same_now = memcmp(sh, now, 4096) == 0; + if (same_was != same_now || memcmp(was, now, 4096) != 0) fails++, printf("FAIL capture %d page %u: the verdict or the bytes differ\n", t, i); + const bool in_stage = !c.sidx.empty() && c.sidx[i] >= 0; + if (in_stage) { + staged_n++; + if (now != c.stage.get() + (size_t)c.sidx[i] * 4096) fails++, printf("FAIL capture %d page %u: a staged page read from the buffer\n", t, i); + if (late[i] || !tex[i]) fails++, printf("FAIL the model staged a late or non-texture page\n"); + } else { + unstaged_n++; + if (now != was) fails++, printf("FAIL capture %d page %u: an unstaged page not read from the buffer\n", t, i); + } + (same_now ? same_n : changed_n)++; + } + } + if (!staged_n || !unstaged_n || !same_n || !changed_n) fails++, printf("FAIL the trials did not cover both kinds of page and both verdicts\n"); + printf("%ld staged pages read from the staging, %ld unstaged from the buffer; %ld same, %ld changed: the verdicts of the read as it was\n", + staged_n, unstaged_n, same_n, changed_n); + return fails != 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + mem = (VK / "vk_memory.cpp").read_text(encoding="utf-8") + take = block(mem, "bool take_capture(std::unique_ptr c, uint64_t value, uint64_t completed) {") + + # the compare + old = "if (memcmp(s->second.b.get(), (const uint8_t*)c->map + (uint64_t)c->slots[i] * 4096, 4096) != 0) {" + check(old not in take, "take_capture: RX_VK_TEX_SAME's compare still reads the capture buffer") + order = [take.find(s) for s in ( + 'const bool same_from_map = lab_ab::on("texsamemap");', + "cap_ready(*c, c->slots[i]);", + "const uint8_t* const cap_b = same_from_map ? (const uint8_t*)c->map + (uint64_t)c->slots[i] * 4096 : cap_page_bytes(*c, c->slots[i]);", + "if (memcmp(s->second.b.get(), cap_b, 4096) != 0) {", + "same_pages.push_back({p, gens.page_tex(p)});")] + check(min(order) >= 0 and order == sorted(order), + "take_capture: the compare does not read cap_page_bytes(*c, c->slots[i]) after cap_ready, or the lab knob is not read once a take") + check(take[order[1]:order[3]].count("cap_ready(") == 1 and "for (" not in take[order[1]:order[3]], + "take_capture: the compare's page is not the one made ready just before it") + + # the other reads of the capture buffer in take_capture: the known ones, each marked + reads = [ + ("if (was && !memcmp(was, (const uint8_t*)c->map + (uint64_t)i * 4096, 4096)) {", "a late page is never staged"), + ("memcpy(lp.bytes.data(), (const uint8_t*)c->map + (uint64_t)i * 4096, 4096);", "a late page is never staged"), + ("else pairs.push_back({slot.b.get(), (const uint8_t*)c->map + (uint64_t)c->slots[i] * 4096}), from_map++;", "unstaged pages only"), + ] + for text, mark in reads: + check(take.count(text) == 1, "take_capture: the read %r is not there once" % text) + at = take.index(text) + note = take[take.rindex("// ((Triforce, 11-22, MEM-39)", 0, at):at] + check(mark in note and note.count("\n") <= 2, "take_capture: the read %r is not marked (%s)" % (text, mark)) + # (reads: a (const uint8_t*) view of the buffer. The writes, the late pages' second copies, cast to (uint8_t*)) + n_reads = take.count("(const uint8_t*)c->map") + check(n_reads == len(reads) + 1, "take_capture reads the capture buffer in %d places, not the %d known ones and the lab knob's" + % (n_reads, len(reads))) + check("c.map" not in take.replace("f.map", ""), "take_capture reads a capture's map some other way") + + # the first uncached buffer is logged, the lab's switch + make = block(mem, "std::unique_ptr capture_make(bool behind) {") + order = [make.find(s) for s in ("c->buf_cached = host_cached(c->alloc);", "if (!c->buf_cached) {", "if (!told.exchange(true)) {", + "alloc_memory_type(c->alloc, &flags);", "if (stage_on() && !texr.empty() && !c->buf_cached) {")] + check(min(order) >= 0 and order == sorted(order), "capture_make does not log the first capture buffer that is not cached, once") + unc = block(mem, "bool cap_uncached() {") + check('getenv("RX_VK_CAP_UNCACHED")' in unc and "e && atoi(e) != 0" in unc and "static const bool on" in unc, + "RX_VK_CAP_UNCACHED is not off unless set, or is read more than once") + mk = block(mem, "bool cap_buf_make(uint64_t size, VkBuffer& buffer, VmaAllocation& alloc, void** map, bool own) {") + check("cap_uncached() ? create_buffer_uncached(size, cap_usage(), buffer, alloc, map, own)" in mk and + ": create_buffer_cached(size, cap_usage(), buffer, alloc, map, own);" in mk, "cap_buf_make is not the cached buffer unless the lab's switch is set") + check(mem.count("create_buffer_uncached(") == 1, "an uncached buffer is made elsewhere than in cap_buf_make") + vma = (VK / "vk_vma.cpp").read_text(encoding="utf-8") + cached, uncached = block(vma, "bool create_buffer_cached("), block(vma, "bool create_buffer_uncached(") + check("VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT" in cached and "SEQUENTIAL_WRITE" not in cached, + "create_buffer_cached no longer asks for cached memory") + check("VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT" in uncached, "the lab's buffer is not asked for as written in order") + stage = block(mem, "bool stage_on() {") + check('getenv("RX_VK_SHADOW_STAGE")' in stage, "stage_on no longer reads RX_VK_SHADOW_STAGE") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`RX_VK_SHADOW_STAGE=0`" in dev and "`RX_VK_CAP_UNCACHED=1`" in dev and "`texsamemap`" in dev, + "docs/development.md: RX_VK_SHADOW_STAGE=0, the lab's RX_VK_CAP_UNCACHED=1 or the knob texsamemap is not in the switches table") + + with tempfile.TemporaryDirectory(prefix="tex-same-staged-") as tmp: + src, exe = Path(tmp) / "staged.cpp", Path(tmp) / "staged.exe" + src.write_text(HEAD + block(mem, "inline const uint8_t* cap_page_bytes(const Capture& c, uint32_t slot) {") + TAIL, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", str(src), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True, timeout=120) + print(r.stdout.rstrip()) + check(r.returncode == 0, "the staged read's checks failed") + print("PASS: RX_VK_TEX_SAME's compare reads the staging when it holds the page: the same bytes, the same verdicts") + + +if __name__ == "__main__": + main() diff --git a/tests/test_tex_warm_list.py b/tests/test_tex_warm_list.py new file mode 100644 index 0000000..e439b7c --- /dev/null +++ b/tests/test_tex_warm_list.py @@ -0,0 +1,257 @@ +"""Textures made ahead by a list of what was drawn (gpu/vk/tex_warm.h, RX_VK_TEX_WARM_LIST; the performance audit's TEX-23, +after Source A's; no game, no GPU). + +The list's rules, compiled and run: the switch is off unless 1 or 2; a texture is learnt within 90 s of the game making it, +not at 91 s; its signature changes with a byte of any sample and with each field of its shape, and its place with its +address; the samples lie inside the storage; the list starts again at 32,768; a listed texture is made only while the +textures and 256 MB more fit their budget and the target pool is not being given back; the file round trips, and one cut +short, of another version or magic, with a byte changed or a count past the list's bound loads nothing. Each rule taken out +in turn (a mutation) must make the test fail. Then the sources: the look by the list runs only with the switch, before the +40% rule of the look without it; the list is asked before a slot is looked up as a draw's first look (texture_get); the look +learns nothing (warm_looking), a draw's first upload and a made-ahead texture's first draw do; only BC and ASTC storages +under 2^39 are noted, at the texture's creation, and forgotten at its release; the list is read once and saved whole every +300 presents when it changed; the switch's row is in the development guide. + + python -I -B tests/test_tex_warm_list.py [--cxx clang++] +""" +import argparse +import os +import subprocess +import tempfile +from pathlib import Path + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def in_order(text, parts, where): + at = 0 + for part in parts: + found = text.find(part, at) + if found < 0: + raise SystemExit("%s: %r is missing, or not after the part before it (update the test with the code)" % (where, part)) + at = found + len(part) + + +HEADER_TEST = r''' +#include +#include +#include +#include "tex_warm.h" +#define CHECK(x) do { if (!(x)) { std::fprintf(stderr, "FAIL:%d %s\n", __LINE__, #x); std::exit(1); } } while (0) +using namespace tex_warm; +int main() { + // (the switch: off unless 1 or 2) + CHECK(list_mode_of(nullptr) == 0 && list_mode_of("") == 0 && list_mode_of("0") == 0 && list_mode_of("3") == 0); + CHECK(list_mode_of("x") == 0 && list_mode_of("-1") == 0 && list_mode_of("1") == 1 && list_mode_of("2") == 2); + // (learnt within 90 s of the game making it, not at 91 s; a time not known, or ahead of now, learns nothing) + const int64_t s = 1000000000ll, born = 5000 * s; + CHECK(learnable(born, born) && learnable(born, born + 90 * s) && !learnable(born, born + 91 * s)); + CHECK(!learnable(0, 10 * s) && !learnable(born, born - 1)); + // (the samples: the first at the start, the last at the end, each inside the storage, 64-byte aligned) + CHECK(sample_at(1 << 20, 0) == 0 && sample_at(1 << 20, kSamples - 1) == (1 << 20) - kSampleBytes); + for (uint64_t size : {64ull, 65ull, 100ull, 512ull, 4096ull, 4100ull, 1ull << 20, 3ull << 30}) + for (int i = 0; i < kSamples; i++) + CHECK(sample_at(size, i) + sample_len(size) <= size && sample_at(size, i) % kSampleBytes == 0); + CHECK(sample_at(32, 5) == 0 && sample_len(32) == 32 && sample_len(64) == 64 && sample_len(1 << 20) == 64); + CHECK(sample_at(1 << 20, 1) > 0 && sample_at(1 << 20, 1) < sample_at(1 << 20, 2)); + // (the signature: a byte of any sample, each field of the shape; the place: the address) + uint8_t b[kSamples * kSampleBytes]; + for (size_t i = 0; i < sizeof b; i++) b[i] = (uint8_t)(i * 7); + const Shape sh{1024, 512, 1, 11, 0x9e, 1, 1, 699056}; + const uint64_t sig = signature(sh, b); + CHECK(sig == signature(sh, b)); + for (size_t i : {size_t(0), size_t(63), size_t(64), size_t(300), sizeof b - 1}) { + b[i] ^= 1; + CHECK(signature(sh, b) != sig); + b[i] ^= 1; + } + for (int f = 0; f < 8; f++) { + Shape o = sh; + switch (f) { + case 0: o.width++; break; + case 1: o.height++; break; + case 2: o.depth++; break; + case 3: o.levels++; break; + case 4: o.format++; break; + case 5: o.target++; break; + case 6: o.layers++; break; + default: o.size += 512; break; + } + CHECK(signature(o, b) != sig); + } + CHECK(place(sig, 0x12340000) != place(sig, 0x12350000) && place(sig, 0x12340000) == place(sig, 0x12340000)); + CHECK(place(sig, 0x12340000) != place(sig + 1, 0x12340000) && place(0, 0) != 0); + // (the list: once a place, and it starts again past kListMost) + std::unordered_set list; + CHECK(remember(list, 5) && !remember(list, 5) && !remember(list, 0) && list.size() == 1); + list.clear(); + for (uint64_t i = 1; i <= kListMost; i++) remember(list, i * 2 + 1); + CHECK(list.size() == kListMost && kListMost == 32768); + CHECK(remember(list, 7777777) && list.size() == 1 && list.count(7777777)); + // (a listed texture's room: the pool not given back, the textures and 256 MB more inside a budget that is set) + const uint64_t MB = 1ull << 20, GB = 1ull << 30; + CHECK(listed_room(false, 3 * GB, 3 * GB + 256 * MB) && !listed_room(false, 3 * GB, 3 * GB + 255 * MB)); + CHECK(!listed_room(true, 0, 8 * GB) && !listed_room(false, 0, 0) && !listed_room(false, 0, ~0ull)); + CHECK(!listed_room(false, 5 * GB, 4 * GB) && listed_room(false, 0, 256 * MB) && kListRoom == 256 * MB); + // (the file: a round trip; cut short, another version or magic, a byte changed, a count past the bound: nothing) + std::unordered_set in{11, 13, 0xFFFFFFFFFFFFFFFFull, 0x123456789ull}, out{99}; + std::vector f = encode(in); + CHECK(f.size() == 16 + in.size() * 8 + 8 && decode(f.data(), f.size(), out) && out == in); + CHECK(!decode(f.data(), f.size() - 1, out) && out.empty()); + out = {1}; + CHECK(!decode(f.data(), 20, out) && out.empty() && !decode(nullptr, 0, out)); + std::vector g = f; + g[8] ^= 2; // (the version) + CHECK(!decode(g.data(), g.size(), out) && out.empty()); + g = f, g[0] ^= 1; // (the magic) + CHECK(!decode(g.data(), g.size(), out)); + g = f, g[20] ^= 0x40; // (a place's byte) + CHECK(!decode(g.data(), g.size(), out) && out.empty()); + g = f, g.push_back(0); // (one byte too many) + CHECK(!decode(g.data(), g.size(), out)); + g = f; + const uint32_t big = (uint32_t)kListMost + 1; + std::memcpy(g.data() + 12, &big, 4); // (a count past the bound) + CHECK(!decode(g.data(), g.size(), out)); + std::unordered_set none; + f = encode(none); + out = {3}; + CHECK(f.size() == 24 && decode(f.data(), f.size(), out) && out.empty()); + std::puts("PASS header: the switch, 90 s, the samples, the signature and place, the list's bound, the room, the file"); + return 0; +} +''' + +MUTATIONS = [ + ("off unless 1 or 2", "return v == 1 || v == 2 ? v : 0;", "return v;"), + ("90 s, not more", "now_ns - born_ns <= kLearnNs;", "now_ns - born_ns <= kLearnNs + 1000000000;"), + ("a time not known learns nothing", "return born_ns > 0 && now_ns >= born_ns", "return now_ns >= born_ns"), + ("the samples spread over the storage", "((size - kSampleBytes) * (uint64_t)i / (kSamples - 1))", "((size - kSampleBytes) / (kSamples - 1))"), + ("the shape in the signature", "add(s.width), add(s.height), add(s.depth), add(s.levels), add(s.format), add(s.target), add(s.layers), add(s.size);", + "add(s.width), add(s.height), add(s.depth), add(s.format), add(s.target), add(s.layers), add(s.size);"), + ("the address in the place", "uint64_t h = sig ^ (gpu * 0x9E3779B97F4A7C15ull);", "uint64_t h = sig ^ (0 * gpu);"), + ("the list starts again at its bound", " if (list.size() >= kListMost) list.clear();\n", ""), + ("no room while the pool is given back", "if (pressed || !budget || budget == ~0ull) return false;", "if (!budget || budget == ~0ull) return false;"), + ("256 MB of room", "budget - texture_bytes >= kListRoom;", "budget - texture_bytes >= 0;"), + ("a file of another version", "if (head[0] != kListVersion || head[1] > kListMost", "if (head[1] > kListMost"), + ("the check of the places", " if (check != list_check(b + 16, (size_t)head[1] * 8)) return false;\n", ""), +] + + +def check_source(tex, replay, dev): + step = block(tex, "void texture_warm_step(int64_t last_frame_work) {") + in_order(step, ["if (!tex_warm::may_warm(texture_warm_on(), last_frame_work, frame_upload_ns.load(std::memory_order_relaxed), warm_frame_ns)) return;", + "if (const int mode = texture_warm_list_mode()) return warm_step_listed(mode);", + "if (!tex_warm::has_room(warm_known, warm_used, warm_allowed, total_bytes, warm_budget)) return;"], + "vk_texture.cpp texture_warm_step: the list's look with its switch, before the 40% rule") + mode = block(tex, "int texture_warm_list_mode() {") + if 'texture_warm_on() ? tex_warm::list_mode_of(getenv("RX_VK_TEX_WARM_LIST")) : 0' not in mode: + raise SystemExit("vk_texture.cpp texture_warm_list_mode: the switch is not read through tex_warm::list_mode_of with RX_VK_TEX_WARM") + listed = block(tex, "void warm_step_listed(int mode) {") + in_order(listed, ["warm_list_load_once();", "Looking() { warm_looking = true; }", "~Looking() { warm_looking = false; }", + "ParLock lk;", "Texture* tex = texture_get(&t, true);", "if (made && tex) tex->warmed = true;", + "if (warm_pending.empty() || !tex_warm::ripe(warm_pending.front().second, frame_no)) break;", + "const uint64_t p = warm_place_of(t.d, t.L, t.gpu);", "listed = p && warm_list.count(p);", + "if (mode == 1 && warm_unlisted.size() < tex_warm::kMaxPending) warm_unlisted.push_back(e.first)", + "} else if (!tex_warm::listed_room(pool_pressed, total_bytes, warm_budget)) {", + "st_warm_made += make(t);", "if (!(more = goes_on())) break;", + "while (more && mode == 1 && !warm_unlisted.empty() &&", + "tex_warm::has_room(warm_known, warm_used, warm_allowed, total_bytes, warm_budget)) {", + "st_warm_made += make(t);", "warm_frame_ns += spent;"], "vk_texture.cpp warm_step_listed") + if listed.count("texture_get(") != 1: + raise SystemExit("vk_texture.cpp warm_step_listed: a texture is looked up in a place this test does not know") + learn = block(tex, "void warm_learn(const Texture& tex) {") + in_order(learn, ["if (warm_looking || !texture_warm_list_mode() || !tex.gpu || tex.gpu >= (1ull << 39)) return;", + "const auto it = warm_born.find(tex.gpu);", "if (!tex_warm::learnable(born, warm_now_ns())) return;", + "const uint64_t p = warm_place_of(tex.d, tex.L, tex.gpu);", "warm_list_load_once();", + "if (tex_warm::remember(warm_list, p)) warm_list_changed = true"], "vk_texture.cpp warm_learn") + up = block(tex, "bool upload(Texture& tex, const Ranges* dirty) {") + in_order(up, ["if (!tex.uploaded) warm_learn(tex);", "const bool ok = upload_inner(tex, dirty);"], "vk_texture.cpp upload: a first upload learns") + get = block(tex, "Texture* texture_get(const nvn_tex* t, bool for_sampling) {") + in_order(get, ["tex = it->second.get();", "if (tex->warmed && for_sampling && !warm_looking) tex->warmed = false, warm_learn(*tex);", + "return texture_touch(tex, for_sampling);"], "vk_texture.cpp texture_get: a made-ahead texture's first draw learns") + born = block(tex, "void texture_warm_born(const nvn_tex* t) {") + in_order(born, ["if (!texture_warm_list_mode() || !t || !t->gpu || t->gpu >= (1ull << 39)) return;", + "if (nvn_block_size(t->d.format, &bw, &bh) <= 0 || bw <= 1) return;", "warm_born[t->gpu] = now;"], + "vk_texture.cpp texture_warm_born: BC and ASTC under 2^39, with the switch") + gone = block(tex, "void texture_warm_gone(const nvn_tex* t) {") + in_order(gone, ["if (!texture_warm_list_mode() || !t || !t->gpu) return;", "warm_born.erase(t->gpu);"], "vk_texture.cpp texture_warm_gone") + load = block(tex, "void warm_list_load_once() {") + in_order(load, ["std::call_once(once,", "fopen(warm_list_path().c_str(), \"rb\")", "ok = tex_warm::decode(b.data(), b.size(), had);", + "for (const uint64_t p : had) tex_warm::remember(warm_list, p);"], "vk_texture.cpp warm_list_load_once") + save = block(tex, "void warm_list_save_due() {") + in_order(save, ["if (!texture_warm_list_mode() || frame_no % tex_warm::kSaveEvery) return;", "if (!warm_list_changed) return;", + "b = tex_warm::encode(warm_list);", "if (!file_writers::write_whole(warm_list_path(), b.data(), b.size())) {"], + "vk_texture.cpp warm_list_save_due") + in_order(tex, ["warm_known = known, warm_used = used, warm_allowed = allowed, warm_budget = budget;", "warm_list_save_due();", + "texture_sweep(completed, total_bytes <= budget, budget, evict_to, known, used, allowed, false, hold_to, rule_to);"], + "vk_texture.cpp texture_frame: the list saved at the looks") + if '"triforce_textures_drawn.bin"' not in tex: + raise SystemExit("vk_texture.cpp: the list's file is not triforce_textures_drawn.bin") + created = block(replay, "void texture_created_of(const nvn_tex* t) {") + in_order(created, ["tex_pages_add(t, +1);", "texture_warm_born(t);"], "vk_replay.cpp texture_created_of") + released = block(replay, "void texture_released_of(const nvn_tex* t) {") + in_order(released, ["if (!t) return;", "tex_pages_add(t, -1);", "texture_warm_gone(t);", "if (!async_on()) {"], + "vk_replay.cpp texture_released_of: forgotten on every path") + if "RX_VK_TEX_WARM_LIST" not in dev: + raise SystemExit("docs/development.md: RX_VK_TEX_WARM_LIST is not in the switches table") + + +def build(cxx, d, name, src, include): + cpp, exe = d / (name + ".cpp"), d / (name + ".exe") + cpp.write_text(src, encoding="utf-8") + r = subprocess.run([cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-Wno-unused-function", "-I" + str(include), str(cpp), + "-o", str(exe)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + if r.returncode: + return None, r.stdout + return exe, "" + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--cxx", default=os.environ.get("CXX", "g++")) + a = ap.parse_args() + check_source((VK / "vk_texture.cpp").read_text(encoding="utf-8"), (VK / "vk_replay.cpp").read_text(encoding="utf-8"), + (ROOT / "docs" / "development.md").read_text(encoding="utf-8")) + print("PASS sources: the list's look with its switch before the 40% rule, the list asked before the look, nothing learnt by " + "the look, a draw's first upload and a made-ahead texture's first draw learnt, BC and ASTC noted and forgotten, the " + "file read once and saved whole, the switch's row") + header = (VK / "tex_warm.h").read_text(encoding="utf-8") + with tempfile.TemporaryDirectory(prefix="tex-warm-list-") as tmp: + d = Path(tmp) + exe, out = build(a.cxx, d, "header", HEADER_TEST, VK) + if not exe: + raise SystemExit("header: the build failed:\n" + out) + r = subprocess.run([str(exe)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=60) + if r.returncode: + raise SystemExit("the header test failed:\n" + r.stdout) + print(r.stdout.strip()) + for i, (what, old, new) in enumerate(MUTATIONS): + if header.count(old) != 1: + raise SystemExit("mutation %r: its text is not once in tex_warm.h (update the test with the header)" % what) + inc = d / ("mutant%d" % i) + inc.mkdir() + (inc / "tex_warm.h").write_text(header.replace(old, new), encoding="utf-8") + exe, out = build(a.cxx, d, "mutant%d" % i, HEADER_TEST, inc) + if exe: + r = subprocess.run([str(exe)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=60) + if r.returncode == 0: + raise SystemExit("mutation %r: the header test still passed without it" % what) + print("PASS mutations: %d rules of the list, each taken out, fail the header's test" % len(MUTATIONS)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_texels_vcore.py b/tests/test_texels_vcore.py new file mode 100644 index 0000000..f09fc0e --- /dev/null +++ b/tests/test_texels_vcore.py @@ -0,0 +1,129 @@ +"""ReadTexels and WriteTexels give their virtual core up for the back end's call (host/rx_backend.c; no game, no GPU). + +(Triforce, 11-21, the performance audit's THREAD-20, after Source A's.) Both calls reach the back end, which waits for all +the GPU work submitted and then for the replay's lock. The calling thread kept its virtual core through that wait, and +the game threads allowed only on that core waited with it. + +This cuts the two wrappers and their switch out of rx_backend.c as they are and compiles them with stand-ins for the back +ends and for hos_vcore_away and hos_vcore_back: no core is held inside either back end's call, on Vulkan and on the +fallback; the core is taken back after it; a thread that had no core to give is not handed one; a write kept for a +renderer not started yet keeps the core and never reaches the back end; and with HOS_TEXELS_KEEP_CORE=1 the core is held +through the call, as before. + +And reads the source for the switches row. + + python -I -B tests/test_texels_vcore.py [--cxx clang++] +""" +import argparse +import os +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(source, marker): + start = source.index(marker) + end = source.index("{", start) + 1 + depth = 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[start:end] + + +FIXTURE = r""" +#include +#include +#include +#include +#include +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +struct nvn_texdesc { int format; }; +struct nvn_tex { nvn_texdesc d; }; +struct NVNcopyRegion { int xoffset, yoffset, zoffset, width, height, depth; }; +typedef struct { int kind, pad; nvn_tex t; unsigned level, layer0; NVNcopyRegion r; size_t row, img; uint8_t *data; } early_call; +// the thread's virtual core: held or not, and how often it was given up and taken back +static int held = 1, aways = 0, backs = 0, can_give = 1; +static int hos_vcore_away(void) { if (!can_give || !held) return 0; held = 0, aways++; return 1; } +static void hos_vcore_back(void) { CHECK(!held); held = 1, backs++; } +// the back ends: what they saw of the core +static int vk_reads = 0, vk_writes = 0, wgr_reads = 0, wgr_writes = 0, held_inside = 0, kept = 0; +struct Renderer { + void (*nvn_read_texels)(const nvn_tex *, unsigned, unsigned, const NVNcopyRegion *, uint8_t *, size_t, size_t); + void (*nvn_write_texels)(const nvn_tex *, unsigned, unsigned, const NVNcopyRegion *, const uint8_t *, size_t, size_t); +}; +static void vk_read(const nvn_tex *, unsigned, unsigned, const NVNcopyRegion *, uint8_t *, size_t, size_t) { vk_reads++, held_inside += held; } +static void vk_write(const nvn_tex *, unsigned, unsigned, const NVNcopyRegion *, const uint8_t *, size_t, size_t) { vk_writes++, held_inside += held; } +static void wgr_nvn_read_texels(const nvn_tex *, unsigned, unsigned, const NVNcopyRegion *, uint8_t *, size_t, size_t) { wgr_reads++, held_inside += held; } +static void wgr_nvn_write_texels(const nvn_tex *, unsigned, unsigned, const NVNcopyRegion *, const uint8_t *, size_t, size_t) { wgr_writes++, held_inside += held; } +static const Renderer vulkan = {vk_read, vk_write}; +static const Renderer *vk = &vulkan; +static int started = 1; +static int nvn_fmt_bpb(int, int *bw, int *bh) { *bw = *bh = 1; return 4; } +static int keep_early(const early_call *c) { kept++, held_inside += 0; CHECK(held); free(c->data); return 1; } + +@WRAPPERS@ + +int main(int argc, char **argv) { + const bool keep = argc > 1 && !strcmp(argv[1], "keep"); // (run with HOS_TEXELS_KEEP_CORE=1 in the environment) + nvn_tex t{{0x25}}; + NVNcopyRegion r{0, 0, 0, 16, 16, 1}; + uint8_t buf[16 * 16 * 4] = {0}; + rx_nvn_read_texels(&t, 0, 0, &r, buf, 64, 1024); + rx_nvn_write_texels(&t, 0, 0, &r, buf, 64, 1024); + CHECK(vk_reads == 1 && vk_writes == 1 && held == 1); + if (keep) CHECK(held_inside == 2 && aways == 0 && backs == 0); // (as before: the core held through both) + else CHECK(held_inside == 0 && aways == 2 && backs == 2); // (given up for each call, and taken back) + vk = nullptr; // (the fallback back end alike) + held_inside = aways = backs = 0; + rx_nvn_read_texels(&t, 0, 0, &r, buf, 64, 1024); + rx_nvn_write_texels(&t, 0, 0, &r, buf, 64, 1024); + CHECK(wgr_reads == 1 && wgr_writes == 1 && held == 1); + if (keep) CHECK(held_inside == 2 && aways == 0); + else CHECK(held_inside == 0 && aways == 2 && backs == 2); + vk = &vulkan; + can_give = 0, held = 0, aways = backs = 0; // (a thread with no core to give: none is handed to it) + rx_nvn_read_texels(&t, 0, 0, &r, buf, 64, 1024); + rx_nvn_write_texels(&t, 0, 0, &r, buf, 64, 1024); + CHECK(held == 0 && aways == 0 && backs == 0 && vk_reads == 2 && vk_writes == 2); + can_give = 1, held = 1, started = 0, aways = backs = 0; // (the renderer not started: the write is kept, the core too) + rx_nvn_write_texels(&t, 0, 0, &r, buf, 64, 1024); + CHECK(kept == 1 && vk_writes == 2 && aways == 0 && backs == 0 && held == 1); + return fails ? 1 : 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + src = (ROOT / "host" / "rx_backend.c").read_text(encoding="utf-8").replace("\r\n", "\n") + wrappers = "\n".join(block(src, m) for m in ("static int texels_keep_core(void) {", "void rx_nvn_read_texels(", "void rx_nvn_write_texels(")) + check('getenv("HOS_TEXELS_KEEP_CORE")' in wrappers, "the wrappers' switch is not HOS_TEXELS_KEEP_CORE") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`HOS_TEXELS_KEEP_CORE=1`" in dev, "docs/development.md: HOS_TEXELS_KEEP_CORE is not in the switches table") + with tempfile.TemporaryDirectory(prefix="texels-vcore-") as tmp: + c, exe = Path(tmp) / "texels.cpp", Path(tmp) / "texels.exe" + c.write_text(FIXTURE.replace("@WRAPPERS@", wrappers), encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Wno-unused-function", "-Wno-missing-field-initializers", "-Werror", str(c), "-o", str(exe)], check=True) + for arm, value in (("", None), ("keep", "1"), ("", "0")): + env = {k: v for k, v in os.environ.items() if k != "HOS_TEXELS_KEEP_CORE"} + if value is not None: + env["HOS_TEXELS_KEEP_CORE"] = value + r = subprocess.run([str(exe)] + ([arm] if arm else []), capture_output=True, text=True, env=env) + check(r.returncode == 0, "the wrappers with HOS_TEXELS_KEEP_CORE %s:\n%s" % ("not set" if value is None else "= " + value, r.stdout)) + print("PASS: ReadTexels and WriteTexels give their virtual core up for the back end's call") + + +if __name__ == "__main__": + main() diff --git a/tests/test_texture_pipelines_warm.py b/tests/test_texture_pipelines_warm.py new file mode 100644 index 0000000..88f46a3 --- /dev/null +++ b/tests/test_texture_pipelines_warm.py @@ -0,0 +1,192 @@ +"""The texture converters' compute pipelines through the pipeline cache, made before the game (gpu/vk/vk_texture.cpp +texture_compute_pipeline, texture_pipelines_warm, RX_VK_TEX_PIPES_AHEAD). + +The ASTC decoder, the BC encoder, the unswizzle and the swizzle back were each made without a pipeline cache inside the first +upload that needed it: with an empty driver cache (a first start, a new driver) that upload waited for the compile. What this +checks without the game: + +1. The source: none of the four ready functions creates its pipeline without the cache any more (each goes through + texture_compute_pipeline); precompile_run makes them right after the pipeline cache is read, inside its .running marker + (a driver that crashes there counts as a start that did not get through); a start whose preparation was skipped at the + build makes them too (texture_pipelines_unprepared, after pcache_init); the switch has its docs row. +2. The code, cut out and compiled with stand-ins (a fake vkCreateComputePipelines and shared cache): the shared cache first, + no cache when that fails, no cache at all with RX_VK_TEX_PIPES_AHEAD=0; the warm makes all four where the GPU does not + sample ASTC, only the unswizzle and the swizzle back where it does, once, and nothing with the switch off. + + python -I -B tests/test_texture_pipelines_warm.py [--cxx clang++] +""" +import argparse +import os +from pathlib import Path +import subprocess +import tempfile + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] +READY = {"astc_ready": "astc_pipe", "bc_ready": "bc_pipe", "unswizzle_ready": "unsw_pipe", "swizzle_back_ready": "swz_pipe"} + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def fail(message): + raise SystemExit("FAIL: " + message) + + +def check_source(tex, draw, main, dev): + for fn, pipe in READY.items(): + body = block(tex, "bool %s() {" % fn) + if "vkCreateComputePipelines(dev.device, VK_NULL_HANDLE" in body: + fail("%s still makes its pipeline without the pipeline cache" % fn) + if "texture_compute_pipeline(ci, &%s)" % pipe not in body: + fail("%s does not make its pipeline through texture_compute_pipeline" % fn) + pre = block(draw, "void precompile_run(") + marker, init = pre.index("done_marker{running};"), pre.index("pcache_init();") + warm = pre.index("texture_pipelines_warm();") + if not marker < init < warm < pre.index("pipe_notes_read("): + fail("precompile_run does not make the converters after the cache is read, inside its marker, before the notes") + early = pre.index("its notes are dropped") + if early > warm: + fail("precompile_run makes the converters before the two-failed-starts return: a crash there would repeat every start") + unprepared = block(draw, "void texture_pipelines_unprepared() {") + if not unprepared.index("pcache_init();") < unprepared.index("texture_pipelines_warm();"): + fail("texture_pipelines_unprepared does not read the cache before the converters") + skipped = main[main.index("the shaders' preparation was skipped at the build"):] + skipped = skipped[:skipped.index("return true;")] + if "texture_pipelines_unprepared();" not in skipped: + fail("a start whose preparation was skipped at the build does not make the converters") + if "`RX_VK_TEX_PIPES_AHEAD=0`" not in dev: + fail("docs/development.md has no RX_VK_TEX_PIPES_AHEAD row") + + +HEAD = r'''#include +#include +#include +#include +#include +#include +using VkResult = int; +using VkPipeline = uintptr_t; +using VkPipelineCache = uintptr_t; +constexpr VkResult VK_SUCCESS = 0; +constexpr uintptr_t VK_NULL_HANDLE = 0; +constexpr int VK_FORMAT_ASTC_4x4_UNORM_BLOCK = 157; +constexpr unsigned VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT = 1; +struct VkComputePipelineCreateInfo { int sType; uintptr_t layout; }; +struct VkFormatProperties { unsigned linearTilingFeatures, optimalTilingFeatures, bufferFeatures; }; +struct { uintptr_t device = 0x1234, physical = 0x99; } dev; +#define CHECK(c) do { if (!(c)) { std::fprintf(stderr, "CHECK failed line %d: %s\n", __LINE__, #c); std::exit(7); } } while (0) +int shared_calls = 0, shared_fail = 0, null_calls = 0, native = 0; +int made[4]; // (astc, bc, unswizzle, swizzle back: each ready function's calls) +void logf(const char*, ...) {} +VkResult shared_cache_compute(const VkComputePipelineCreateInfo* ci, VkPipeline* out) { + CHECK(ci && out); + shared_calls++; + if (shared_fail) { *out = 0xdead; return -3; } + *out = 0xcace; + return VK_SUCCESS; +} +VkResult vkCreateComputePipelines(uintptr_t device, VkPipelineCache cache, unsigned n, const VkComputePipelineCreateInfo* ci, + const void* alloc, VkPipeline* out) { + CHECK(device == dev.device && cache == VK_NULL_HANDLE && n == 1 && ci && !alloc && out); + null_calls++; + *out = 0xf00d; + return VK_SUCCESS; +} +void vkGetPhysicalDeviceFormatProperties(uintptr_t physical, int format, VkFormatProperties* p) { + CHECK(physical == dev.physical && format == VK_FORMAT_ASTC_4x4_UNORM_BLOCK); + *p = {}; + if (native) p->optimalTilingFeatures = VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT; +} +bool astc_ready() { return ++made[0], true; } +bool bc_ready() { return ++made[1], true; } +bool unswizzle_ready() { return ++made[2], true; } +bool swizzle_back_ready() { return ++made[3], true; } +''' + +MAIN = r''' +int main(int argc, char** argv) { + CHECK(argc == 2); + const std::string k = argv[1]; + const bool off = getenv("RX_VK_TEX_PIPES_AHEAD") && !atoi(getenv("RX_VK_TEX_PIPES_AHEAD")); + VkComputePipelineCreateInfo ci{29, 0x1357}; + VkPipeline out = 0; + if (k == "create") { + CHECK(texture_compute_pipeline(ci, &out) == VK_SUCCESS); + if (off) CHECK(shared_calls == 0 && null_calls == 1 && out == 0xf00d); + else CHECK(shared_calls == 1 && null_calls == 0 && out == 0xcace); + } else if (k == "fallback") { + shared_fail = 1; + CHECK(texture_compute_pipeline(ci, &out) == VK_SUCCESS); + CHECK(null_calls == 1 && out == 0xf00d && shared_calls == (off ? 0 : 1)); + } else if (k == "warm" || k == "warm-native") { + native = k == "warm-native"; + texture_pipelines_warm(); + texture_pipelines_warm(); // (once) + if (off) CHECK(!made[0] && !made[1] && !made[2] && !made[3]); + else if (native) CHECK(!made[0] && !made[1] && made[2] == 1 && made[3] == 1); + else CHECK(made[0] == 1 && made[1] == 1 && made[2] == 1 && made[3] == 1); + } else { + CHECK(!"no such case"); + } + std::printf("OK %s\n", k.c_str()); + return 0; +} +''' + + +def fixture(tex): + parts = [block(tex, "bool tex_pipes_ahead() {"), block(tex, "VkResult texture_compute_pipeline("), + block(tex, "void texture_pipelines_warm() {")] + return HEAD + "\n".join(parts) + MAIN + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--cxx", default="g++") + a = p.parse_args() + vk = ROOT / "gpu" / "vk" + tex = (vk / "vk_texture.cpp").read_text(encoding="utf-8") + draw = (vk / "vk_draw.cpp").read_text(encoding="utf-8") + main_cpp = (vk / "vk_main.cpp").read_text(encoding="utf-8") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + try: + check_source(tex, draw, main_cpp, dev) + text = fixture(tex) + except ValueError as e: + fail("a text this test reads moved, or is not there (%s)" % e) + problems = [] + with tempfile.TemporaryDirectory(prefix="tex-pipes-ahead-") as tmp: + cpp = Path(tmp) / "fixture.cpp" + exe = Path(tmp) / ("fixture" + (".exe" if os.name == "nt" else "")) + cpp.write_text(text, encoding="utf-8") + built = subprocess.run([a.cxx, "-std=c++20", "-O1", "-Wall", "-Wno-unused-function", str(cpp), "-o", str(exe)], + capture_output=True, text=True) + if built.returncode: + print(built.stdout + built.stderr) + fail("the fixture did not build") + base = {k: v for k, v in os.environ.items() if k != "RX_VK_TEX_PIPES_AHEAD"} + n = 0 + for value in (None, "1", "0"): + env = dict(base, **({"RX_VK_TEX_PIPES_AHEAD": value} if value is not None else {})) + for case in ("create", "fallback", "warm", "warm-native"): + n += 1 + done = subprocess.run([str(exe), case], env=env, capture_output=True, text=True, timeout=30) + if done.returncode or done.stdout.strip() != "OK " + case: + problems.append("%s with RX_VK_TEX_PIPES_AHEAD=%s: %s" % (case, value, (done.stdout + done.stderr).strip())) + if problems: + fail("\n ".join([""] + problems)) + print("PASS: the texture converters through the pipeline cache (no cache when it fails or with the switch off), made " + "before the game inside the precompile's marker and on a start without one (%d cases)" % n) + + +if __name__ == "__main__": + main() diff --git a/tests/test_texture_pool_ids.py b/tests/test_texture_pool_ids.py index 63bc375..61cdcdc 100644 --- a/tests/test_texture_pool_ids.py +++ b/tests/test_texture_pool_ids.py @@ -80,7 +80,7 @@ def main(): # ((Triforce) the sampled textures' sampler through kept_texture, the per-binding reuse: the whole handle its input) need("vk_draw.cpp: kept_texture()", block(draw, "VkSampler kept_texture("), ["const VkSampler smp = sampler_for((int)(handle >> 20), compare, scene);", "in = handle | (uint64_t)vt << 32"]) - need("vk_draw.cpp", draw, ["kept_texture(kp, jb + k, handle, vt, d.is_depth,", "dw().imgs.push_back({smp, view, VK_IMAGE_LAYOUT_GENERAL});", + need("vk_draw.cpp", draw, ["kept_texture(kp, jb + k, handle, vt, d.is_depth,", "dwr.imgs.push_back({smp, view, VK_IMAGE_LAYOUT_GENERAL});", "if (!on || id < 0 || id >= 4096) return sampler_make(id, compare, scene);"]) for where, text in [("vk_draw.cpp", draw), ("vk_bank.cpp", bank), ("vk_replay.cpp", replay)]: for mask in re.findall(r"handle & (0x[0-9A-Fa-f]+)", text): diff --git a/tests/test_tls_once.py b/tests/test_tls_once.py new file mode 100644 index 0000000..42f8a4c --- /dev/null +++ b/tests/test_tls_once.py @@ -0,0 +1,90 @@ +"""pass() and dw() are taken once in each of the draw's hot functions (gpu/vk/vk_draw.cpp; no game, no GPU). + +(Triforce, 11-22, the performance audit's DESC-20, after Source A's.) The thread's Pass and DescWriter are reached through +plain thread-local pointers (TLS_OBJECT, tests/test_tls_objects.py): each pass() or dw() is a thread-local read and a +null test, and in a Linux library built with -fPIC a call for the address. The hot functions asked at every use: end_pass +57 times, build_descriptors 31. They now take the object once at their top and use the reference. + +Reads the source: + - build_descriptors and bind_descriptors: their first statement is `auto& dwr = dw();` and there is no other dw() in them; + - begin_pass, dynamic_state, draw_hazards, end_pass and draw: their first statement is `Pass& ps = pass();` and there is + no other pass() in them; draw keeps its four dw() (tests/test_mesh_fallback.py reads them); + - the bound names are not declared a second time in those functions (bind_descriptors' own `ps`, the pushed state, is + not a Pass and stays); + - what makes the reference sound: a recording's start assigns into the thread's Pass (pass() = Pass{}), TLS_OBJECT is + the only place a per-thread pointer is set, and nothing names pass_tls or dw_tls outside it. + + python -I -B tests/test_tls_once.py +""" +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + +DESCRIPTORS = ["void build_descriptors(const NvnState& s, const Program& p, bool keep = false, bool scene = false) {", + "bool bind_descriptors(VkCommandBuffer cmd, VkPipelineBindPoint bp, const Program& p) {"] +PASSES = ["void begin_pass(VkCommandBuffer cmd, const Targets& at) {", + "void dynamic_state(VkCommandBuffer cmd, const NvnState& s, const Targets& at, VkPrimitiveTopology topo, bool mesh) {", + "uint16_t draw_hazards(VkCommandBuffer cmd, const Program& p, const NvnState& s, const Targets& at) {", + "void end_pass(VkCommandBuffer cmd) {", "void draw(VkCommandBuffer cmd, NvnState& s, const DrawArgs& d) {"] + + +def block(text, marker): + if text.count(marker) != 1: + raise SystemExit("FAIL: vk_draw.cpp does not have exactly one: " + marker) + start = text.index(marker) + at = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[at] == "{") - (text[at] == "}") + at += 1 + return text[start:at] + + +def code(text): + """Without its line comments (a comment may name the accessor)""" + return "\n".join(line.split("//")[0] for line in text.split("\n")) + + +def calls(text, name): + return len(re.findall(r"(?])%s\." % name, code(body))) + need(uses >= 4, "%s takes %s() and uses it %d times" % (what, accessor, uses)) + # (the name on the left of an = once: its binding; a second would be another variable of that name, or the object copied over) + declared = len(re.findall(r"(?])%s\s*=(?!=)" % name, code(body))) + if what == "bind_descriptors": + # (its own ps is the pushed state of the bind point, not a Pass) + need("Pass& ps" not in body and re.search(r"Pushed& ps = pushed\(\)\[", body), "bind_descriptors' ps is no longer its pushed state") + need(declared == 1, "%s sets %s %d times: once, where it is taken" % (what, name, declared)) + need(calls(block(draw, PASSES[-1]), "dw") == 4, "draw no longer has its four dw() (tests/test_mesh_fallback.py reads dw().prog there)") + # what the references stand on + need("TLS_OBJECT(pass, Pass)" in draw and "TLS_OBJECT(dw, DescWriter)" in draw, "pass and dw are no longer TLS_OBJECTs") + at = draw.index("#define TLS_OBJECT(name, ...)") + macro = draw[at:draw.index("\n\n", at)] # (the macro: up to the first empty line after it) + need("return p ? *p : tls_make(name##_tls);" in macro and "slot = new T();" in block(draw, "T& tls_make(T*& slot) {"), + "TLS_OBJECT no longer makes a thread's object once and hands the same one back") + need(not re.search(r"(? (off by default; 8 to try), a draw +into targets that hold an earlier frame, of a program key whose translated variants write no memory, waits at most that +much a frame in all, never running the job, then is left out; a spent budget leaves it out at once. A key never translated, +a key with a writing variant and a draw into new targets wait as before. + +The header is compiled and run: the strict parse (digits only, 1 to 100), the choice, the notes of which keys write. +Then the draw's translation block is cut out of draw() and run over stand-ins whose translation finishes after a set time: + - off (unset or a bad value): every draw waits for it, as before (RX_VK_PIPE_SKIP=2's skip kept); + - on: a known clean key into kept targets waits at most the budget, hurried and not run here, and is drawn when the + translation comes in time; left out and counted when not; the budget is charged, and a spent one leaves the next out + at once; a new frame fills it again; + - on: an unknown key, a writer's key and a draw into new targets wait as before; + - RX_VK_PIPE_SKIP=0 waits for all. +And translate_note, cut out, notes a key only while the switch is on, and a writer sticks. The source: the notes are taken +at graphics_pipeline's new entry and the precompile's loads; the switches table has the row. + + python -I -B tests/test_translate_wait.py [--cxx clang++] +""" +import argparse +import os +import subprocess +import tempfile +from pathlib import Path + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] + +HEADER = r""" +#include +#include "translate_wait.h" +#define CHECK(x) do { if (!(x)) { std::printf("line %d: %s\n", __LINE__, #x); return 1; } } while (0) +using namespace translate_wait; +int main() { + CHECK(parse(nullptr) == 0 && parse("") == 0 && parse("8") == 8 && parse("100") == 100 && parse("1") == 1); + CHECK(parse("0") == 0 && parse("101") == 0 && parse("8ms") == 0 && parse("-8") == 0 && parse(" 8") == 0 && parse("99999999999") == 0); + CHECK(choose(0, true, true, 8 * kMs) == Act::AsBefore && choose(8, false, true, 8 * kMs) == Act::AsBefore); + CHECK(choose(8, true, false, 8 * kMs) == Act::AsBefore && choose(8, true, true, 8 * kMs) == Act::Bounded); + CHECK(choose(8, true, true, kMs) == Act::Bounded && choose(8, true, true, kMs - 1) == Act::Skip && choose(8, true, true, 0) == Act::Skip); + Notes n; + CHECK(!n.clean(1)); + n.note(1, false); + CHECK(n.clean(1)); + n.note(1, true); + n.note(1, false); + CHECK(!n.clean(1)); // (a writing variant sticks) + n.note(2, true); + CHECK(!n.clean(2) && !n.clean(3)); + std::printf("ok\n"); + return 0; +} +""" + +FIXTURE = r""" +#include +#include +#include +#include +#include +#include +#include +#include +#include "pipe_wait.h" +#include "translate_wait.h" +#define CHECK(x) do { if (!(x)) { std::printf("line %d: %s\n", __LINE__, #x); return 1; } } while (0) +struct Program { uint64_t key; bool failed = false, compute = false; bool writes = false; std::atomic ready{false}; }; +struct Targets { bool kept; }; +uint64_t frame_now = 1; +uint64_t texture_frame_no() { return frame_now; } +bool pass_kept(const Targets& at, bool) { return at.kept; } +bool targets_persistent(const Targets& at) { return at.kept; } +bool pipe_skip_new() { return true; } +int64_t left_count, st_pending_skips; +void left_out_pending() { left_count++; } +std::atomic st_shader_wait_us{0}, st_translate_bounded{0}, st_translate_left{0}; +void note_wait(std::chrono::steady_clock::time_point, std::atomic&) {} +// (the translation: done after finish_ms of waiting; a bounded wait hurries it and never runs it) +int finish_ms = 0, bounded_calls = 0, full_waits = 0, last_bound_ms = 0; +bool jobs_wait_for(const std::atomic& flag, int ms) { + bounded_calls++, last_bound_ms = ms; + if (finish_ms <= ms) const_cast&>(flag).store(true); + std::this_thread::sleep_for(std::chrono::milliseconds(std::min(ms, finish_ms))); + return flag.load(); +} +bool writes_memory(const Program& p) { return p.writes; } +/* SWITCH */ +/* NOTE */ +static std::string run(Program& prog, const Targets& at) { + const Program* cp = &prog; + static const bool always_wait = getenv("RX_VK_PIPE_SKIP") && atoi(getenv("RX_VK_PIPE_SKIP")) == 0; + static const bool legacy_translation_skip = getenv("RX_VK_PIPE_SKIP") && atoi(getenv("RX_VK_PIPE_SKIP")) == 2; + auto wait_for = [&](const std::atomic& ready, std::atomic&) { + full_waits++; + const_cast&>(ready).store(true); // (waited out: the job ran here) + }; + const int64_t left_before = left_count; + [&] { + /* BLOCK */ + }(); + return left_count != left_before ? "left" : full_waits ? "waited" : "drawn"; +} +int main(int argc, char** argv) { + const std::string arm = argc > 1 ? argv[1] : ""; + const Targets kept{true}, fresh{false}; + Program clean{1}, writer{2}, unknown{3}; + writer.writes = true; + translate_note(clean); + translate_note(writer); + const auto reset = [&](Program& p) { p.ready = false, full_waits = 0, bounded_calls = 0; }; + if (arm == "off" || arm == "bad" || arm == "always") { // (every draw waits, as before) + for (Program* p : {&clean, &writer, &unknown}) + for (const Targets* t : {&kept, &fresh}) { + reset(*p); + CHECK(run(*p, *t) == "waited" && bounded_calls == 0); + } + CHECK(st_translate_bounded == 0 && st_translate_left == 0 && left_count == 0); + std::printf("ok\n"); + return 0; + } + if (arm == "legacy") { // (RX_VK_PIPE_SKIP=2: its skip, as before) + reset(clean); + CHECK(run(clean, kept) == "left" && bounded_calls == 0 && st_pending_skips == 1); + std::printf("ok\n"); + return 0; + } + // on, 8 ms: a known clean key into kept targets, translated in time: drawn, no full wait, the job not run here + finish_ms = 3, reset(clean); + CHECK(run(clean, kept) == "drawn" && bounded_calls == 1 && last_bound_ms == 8 && full_waits == 0 && st_translate_bounded == 1); + // not in time: left out and counted + finish_ms = 50, reset(clean); + CHECK(run(clean, kept) == "left" && st_translate_left == 1 && st_pending_skips == 1 && full_waits == 0); + // the budget is spent: the next is left out at once, not waited for + reset(clean); + CHECK(run(clean, kept) == "left" && bounded_calls == 0 && st_translate_left == 2); + // a new frame fills it again + frame_now++, finish_ms = 2, reset(clean); + CHECK(run(clean, kept) == "drawn" && bounded_calls == 1); + // an unknown key, a writer's key, new targets: waited for, as before + finish_ms = 50; + for (Program* p : {&unknown, &writer}) { + reset(*p); + CHECK(run(*p, kept) == "waited" && bounded_calls == 0); + } + reset(clean); + CHECK(run(clean, fresh) == "waited" && bounded_calls == 0); + // a program already translated: nothing of it + clean.ready = true, full_waits = 0, bounded_calls = 0; + CHECK(run(clean, kept) == "drawn" && bounded_calls == 0 && full_waits == 0); + std::printf("ok\n"); + return 0; +} +""" + + +def block(text, marker, start_at=0): + start = text.index(marker, start_at) + at = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[at] == "{") - (text[at] == "}") + at += 1 + return text[start:at] + + +def need(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def ordered(text, parts, why): + last = 0 + for p in parts: + k = text.find(p, last) + need(k >= 0, why + " (missing, or out of order: " + p + ")") + last = k + len(p) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + draw = (ROOT / "gpu/vk/vk_draw.cpp").read_text(encoding="utf-8") + need('#include "translate_wait.h"' in draw, "vk_draw.cpp does not include translate_wait.h") + after = draw.index(" auto wait_for = [&](const std::atomic& ready, std::atomic& into) {") + blk = block(draw, "if (!cp->ready.load(std::memory_order_acquire)) {", after) + ordered(blk, ["if (legacy_translation_skip && targets_persistent(at)) return left_out_pending(), void(st_pending_skips++);", + "if (translate_wait_ms() && !always_wait) {", + "translate_wait::choose(translate_wait_ms(), pass_kept(at, pipe_skip_new()),", "translate_notes().clean(cp->key), translate_pool().left(frame));", + "jobs_wait_for(cp->ready,", "translate_pool().charge(frame,", "if (!got) return st_translate_left.fetch_add(1", + "if (!cp->ready.load(std::memory_order_acquire)) wait_for(cp->ready, st_shader_wait_us);"], + "draw() does not bound the wait for a translating program as the switch says") + need("jobs_wait(" not in blk.replace("wait_for(cp->ready, st_shader_wait_us)", ""), "the bounded wait runs the job on the draw's thread") + switch = "\n".join(block(draw, m) for m in ("int translate_wait_ms() {", "pipe_wait::Pool& translate_pool() {", + "translate_wait::Notes& translate_notes() {")) + note = block(draw, "void translate_note(const Program& p) {") + gp = block(draw, "PipeEntry* graphics_pipeline(const Program& p, const NvnState& s, const Targets& at, VkPrimitiveTopology topo, const uint8_t color_types[8]) {") + ordered(gp, ["pipeline_note(key, p, s, at, topo, color_types);", "translate_note(p);"], "graphics_pipeline's new entry takes no note") + pre = block(draw, "void precompile_run(void (*progress)(size_t done, size_t total)) {") + ordered(pre, ["program_cache_load_some(wanted, [&](Program& p) {", "ensure_layout(p);", "translate_note(p);"], + "the precompile's loads take no note") + docs = (ROOT / "docs/development.md").read_text(encoding="utf-8") + need("`RX_VK_TRANSLATE_WAIT_MS=`" in docs, "docs/development.md: the switches table has no RX_VK_TRANSLATE_WAIT_MS row") + base = {k: v for k, v in os.environ.items() if not k.startswith("RX_VK_")} + with tempfile.TemporaryDirectory(prefix="translate-wait-") as tmp: + tmp = Path(tmp) + fixture = (FIXTURE.replace("/* SWITCH */", "void logf(const char*, ...) {}\n" + switch) + .replace("/* NOTE */", note).replace("/* BLOCK */", blk)) + for name, text, runs in (("header", HEADER, [("", {})]), + ("draw", fixture, [("on", {"RX_VK_TRANSLATE_WAIT_MS": "8"}), ("off", {}), + ("bad", {"RX_VK_TRANSLATE_WAIT_MS": "8ms"}), + ("always", {"RX_VK_TRANSLATE_WAIT_MS": "8", "RX_VK_PIPE_SKIP": "0"}), + ("legacy", {"RX_VK_TRANSLATE_WAIT_MS": "8", "RX_VK_PIPE_SKIP": "2"})])): + src, exe = tmp / (name + ".cpp"), tmp / (name + ".exe") + src.write_text(text, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-Wno-unused-parameter", "-I" + str(ROOT / "gpu/vk"), + str(src), "-o", str(exe)], check=True) + for arg, env in runs: + r = subprocess.run([str(exe)] + ([arg] if arg else []), capture_output=True, text=True, timeout=60, env={**base, **env}) + need(r.returncode == 0 and r.stdout.strip() == "ok", "%s %s: %s" % (name, arg, r.stdout.strip() or r.returncode)) + print("PASS: RX_VK_TRANSLATE_WAIT_MS bounds the wait for a translating program known to write no memory into kept targets; " + "unset, every such draw waits as before") + + +if __name__ == "__main__": + main() diff --git a/tests/test_ub_update_batch.py b/tests/test_ub_update_batch.py new file mode 100644 index 0000000..dd619ea --- /dev/null +++ b/tests/test_ub_update_batch.py @@ -0,0 +1,154 @@ +"""A recording's uniform updates as copies of many regions from its stream, ordered where they overlap (gpu/vk/ub_batch.h, +vk_replay.cpp flush_ub_updates and flush_ub_batched; no game, no GPU). + +(Triforce, 11-22, the performance audit's SYNC-21, after Source A's.) The flush wrote a vkCmdUpdateBuffer an update +between its two barriers, with nothing between two updates of the same bytes. RX_VK_UB_BATCH=1 copies each run of them +into one buffer with one vkCmdCopyBuffer; a run ends where a piece would write bytes the run already wrote, or where the +destination buffer changes, and the next starts behind a barrier. Off by default. + +tests/ub_update_batch_test.cpp gets the flush, the batch and gpu_write cut from the source and runs 600 flushes a +scenario on a stand-in that records the commands and then applies them to memory in order: + - batched: memory is the updates applied in stream order, in every scenario (aligned pieces in pools; any: unaligned + pieces, pieces outside the pools, a second buffer onto another's memory; the stream full; the stream moved to another + buffer); no copy's regions overlap; no byte written since the last barrier is written again; + - as before (unset): only updates between the two barriers, two barriers a flush; with aligned pieces in pools its + memory is the model's too, and it is the batched run's memory, byte for byte (hash); + - the bindings asked for (address, size, written) are the same sequence either way, in every scenario; + - RX_VK_UB_BATCH=2: the commands and the memory of the unset run, with the overlaps counted in the 600-flush line. +Then the rule is taken away a part at a time and the batched run must fail: the overlap rule, the destination rule, the +barrier between runs, and the copy before the stream's buffer changes. + +And reads the source: the flush's order (the span, the first barrier, the batch or gpu_write as before, the last barrier), +the switch read once and off unless 1 or 2, the lab knob, the switches table. + + python -I -B tests/test_ub_update_batch.py [--cxx clang++] +""" +import argparse +import os +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] +SCENARIOS = ("aligned", "any", "full", "moved") + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + check(marker in text, "the source has no " + marker) + start = text.index(marker) + end = text.index("{", start + len(marker) - 1) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + check((VK / "ub_batch.h").exists(), "gpu/vk/ub_batch.h is missing") + header = (VK / "ub_batch.h").read_text(encoding="utf-8") + rep = (VK / "vk_replay.cpp").read_text(encoding="utf-8") + check('#include "ub_batch.h"' in rep, "vk_replay.cpp does not include ub_batch.h") + + # the source + flush = block(rep, "void flush_ub_updates(VkCommandBuffer cmd, size_t lim = SIZE_MAX) {") + order = [flush.find(s) for s in ( + "UbReadGuard ub_guard;", "const size_t end = std::min(lim, ub_updates.size());", "if (ub_first_of_recording >= end) return;", + 'GprofSpan gp(cmd, "ubupd (the stream\'s uniform updates -> guest memory, with its barriers)");', + "gp.items = (int)(end - ub_first_of_recording);", "const int mode = ub_batch_mode();", + 'const bool batched = mode == 1 || lab_ab::on("ubbatch");', "full_barrier(cmd);", "if (batched) {", + "flush_ub_batched(cmd, ub_first_of_recording, end, gp);", "} else {", + "for (size_t i = ub_first_of_recording; i < end; i++)\n gpu_write(cmd, ub_updates[i].dst, ub_updates[i].bytes.data(), " + "ub_updates[i].bytes.size()), gp.bytes += ub_updates[i].bytes.size();", + "full_barrier(cmd);\n if (batched || mode == 2) ub_batch_line(batched);")] + check(min(order) >= 0 and order == sorted(order), "flush_ub_updates: the guard, the span, the first barrier, the batch or the " + "updates as before, the last barrier are not there in that order") + check(flush.count("full_barrier(cmd);") == 2, "flush_ub_updates no longer has its two barriers") + mode = block(rep, "int ub_batch_mode() {") + check('getenv("RX_VK_UB_BATCH")' in mode and "static const int mode" in mode and "return v == 1 || v == 2 ? v : 0;" in mode, + "RX_VK_UB_BATCH is not read once, or is on for anything but 1 and 2") + batch = block(rep, "void flush_ub_batched(VkCommandBuffer cmd, size_t first, size_t end, GprofSpan& gp) {") + barrier = " full_barrier(cmd); // (the next run's writes come after this one's: the same bytes, or the same memory through another buffer)\n" + before_copy = " } else if (what == ub_batch::COPY || !s) {\n" + for s in ("const BufferRef b = gpu_buffer_bind(gpu, k, true);", "guest_cpu_write(gpu, d, k), on_cpu++;", "const BufferRef s = stream_copy(d, k);", + "run.before(s.buffer, b.buffer, b.offset, b.offset + k);", barrier, before_copy, + "vkCmdCopyBuffer(cmd, run.src, run.dst, (uint32_t)regions.size(), regions.data());", + "if ((b.offset & 3) || (k & 3)) guest_cpu_write(gpu, d, k);", + "else vkCmdUpdateBuffer(cmd, b.buffer, b.offset, k, d), run.wrote(b.offset, b.offset + k);"): + check(batch.count(s) == 1, "flush_ub_batched lost %r" % s.strip()) + check("full_barrier" not in batch.replace(barrier, ""), "flush_ub_batched has a barrier other than the one between runs") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`RX_VK_UB_BATCH=1`" in dev and "`ubbatch`" in dev, "docs/development.md: RX_VK_UB_BATCH=1 is not in the switches table") + + update = block(rep, "struct UbUpdate {") + ";" + counts = "std::atomic st_ubb_flushes{0}, st_ubb_updates{0}, st_ubb_overlaps{0}, st_ubb_imports{0}, st_ubb_copies{0}, st_ubb_regions{0},\n st_ubb_fallbacks{0}, st_ubb_cpu{0};" + check(counts in rep, "vk_replay.cpp has no counts of the batch") + cut = "\n".join([block(rep, "void gpu_write(VkCommandBuffer cmd, uint64_t gpu, const void* data, uint64_t n) {"), mode, counts, + block(rep, "void ub_batch_line(bool batched) {"), batch, flush]) + fixture = (ROOT / "tests" / "ub_update_batch_test.cpp").read_text(encoding="utf-8") + base_env = {k: v for k, v in os.environ.items() if not k.startswith("RX_VK_")} + + with tempfile.TemporaryDirectory(prefix="ub-update-batch-") as tmp: + def build(name, code=cut, head=header): + inc = Path(tmp) / (name + "-inc") + inc.mkdir(exist_ok=True) + (inc / "ub_batch.h").write_text(head, encoding="utf-8") + src, exe = Path(tmp) / (name + ".cpp"), Path(tmp) / (name + ".exe") + src.write_text(fixture.replace("/* ACTUAL-1 */", update).replace("/* ACTUAL-2 */", code), encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Wno-unused-parameter", "-I" + str(inc), str(src), "-o", str(exe)], check=True) + return str(exe) + + def run(exe, scenario, mode_value): + env = dict(base_env) + if mode_value: + env["RX_VK_UB_BATCH"] = mode_value + r = subprocess.run([exe, scenario], env=env, capture_output=True, text=True, timeout=300) + lines = dict(l.split(" ", 1) for l in r.stdout.splitlines() if l.split(" ", 1)[0] in ("binds", "memory", "counts:", "line:")) + return r.returncode, lines, r.stdout + + exe = build("now") + for scenario in SCENARIOS: + rc_old, old, out_old = run(exe, scenario, None) + rc_new, new, out_new = run(exe, scenario, "1") + rc_cnt, cnt, out_cnt = run(exe, scenario, "2") + check(rc_old == 0, "%s as before:\n%s" % (scenario, out_old)) + check(rc_new == 0, "%s batched:\n%s" % (scenario, out_new)) + check(rc_cnt == 0, "%s counted:\n%s" % (scenario, out_cnt)) + check(old["binds"] == new["binds"] == cnt["binds"], "%s: the batch asks for other bindings than the updates did" % scenario) + check(old["memory"] == cnt["memory"] and old["counts:"] == cnt["counts:"], "%s: RX_VK_UB_BATCH=2 changes what is written" % scenario) + if scenario != "any": # (aligned pieces in pools: the old way is in stream order too) + check(old["memory"] == new["memory"], "%s: the batch leaves other bytes than the updates did" % scenario) + check("600 flushes, batched" in new["line:"] and "600 flushes, as before, counted" in cnt["line:"] and not old["line:"].strip(), + "%s: the 600-flush line is not there with the switch at 1 and 2, or is there without it" % scenario) + print("%-8s as before: %s" % (scenario, old["counts:"])) + print("%-8s batched: %s" % ("", new["counts:"])) + + # each part of the rule taken away: the batched run must fail somewhere + mutations = ( + ("the overlap rule", cut, header.replace(" if (overlaps(at, end)) return RUN_OVERLAP;\n", "")), + ("the destination rule", cut, header.replace(" if (d != dst) return RUN_IMPORT;\n", "")), + ("the barrier between runs", cut.replace(barrier, ""), header), + ("the copy before the stream's buffer changes", cut, header.replace(" if (pending && s != src) return COPY;\n", "")), + ) + for name, code, head in mutations: + check(code != cut or head != header, "the mutation %r changed nothing: the source moved" % name) + mexe = build("mut-" + name.split()[1], code, head) + failed = [s for s in SCENARIOS if run(mexe, s, "1")[0] != 0] + check(failed, "without %s every batched scenario still passes: the fixture does not guard it" % name) + print("without %s: %s fail%s" % (name, ", ".join(failed), "s" if len(failed) == 1 else "")) + print("PASS: the uniform updates batched are the updates in stream order, no region over another, a barrier between two writes " + "of the same bytes; unset, the flush is as before") + + +if __name__ == "__main__": + main() diff --git a/tests/test_udmabuf_limit_unreadable.py b/tests/test_udmabuf_limit_unreadable.py new file mode 100644 index 0000000..aaee05f --- /dev/null +++ b/tests/test_udmabuf_limit_unreadable.py @@ -0,0 +1,140 @@ +"""The guest window is a memfd's when /dev/udmabuf opens, even when udmabuf's size limit cannot be read (host/hos_main.c, the +window's setup on Linux; HOS_GUEST_MEMFD; no game). + +Flatpak and other sandboxes hide /sys/module while /dev/udmabuf opens, and the window stayed anonymous memory there: amdgpu checks +every page of a host-pointer import again at each submission, 15-45 ms a submission. The limit file now decides only when it +reads 0 (udmabuf then refuses every dma-buf); unreadable, it counts as udmabuf's default, 64 MB, as the renderer takes it. + +The rule, from HOS_GUEST_MEMFD's read to the test that makes the memfd, is cut out of hos_main.c unchanged and compiled with +stand-ins for open, close and the limit file (any compiler: the rule has no Linux calls of its own). Checked: usable with the limit +file missing (the 64 MB line logged), holding 64, 256, x or nothing; not with 0 or when /dev/udmabuf does not open (the line that +says why logged); HOS_GUEST_MEMFD=1 and =0 over all of them; HOS_GUEST_MEMFD has its docs row. + + python -I -B tests/test_udmabuf_limit_unreadable.py [--cc clang] +""" +import argparse +import os +from pathlib import Path +import subprocess +import tempfile + +RUN_ALL_ARGS = ["--cc", "gcc"] + +ROOT = Path(__file__).resolve().parents[1] + +FIXTURE = r'''#include +#include +#include +#include +#ifndef O_RDWR +#define O_RDWR 2 +#endif +#ifndef O_CLOEXEC +#define O_CLOEXEC 0 +#endif +static int dev_opens; +static const char *limit_path; /* (NULL: the limit file missing) */ +static int fake_open(const char *p, int flags) { (void)flags; return !strcmp(p, "/dev/udmabuf") && dev_opens ? 5 : -1; } +static int fake_close(int fd) { (void)fd; return 0; } +static FILE *fake_fopen(const char *p, const char *mode) { + if (strcmp(p, "/sys/module/udmabuf/parameters/size_limit_mb")) return NULL; + return limit_path ? fopen(limit_path, mode) : NULL; +} +static char logged[4096]; +static void hos_log(const char *fmt, ...) { + va_list a; + va_start(a, fmt); + vsnprintf(logged + strlen(logged), sizeof logged - strlen(logged), fmt, a); + va_end(a); + strncat(logged, "\n", sizeof logged - strlen(logged) - 1); +} +#define open fake_open +#define close fake_close +#define fopen fake_fopen +static int memfd_chosen(void) { +@CUT@ + return 1; + } + return 0; +} +int main(int argc, char **argv) { + dev_opens = atoi(argv[1]); + limit_path = argc > 2 ? argv[2] : NULL; + printf("%d\n%s", memfd_chosen(), logged); + return 0; +} +''' + +# (/dev/udmabuf opens, the limit file's text or None for missing, HOS_GUEST_MEMFD or None) -> (memfd chosen, a text the log has) +CASES = [ + ((1, None, None), (1, "udmabuf's size_limit_mb cannot be read (in a container?): its default, 64 MB, assumed")), + ((1, "64\n", None), (1, None)), + ((1, "256\n", None), (1, None)), + ((1, "x\n", None), (1, None)), + ((1, "", None), (1, None)), + ((1, "0\n", None), (0, "udmabuf's size_limit_mb is 0")), + ((0, "64\n", None), (0, "/dev/udmabuf cannot be opened")), + ((0, None, None), (0, "/dev/udmabuf cannot be opened")), + ((0, None, "1"), (1, None)), + ((1, "0\n", "1"), (1, None)), + ((1, None, "0"), (0, None)), + ((1, "64\n", "0"), (0, None)), +] + + +def fail(message): + raise SystemExit("FAIL: " + message) + + +def cut(main): + start = main.index(' const char *e = getenv("HOS_GUEST_MEMFD");') + end = main.index(" if (e ? atoi(e) != 0 : usable) {", start) + return main[start:end] + " if (e ? atoi(e) != 0 : usable) {\n" + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--cc", default="gcc") + a = p.parse_args() + src = (ROOT / "host/hos_main.c").read_text(encoding="utf-8") + dev = (ROOT / "docs/development.md").read_text(encoding="utf-8") + try: + text = FIXTURE.replace("@CUT@", cut(src)) + except ValueError as e: + fail("a text this test reads moved (%s)" % e) + if "`HOS_GUEST_MEMFD=0`" not in dev: + fail("docs/development.md has no HOS_GUEST_MEMFD row") + flags = subprocess.BELOW_NORMAL_PRIORITY_CLASS if os.name == "nt" else 0 + problems = [] + with tempfile.TemporaryDirectory(prefix="udmabuf-limit-") as tmp: + c = Path(tmp) / "fixture.c" + exe = Path(tmp) / ("fixture" + (".exe" if os.name == "nt" else "")) + c.write_text(text, encoding="utf-8") + built = subprocess.run([a.cc, "-std=gnu17", "-O1", "-Wall", "-Wno-unused-function", str(c), "-o", str(exe)], + capture_output=True, text=True, creationflags=flags) + if built.returncode: + print(built.stdout + built.stderr) + fail("the fixture did not build") + base = {k: v for k, v in os.environ.items() if k != "HOS_GUEST_MEMFD"} + for i, ((opens, limit, switch), (want, line)) in enumerate(CASES): + args = [str(exe), str(opens)] + if limit is not None: + path = Path(tmp) / ("limit%d" % i) + path.write_text(limit, encoding="ascii") + args.append(str(path)) + env = dict(base, **({"HOS_GUEST_MEMFD": switch} if switch is not None else {})) + done = subprocess.run(args, env=env, capture_output=True, text=True, timeout=60, creationflags=flags) + out = done.stdout.split("\n", 1) + got = int(out[0]) if not done.returncode and out[0].strip() in ("0", "1") else None + log = out[1] if len(out) > 1 else "" + if got != want or (line and line not in log): + problems.append("udmabuf %s, limit %r, HOS_GUEST_MEMFD=%r: memfd %s (wanted %d), log %r" + % ("opens" if opens else "does not open", limit, switch, got, want, log.strip())) + if problems: + fail("\n ".join([""] + problems)) + print("PASS: the guest window a memfd's where /dev/udmabuf opens, its limit unreadable too; not with a limit of 0 or no " + "udmabuf; HOS_GUEST_MEMFD over both (%d cases)" % len(CASES)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_upscale_fallback.py b/tests/test_upscale_fallback.py index 5932076..385814a 100644 --- a/tests/test_upscale_fallback.py +++ b/tests/test_upscale_fallback.py @@ -1,5 +1,10 @@ """Exercise the actual generic upscale fallback without Vulkan or a GPU. +(Triforce, 11-22, the performance audit's GPUWORK-21, after Source B's and Source A's; gpu/vk/stub_same_size.h.) A gameplay +network that cannot run is stood in for at its input's size too, and both gameplay networks take the typed upscale: the +cases 9dc, 9dc-same, fd-same, same and 9dc-pipeline-failure, and all of them once more with RX_VK_STUB_SAME_SIZE=0 (as +before: a same-size 9dc98871 declined, an upscaling one blitted). The rule's header is compiled with the function. + python tests/test_upscale_fallback.py [--source gpu/vk/vk_draw.cpp] [--cxx clang++] Use --cxx g++ on Linux, or --cxx cl in a configured Visual Studio compiler shell. No SDK needed. """ @@ -36,6 +41,7 @@ PREFIX = r''' #include #include #include +#include "stub_same_size.h" // ((Triforce, 11-22, GPUWORK-21) the rule upscale_stub asks, as it is) // Only the Vulkan declarations used by upscale_stub; no Vulkan SDK or runtime. using VkCommandBuffer = uintptr_t; @@ -200,15 +206,18 @@ void vkCmdBlitImage(VkCommandBuffer, VkImage a, VkImageLayout al, VkImage b, VkI bl == VK_IMAGE_LAYOUT_GENERAL && n == 1 && filter == VK_FILTER_LINEAR); assert(r->srcSubresource.mipLevel == 0 && r->srcSubresource.baseArrayLayer == 0 && r->dstSubresource.mipLevel == 0 && r->dstSubresource.baseArrayLayer == 0); - assert(r->srcOffsets[1].x == 64 && r->srcOffsets[1].y == 32 && r->dstOffsets[1].x == 128 && - r->dstOffsets[1].y == 64); + assert(r->srcOffsets[1].x == 64 && r->srcOffsets[1].y == 32 && r->dstOffsets[1].x == (int32_t)dst.width && + r->dstOffsets[1].y == (int32_t)dst.height); blits++; } ''' SUFFIX = r''' int main(int argc, char **argv) { - assert(argc == 2); + assert(argc == 2 || argc == 3); const std::string mode = argv[1]; + // ((Triforce, 11-22, GPUWORK-21) "off": run with RX_VK_STUB_SAME_SIZE=0, where everything is as it was before the rule) + const bool rule = !(argc == 3 && std::string(argv[2]) == "off"); + const bool same = mode == "same" || mode == "9dc-same" || mode == "fd-same"; uint64_t key = 0x0775215f8da720e6ull; destination.gpu = 20; destination.d.format = 63; @@ -233,9 +242,13 @@ int main(int argc, char **argv) { source.d.swizzle_r = 4; if (mode.compare(0, 2, "fd") == 0) key = 0xfd029a317a9a4b94ull; - if (mode == "9dc") + if (mode.compare(0, 3, "9dc") == 0) key = 0x9dc98871eb7fc101ull; - if (mode == "pipeline-failure" || mode == "fd-pipeline-failure") + if (same) { // (the output as large as the input: the network of Render resolution 100%) + destination.d.width = 64, destination.d.height = 32; + dst.width = 64, dst.height = 32; + } + if (mode == "pipeline-failure" || mode == "fd-pipeline-failure" || mode == "9dc-pipeline-failure") pipeline_ok = false; if (mode == "no-storage" || mode == "fd-no-storage") missing_storage = true; @@ -252,8 +265,25 @@ int main(int argc, char **argv) { NvnState state; state.tex[5][0] = 1; state.img[5][0] = 2; - assert(upscale_stub((VkCommandBuffer)5, state, key)); - if (mode == "captured" || mode == "fd") { + // ((Triforce, 11-22, GPUWORK-21) the same size: a gameplay network is stood in for, any other program declined; with the + // rule off every one is declined, as before, and nothing is drawn or written) + const bool taken = upscale_stub((VkCommandBuffer)5, state, key); + if (same && !(rule && mode != "same")) { + assert(!taken && typed_calls == 0 && blits == 0 && writes == 0 && views == 0 && movie_calls == 0); + printf("PASS: %s declined%s\n", mode.c_str(), rule ? "" : " (RX_VK_STUB_SAME_SIZE=0)"); + return 0; + } + assert(taken); + if (rule && (mode == "9dc" || mode == "9dc-same" || mode == "fd-same")) { + // (both gameplay networks through the typed upscale, at either size: nothing blitted) + assert(typed_calls == 1 && views == 2 && blits == 0 && writes == 1 && gameplay_warnings == 0); + assert(gameplay_logs == (mode == "fd-same" ? 1u : 0u)); + } else if (mode == "9dc-pipeline-failure") { + // (the typed upscale could not be made: the blit, and with the rule its warning of wrong colours; without it + // 9dc98871 never asked for the typed upscale) + assert(typed_calls == (rule ? 1u : 0u) && views == (rule ? 2u : 0u) && blits == 1 && writes == 1); + assert(gameplay_warnings == (rule ? 1u : 0u) && gameplay_logs == 0); + } else if (mode == "captured" || mode == "fd") { if (typed_calls != 1 || views != 2 || blits != 0) { puts("RED: RGB9E5 input still used physical-format blit"); return 7; @@ -277,10 +307,35 @@ int main(int argc, char **argv) { assert(gameplay_warnings == 1 && gameplay_logs == 0); if (mode == "fd-r11" || mode == "9dc") assert(gameplay_warnings == 0 && gameplay_logs == 0); + if (mode == "9dc" && !rule) assert(typed_calls == 0 && blits == 1); // (as before: the encoded bits blitted) printf("PASS: %s\n", mode.c_str()); } ''' +RULE = r''' +#include "stub_same_size.h" +#include +using namespace stub_same_size; +// (the rule as constants: it decides at compile time) +static_assert(sized(8192, 2048, false, false) && sized(8192, 2048, false, true), "an upscale's last pass is taken"); +static_assert(sized(2048, 2048, true, true), "a gameplay network at its input's size is taken"); +static_assert(!sized(2048, 2048, true, false), "not with the rule off"); +static_assert(!sized(2048, 2048, false, true), "not a program that is no gameplay network"); +static_assert(!sized(1024, 2048, true, true) && !sized(1024, 2048, false, false), "an output smaller than its input never"); +static_assert(!sized(0, 0, true, true), "nothing written is not taken"); +static_assert(typed(false, false, false) && typed(false, false, true), "a program that is no gameplay network asks for the typed upscale"); +static_assert(typed(true, true, false) && typed(true, true, true), "the 3/4 network does"); +static_assert(typed(true, false, true) && !typed(true, false, false), "the other gameplay network only with the rule"); +static_assert(warns(true, true, false) && warns(true, true, true) && warns(true, false, true), "the gameplay networks' blit warns"); +static_assert(!warns(true, false, false) && !warns(false, false, true) && !warns(false, false, false), "and no other"); +int main() { + const bool ok = on(nullptr) && on("1") && !on("0") && on("2"); + std::puts(ok ? "PASS: the rule" : "FAIL: RX_VK_STUB_SAME_SIZE is not on unless 0"); + return !ok; +} +''' + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--source', type=Path, default=ROOT / 'gpu/vk/vk_draw.cpp') @@ -289,26 +344,53 @@ def main(): body = block(args.source.read_text(encoding='utf-8'), 'bool upscale_stub(') env = os.environ.copy() env.pop('RX_VK_NO_UPSCALE_STUB', None) + env.pop('RX_VK_STUB_SAME_SIZE', None) + # ((Triforce, 11-22, GPUWORK-21) the rule's hooks, and its switch read once) + for line in ('static const bool same_size = stub_same_size::on(getenv("RX_VK_STUB_SAME_SIZE"));', + 'if (!stub_same_size::sized(out_area, in_area, gameplay, same_size))', + 'if (stub_same_size::typed(gameplay, gameplay_taau, same_size) && in.d.format == NVN_FORMAT_RGB9E5F', + 'if (stub_same_size::warns(gameplay, gameplay_taau, same_size) && in.d.format == NVN_FORMAT_RGB9E5F'): + if line not in body: + raise SystemExit('upscale_stub lost: ' + line) + if 'static const bool off = getenv("RX_VK_NO_UPSCALE_STUB") != nullptr;' not in body: + raise SystemExit('RX_VK_NO_UPSCALE_STUB no longer turns the stub off') + docs = (ROOT / 'docs/development.md').read_text(encoding='utf-8') + if '`RX_VK_STUB_SAME_SIZE=0`' not in docs or '`gpu/vk/stub_same_size.h`' not in docs: + raise SystemExit('docs/development.md: the switches table has no RX_VK_STUB_SAME_SIZE row') with tempfile.TemporaryDirectory(prefix='upscale-fallback-') as tmp: tmp = Path(tmp) source = tmp / 'check.cpp' exe = tmp / ('check.exe' if os.name == 'nt' else 'check') source.write_text(PREFIX + body + SUFFIX, encoding='utf-8') if Path(args.cxx).name.lower() in ('cl', 'cl.exe', 'clang-cl', 'clang-cl.exe'): - command = [args.cxx, '/nologo', '/std:c++20', '/EHsc', '/UNDEBUG', str(source), + command = [args.cxx, '/nologo', '/std:c++20', '/EHsc', '/UNDEBUG', '/I' + str(ROOT / 'gpu/vk'), str(source), '/Fo' + str(tmp / 'check.obj'), '/Fe' + str(exe)] cwd = tmp else: - command = [args.cxx, '-std=c++20', '-UNDEBUG', str(source), '-o', str(exe)] + command = [args.cxx, '-std=c++20', '-UNDEBUG', '-I' + str(ROOT / 'gpu/vk'), str(source), '-o', str(exe)] cwd = None subprocess.run(command, cwd=cwd, env=env, check=True) cases = ('captured', 'r11', 'movie', 'view', 'skip', 'mip', 'swizzle', 'fd', 'fd-r11', 'fd-view', 'fd-pipeline-failure', 'fd-no-storage', 'fd-no-e5-sample', 'fd-no-e5-linear', 'fd-null-input-view', 'fd-null-output-view', '9dc', 'pipeline-failure', 'no-storage', 'no-e5-sample', 'no-e5-linear', - 'null-input-view', 'null-output-view', 'fsr3') + 'null-input-view', 'null-output-view', 'fsr3', + '9dc-same', 'fd-same', 'same', '9dc-pipeline-failure') # ((Triforce, 11-22, GPUWORK-21)) for case in cases: subprocess.run([str(exe), case], env=env, check=True) + # (and all of them as before the rule: RX_VK_STUB_SAME_SIZE=0) + for case in cases: + subprocess.run([str(exe), case, 'off'], env={**env, 'RX_VK_STUB_SAME_SIZE': '0'}, check=True, stdout=subprocess.DEVNULL) + # (the rule itself) + rule = tmp / 'rule.cpp' + rule.write_text(RULE, encoding='utf-8') + rule_exe = tmp / ('rule.exe' if os.name == 'nt' else 'rule') + if Path(args.cxx).name.lower() in ('cl', 'cl.exe', 'clang-cl', 'clang-cl.exe'): + subprocess.run([args.cxx, '/nologo', '/std:c++20', '/EHsc', '/I' + str(ROOT / 'gpu/vk'), str(rule), + '/Fo' + str(tmp / 'rule.obj'), '/Fe' + str(rule_exe)], cwd=tmp, env=env, check=True) + else: + subprocess.run([args.cxx, '-std=c++20', '-I' + str(ROOT / 'gpu/vk'), str(rule), '-o', str(rule_exe)], env=env, check=True) + subprocess.run([str(rule_exe)], env=env, check=True) if __name__ == '__main__': main() diff --git a/tests/test_upscale_off.py b/tests/test_upscale_off.py new file mode 100644 index 0000000..cf32d87 --- /dev/null +++ b/tests/test_upscale_off.py @@ -0,0 +1,115 @@ +"""The upscaler Off: which network dispatches the plain stretch stands in for (gpu/vk/upscale_off.h; no game, no GPU). + +(Triforce, 11-22, the performance audit's GPUWORK-20, after Source A's.) With RX_VK_UPSCALE_OFF=1 the game's gameplay +upscale networks are not dispatched: their input is stretched linearly into their output. Before, only the first +network was stood in for, and only on cards that run the networks on software matrix multiplies. + +The rule, compiled and run: not Off, only a listed program is stood in for, with the stand-in's own FXAA and history +bound; Off, both networks known by key and every network of a gameplay upscale's shape, with the plain stretch, and the +ones taken by shape marked so that the caller looks at their input's format; a program that is neither is left alone. + +The source: taau_stand_in asks the rule, checks the scene's input format for a network taken by its shape, and hands +typed_upscale the rule's choice of stretch; compute_dispatch still gives DLSS and FSR the dispatch first; the compile +skips still ask taau_key alone (the networks are compiled as before, so leaving Off needs nothing made); the switch is +in the switches table. tests/test_taau_env_lookup.py runs taau_stand_in itself with Off on and off. + + python -I -B tests/test_upscale_off.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +DRIVER = r''' +#include "upscale_off.h" +#include +static int checks, fails; +static void want(const char* what, upscale_off::Choice c, bool stand_in, bool temporal, bool by_shape) { + checks++; + if (c.stand_in != stand_in || (stand_in && (c.temporal != temporal || c.by_shape != by_shape))) + fails++, printf("FAIL %s: stand_in %d temporal %d by_shape %d\n", what, c.stand_in, c.temporal, c.by_shape); +} +int main() { + using upscale_off::choose; + // listed off by_key shaped + want("not Off, nothing", choose(false, false, false, false), false, false, false); + want("not Off, a network by key", choose(false, false, true, true), false, false, false); + want("not Off, a share's network", choose(false, false, false, true), false, false, false); + want("not Off, the listed network", choose(true, false, true, true), true, true, false); + want("not Off, a listed other program", choose(true, false, false, false), true, true, false); + want("Off, a network by key", choose(false, true, true, true), true, false, false); + want("Off, a network by key, any shape", choose(false, true, true, false), true, false, false); + want("Off, a share's network", choose(false, true, false, true), true, false, true); + want("Off, a program that is neither", choose(false, true, false, false), false, false, false); + want("Off, the listed network", choose(true, true, true, true), true, false, false); + want("Off, a listed other program", choose(true, true, false, false), true, true, false); + want("Off, a listed program of the shape", choose(true, true, false, true), true, true, false); + if (fails) printf("upscale off: FAILED (%d of %d checks)\n", fails, checks); + else printf("upscale off: all %d checks passed\n", checks); + return fails != 0; +} +''' + + +def function(src, head): + at = src.index(head) + return src[at:src.index("\n}\n", at)] + + +def check_source(): + draw = (ROOT / "gpu/vk/vk_draw.cpp").read_text(encoding="utf-8") + if '#include "upscale_off.h"' not in draw: + raise SystemExit("vk_draw.cpp does not include upscale_off.h") + on = function(draw, "static bool upscale_off_on() {") + if 'getenv("RX_VK_UPSCALE_OFF")' not in on or 'lab_ab::on("upoff")' not in on: + raise SystemExit("upscale_off_on() does not read RX_VK_UPSCALE_OFF and the lab's upoff phases") + stand = function(draw, "bool taau_stand_in(VkCommandBuffer cmd, const NvnState& s, const Program& p) {") + order = [stand.find(n) for n in ( + "const bool off = upscale_off_on();", + "upscale_off::choose(taau_key(p.key), off, gameplay_network(p.key), shaped);", + "if (!ch.stand_in) return false;", + "if (ch.by_shape) {", + "if (!in || in->format != VK_FORMAT_E5B9G9R9_UFLOAT_PACK32) return false;", + "if (!out || out->format != VK_FORMAT_B10G11R11_UFLOAT_PACK32) return false;", + "done = typed_upscale(cmd, dw().imgs[0].imageView, dw().imgs[ntex].imageView, out, level, p.key, keys, ch.temporal);", + "if (done) hudless_after_dispatch(cmd, p.key);")] + if min(order) < 0 or order != sorted(order): + raise SystemExit("taau_stand_in no longer asks upscale_off.h, checks both ends' formats, and stretches as it chose: %s" % order) + if "st.info.uses_cooperative_matrix && st.info.texture_descriptors.size() == 1" not in stand: + raise SystemExit("taau_stand_in does not take the shares' networks by their shape") + # (DLSS, FSR 4 and FSR 3.1 take the dispatch first: Off is the game's own upscaler's) + at = [draw.find(n) for n in ("if (fsr3_stand_in(cmd, s, p)) return void(st_dispatches++);", + "if (temporal_stand_in(cmd, s, p)) return void(st_dispatches++);", + "if (taau_stand_in(cmd, s, p)) return void(st_dispatches++);")] + if min(at) < 0 or at != sorted(at): + raise SystemExit("compute_dispatch no longer gives FSR 3.1, then DLSS or FSR 4, the network before the stand-in") + # (the networks are still compiled under Off: the skips ask the env list alone) + rest = draw.replace(on, "").replace(stand, "") + if rest.count("taau_key(p.key)") < 5 or "upscale_off_on()" in rest: + raise SystemExit("a compile skip asks upscale_off_on(): Off would leave the networks unmade, and leaving Off would wait for them") + docs = (ROOT / "docs/development.md").read_text(encoding="utf-8") + if "`RX_VK_UPSCALE_OFF=1`" not in docs: + raise SystemExit("docs/development.md: RX_VK_UPSCALE_OFF is not in the switches table") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cxx", default="g++") + args = parser.parse_args() + check_source() + with tempfile.TemporaryDirectory(prefix="upscale-off-") as tmp: + cpp, exe = Path(tmp) / "upscale_off_test.cpp", Path(tmp) / "upscale-off.exe" + cpp.write_text(DRIVER, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-Wall", "-Wextra", "-Werror", "-I" + str(ROOT / "gpu/vk"), str(cpp), "-o", str(exe)], + check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + print(r.stdout.rstrip()) + if r.returncode: + raise SystemExit("FAIL: the upscaler Off's rule") + + +if __name__ == "__main__": + main() diff --git a/tests/test_upscaler_rows.cpp b/tests/test_upscaler_rows.cpp index 56ccd3e..b496e1c 100644 --- a/tests/test_upscaler_rows.cpp +++ b/tests/test_upscaler_rows.cpp @@ -235,6 +235,31 @@ int main() { !ur::preset_env("05") && ur::dlss_preset(ur::MODEL_M, "7") == 13 && ur::dlss_preset(ur::MODEL_L, "") == 12); CHECK("preset letters", ur::preset_letter(11) == 'K' && ur::preset_letter(13) == 'M' && ur::preset_letter(0) == '0'); + { // ((Triforce, 11-22, the performance audit's PRESENT-36) the K row on a card under 7 GB of video memory: preset E, NVIDIA's CNN + const unsigned long long GiB = 1ull << 30, MiB = 1ull << 20; + CHECK("CNN under 7 GB: the K row", ur::kPresetE == 5 && ur::kCnnVramBytes == 7 * GiB && ur::cnn_for_vram(ur::MODEL_K, 6 * GiB) && + ur::cnn_for_vram(ur::MODEL_K, 7 * GiB - 1) && !ur::cnn_for_vram(ur::MODEL_K, 7 * GiB) && + !ur::cnn_for_vram(ur::MODEL_K, 0)); + CHECK("CNN: the preset is E", ur::model_preset(ur::MODEL_K, 6 * GiB) == 5 && ur::model_preset(ur::MODEL_K, 7 * GiB - 1) == 5 && + ur::model_preset(ur::MODEL_K, 7 * GiB) == 11 && ur::model_preset(ur::MODEL_K, 0) == 11 && + ur::preset_letter(ur::model_preset(ur::MODEL_K, 6 * GiB)) == 'E'); + // (many cards, by what their drivers report: only the K row, only under 7 GB, never with the rule's memory at 0) + const unsigned mb[] = {2048, 3072, 4096, 5892, 6144, 7167, 7168, 8188, 8192, 10240, 11264, 12288, 16376, 24564, 32607}; + for (unsigned m : mb) { + const unsigned long long v = m * MiB; + const bool small = m < 7168; + CHECK("CNN: a card", ur::cnn_for_vram(ur::MODEL_K, v) == small && ur::model_preset(ur::MODEL_K, v) == (small ? 5u : 11u) && + ur::dlss_preset(ur::MODEL_K, nullptr, v) == (small ? 5u : 11u) && ur::dlss_preset(ur::MODEL_K, nullptr) == 11); + CHECK("CNN: M, L and NVIDIA's default as chosen", !ur::cnn_for_vram(ur::MODEL_M, v) && !ur::cnn_for_vram(ur::MODEL_L, v) && + !ur::cnn_for_vram(ur::MODEL_DEFAULT, v) && ur::model_preset(ur::MODEL_M, v) == 13 && + ur::model_preset(ur::MODEL_L, v) == 12 && ur::model_preset(ur::MODEL_DEFAULT, v) == 0 && + ur::dlss_preset(ur::MODEL_M, nullptr, v) == 13); + CHECK("CNN: a value past the row is K, as chosen", ur::model_preset(-1, v) == 11 && ur::model_preset(ur::kModels, v) == 11); + CHECK("CNN: RX_VK_DLSS_PRESET wins", ur::dlss_preset(ur::MODEL_K, "K", v) == 11 && ur::dlss_preset(ur::MODEL_K, "m", v) == 13 && + ur::dlss_preset(ur::MODEL_K, "0", v) == 0 && ur::dlss_preset(ur::MODEL_K, "E", v) == 5 && + ur::dlss_preset(ur::MODEL_K, "7", v) == (small ? 5u : 11u)); + } + } CHECK("model row shown with DLSS built", ur::model_shown(true) && !ur::model_shown(false)); CHECK("model row fixed with the game's own or the switch", ur::model_fixed(GAME, false) && !ur::model_fixed(DLSS, false) && !ur::model_fixed(DLAA, false) && ur::model_fixed(DLSS, true)); diff --git a/tests/test_upscaler_rows.py b/tests/test_upscaler_rows.py index 7f635e2..143ec19 100644 --- a/tests/test_upscaler_rows.py +++ b/tests/test_upscaler_rows.py @@ -100,6 +100,31 @@ def main(): raise SystemExit("vk_dlss.cpp: DLSS's feature is not made again for another DLSS model") if "upscaler_rows::dlss_preset(opts.dlss_model" not in function(dlss, "unsigned wanted_preset() {"): raise SystemExit("vk_dlss.cpp: the DLSS model row (and RX_VK_DLSS_PRESET over it) is not the preset asked for") + # ((Triforce, 11-22, the performance audit's PRESENT-36) the K row's CNN preset on a card under 7 GB: DLSS's preset and the + # menu's row ask the same function for the card's memory, which is 0 (K as chosen) unless RX_VK_DLSS_CNN_VRAM=1 + wanted = function(dlss, "unsigned wanted_preset() {") + if ("const uint64_t vram = dlss_cnn_vram();" not in wanted + or "upscaler_rows::dlss_preset(opts.dlss_model, env && *env ? env : nullptr, vram)" not in wanted): + raise SystemExit("vk_dlss.cpp: wanted_preset does not ask dlss_cnn_vram() for the card's memory") + text = function(menu, "const char* tx_dlss_model(int v) {") + if 'if (ur::cnn_for_vram(v, dlss_cnn_vram())) return "CNN (E)";' not in text or text.index("dlss_preset_env()") > text.index("dlss_cnn_vram()"): + raise SystemExit("vk_menu.cpp: the DLSS model row does not read CNN (E) where DLSS draws with it, after RX_VK_DLSS_PRESET") + hint = function(menu, "const char* hn_dlss_model(int v, int variant) {") + if ("ur::cnn_for_vram(v, dlss_cnn_vram()) ? 2 : 0" not in hint or "if (variant == 2) return v == ur::MODEL_K ? cnn : nullptr;" not in hint + or "under 7 GB" not in hint): + raise SystemExit("vk_menu.cpp: the DLSS model row's help does not say why it reads CNN (E)") + vram = function(dlss, "uint64_t dlss_cnn_vram() {") + if ('static const bool on = env_switch("RX_VK_DLSS_CNN_VRAM") == 1;' not in vram or "return on ? device_vram_bytes() : 0;" not in vram + or dlss.index("uint64_t dlss_cnn_vram() {") < dlss.index("#else // (no NGX SDK in this build")): + raise SystemExit("vk_dlss.cpp: dlss_cnn_vram is not off unless RX_VK_DLSS_CNN_VRAM=1, or not in a build without DLSS") + if "NVSDK_NGX_DLSS_Hint_Render_Preset_E == 5" not in feature: + raise SystemExit("vk_dlss.cpp: preset E's number is not checked against NVIDIA's") + if "uint64_t device_vram_bytes();" not in (ROOT / "gpu" / "vk" / "vk_internal.h").read_text(encoding="utf-8"): + raise SystemExit("vk_internal.h does not declare device_vram_bytes") + if "uint64_t dlss_cnn_vram();" not in (ROOT / "gpu" / "vk" / "vk_dlss.h").read_text(encoding="utf-8"): + raise SystemExit("vk_dlss.h does not declare dlss_cnn_vram") + if "| `RX_VK_DLSS_CNN_VRAM=1` |" not in (ROOT / "docs" / "development.md").read_text(encoding="utf-8"): + raise SystemExit("docs/development.md has no row for RX_VK_DLSS_CNN_VRAM") temporal = (ROOT / "gpu" / "vk" / "vk_temporal.cpp").read_text(encoding="utf-8") mode = function(temporal, "int temporal_mode() {") if "dlss_why() != DLSS_OK" not in mode or "fsr4_why() != FSR_OK" not in mode: diff --git a/tests/test_vbytes.py b/tests/test_vbytes.py new file mode 100644 index 0000000..3a628af --- /dev/null +++ b/tests/test_vbytes.py @@ -0,0 +1,196 @@ +"""TBL with a one-register table, EXT, 128-bit vector loads and DUP by element as nx2/fastvec.py rewrites them, bit-exact +with what the kit's translator writes for them (game-free). + +(Triforce, 11-21, the performance audit's LIFT-02, after Source A's.) The translator writes TBL and EXT as byte loops +through the stack, a 128-bit load as two halves and DUP by element as a loop, each followed by a 16-byte read that +cannot be forwarded from those stores. nx2/fastvec.py turns each into one instruction's helper on the lifted text. + +Guest words (TBL of 8 and 16 bytes; EXT of 16 bytes at every imm 0-15 and of 8 bytes at every imm 0-7; LDR Q, LDP Q +and a post-indexed LDR Q (LD1 is written byte by byte and is not one of them); DUP of every element size, index and width; with Vd apart, Vd = Vn, Vd = Vm, one register +for all, and registers above 15) go through a64emit.translate and a64_lift.op_text twice: as lifted now, and with +A64_NO_VBYTES=1 and A64_NO_VMOVES=1 (the kit's text). The first must call the helper and keep no byte loop; the second +must not have the helper. tests/vbytes_test.c then runs both from the same random Ctx and guest memory (index bytes +biased to the edges: 15, 16, 0x6f-0x90, 0xff) and compares every vector and general register. Built at -march=x86-64-v2 +(SSSE3: PSHUFB and PALIGNR), at -march=x86-64 (SSE2 only: the plain C fallback) and at -march=native. + +And reads the source: nx2/lift.py installs the passes, and the switches are in the switches table. + + python -I -B tests/test_vbytes.py [--cc clang] [--cxx clang++] [--random 40000] [--time] + +Skipped (exit 0) on a host that is not x86-64 or has no C or C++ compiler. +""" +import argparse +import os +import platform +import sys +import tempfile +import time + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(ROOT, "tests")) +sys.path.insert(0, os.path.join(ROOT, "nx2")) +import test_fpx_sticky as fs # noqa: E402 (the build helpers the kit's other lifted-code test uses) + +RUN_ALL_ARGS = ["--cc", "clang", "--cxx", "{cxx}"] +SEED = 20261121 +MARCHES = ("x86-64-v2", "x86-64", "native") +REGS = ((0, 5, 1), (5, 5, 1), (1, 5, 1), (2, 2, 2), (3, 30, 17)) # (rd, rn, rm) + + +def tbl(q, rd, rn, rm): + return 0x0E000000 | (q << 30) | (rm << 16) | (rn << 5) | rd + + +def ext(q, rd, rn, rm, imm): + return 0x2E000000 | (q << 30) | (rm << 16) | (imm << 11) | (rn << 5) | rd + + +def ldr_q(rt, rn, imm): + return 0x3DC00000 | ((imm // 16) << 10) | (rn << 5) | rt + + +def ldp_q(rt, rt2, rn, imm): + return 0xAD400000 | (((imm // 16) & 0x7f) << 15) | (rt2 << 10) | (rn << 5) | rt + + +def ldr_q_post(rt, rn, imm9): + return 0x3CC00400 | ((imm9 & 0x1ff) << 12) | (rn << 5) | rt + + +def dup_elem(q, size, idx, rd, rn): + imm5 = (idx << (size + 1)) | (1 << size) + return 0x0E000400 | (q << 30) | (imm5 << 16) | (rn << 5) | rd + + +def cases(): + """(name, guest word, the helper its lifted text must call)""" + out = [] + for q in (0, 1): + for r in REGS: + out.append(("TBL v%d.%s, {v%d.16b}, v%d" % (r[0], "16b" if q else "8b", r[1], r[2]), tbl(q, *r), "a64_tbl1(")) + for q, n in ((1, 16), (0, 8)): + for imm in range(n): + for r in (REGS if imm in (0, 1, n // 2, n - 1) else REGS[:1]): + out.append(("EXT v%d.%s, v%d, v%d, #%d" % (r[0], "16b" if q else "8b", r[1], r[2], imm), ext(q, *r, imm), + "A64_EXT%d(" % n)) + out += [("LDR q3, [x1, #32]", ldr_q(3, 1, 32), "a64_vld16("), ("LDR q31, [x7]", ldr_q(31, 7, 0), "a64_vld16("), + ("LDP q2, q9, [x1, #-64]", ldp_q(2, 9, 1, -64), "a64_vld16("), ("LDR q5, [x3], #48", ldr_q_post(5, 3, 48), "a64_vld16(")] + for size, lanes in ((0, 16), (1, 8), (2, 4), (3, 2)): + for q in ((1,) if size == 3 else (0, 1)): + for idx in sorted({0, 1, lanes - 1}): + for rd, rn in ((6, 2), (2, 2), (17, 30)): + out.append(("DUP v%d.%d%s, v%d[%d]" % (rd, lanes if q else lanes // 2, "bhsd"[size], rn, idx), + dup_elem(q, size, idx, rd, rn), "a64_vdup(")) + return out + + +def lifted(a64emit, a64_lift, used): + """vbytes_lifted.h: each guest word as lifted now and as the kit writes it, as functions, and the CASES table""" + all_cases = cases() + texts, flags = a64emit.translate([(0x1000 + 4 * i, w) for i, (_, w, _) in enumerate(all_cases)]) + if any(flags): + raise SystemExit("vbytes: a64emit did not translate the guest words: %s" % flags) + now = [a64_lift.op_text(t, used) for t in texts] + os.environ["A64_NO_VBYTES"] = os.environ["A64_NO_VMOVES"] = "1" + try: + old = [a64_lift.op_text(t, used) for t in texts] + finally: + del os.environ["A64_NO_VBYTES"], os.environ["A64_NO_VMOVES"] + out = ["/* Generated by tests/test_vbytes.py: each guest word as lifted now and as the kit's translator writes it. */"] + for i, (n, o, (name, _, helper)) in enumerate(zip(now, old, all_cases)): + if helper not in n or "for(int _i" in n: + raise SystemExit("vbytes: %s is not rewritten: %s" % (name, n[:300])) + if helper in o: + raise SystemExit("vbytes: %s has the helper with the switch set: %s" % (name, o[:300])) + out.append("static void g_now%d(Ctx *__restrict c) { %s }" % (i, n)) + out.append("static void g_old%d(Ctx *__restrict c) { %s }" % (i, o)) + out.append("static const struct { const char *name; void (*now)(Ctx *), (*old)(Ctx *); } CASES[] = {%s};" + % ", ".join('{"%s", g_now%d, g_old%d}' % (c[0], i, i) for i, c in enumerate(all_cases))) + return "\n".join(out) + "\n", len(all_cases) + + +def generate(cxx, tmp, flags): + """a64_fpx.h, a64_suyu.h, a64_suyu.c and vbytes_lifted.h, by the generators the lift runs (a64emit built into tmp)""" + dll = os.path.join(tmp, "a64emit.dll" if os.name == "nt" else "liba64emit.so") + fs.build([cxx, "-std=c++20", "-O0", "-shared", "-Wno-everything", "-o", dll, os.path.join(fs.KIT, "lift", "a64emit.cpp")] + + flags + ([] if os.name == "nt" else ["-fPIC"]), "a64emit") + import build_emit + build_emit.build = lambda force=False: dll # (not a64emit.dll beside the source) + import a64_lift + import a64_rtgen + import a64emit + import fastvec # (the passes under test, around the kit's) + fastvec.install() + used = set() + try: + a64_rtgen.write(tmp) + text, n = lifted(a64emit, a64_lift, used) + finally: + if a64emit._lib is not None: # (unloaded, so the folder can go) + import _ctypes + (_ctypes.FreeLibrary if os.name == "nt" else _ctypes.dlclose)(a64emit._lib._handle) + a64emit._lib = None + with open(os.path.join(tmp, "vbytes_lifted.h"), "w", newline="\n") as f: + f.write(text) + header = a64_lift.fpx_header(used) + for helper in ("a64_tbl1(", "A64_EXT16(", "A64_EXT8(", "a64_vld16(", "a64_vdup("): + if header.count("void %s" % helper) + header.count("#define %s" % helper) < 1: + raise SystemExit("vbytes: a64_fpx.h has no %s: fpx_header no longer carries the helpers" % helper) + with open(os.path.join(tmp, "a64_fpx.h"), "w", newline="\n") as f: + f.write(header) + return n + + +def check_sources(): + with open(os.path.join(ROOT, "nx2", "lift.py"), encoding="utf-8") as f: + lift = f.read() + at = [lift.find(p) for p in ("import fastvec", "lift_fixes.install()", "fastvec.install()")] + if min(at) < 0 or at[1] > at[2]: + raise SystemExit("vbytes: nx2/lift.py does not install nx2/fastvec.py's passes after the kit's fixes") + with open(os.path.join(ROOT, "docs", "development.md"), encoding="utf-8") as f: + if "`A64_NO_VBYTES=1`, `A64_NO_VMOVES=1`" not in f.read(): + raise SystemExit("vbytes: docs/development.md: A64_NO_VBYTES and A64_NO_VMOVES are not in the switches table") + + +def main(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--cc") + p.add_argument("--cxx") + p.add_argument("--random", type=int, default=40000) + p.add_argument("--time", action="store_true", help="also time each kind of guest word, now and as the kit writes it") + a = p.parse_args() + check_sources() + if platform.machine().lower() not in ("x86_64", "amd64"): + fs.skip("not an x86-64 host: %s" % platform.machine()) + cc = fs.compiler(a.cc, "CC", ("clang", "gcc", "cc"), "clang") + cxx = fs.compiler(a.cxx, "CXX", ("clang++", "g++", "c++"), "clang++") + if not cc or not cxx: + fs.skip("no C compiler" if not cc else "no C++ compiler") + t0 = time.time() + failed = [] + with tempfile.TemporaryDirectory(prefix="triforce-vbytes-") as tmp: + n = generate(cxx, tmp, fs.target_flags(cxx, tmp)) + for march in MARCHES: + exe = os.path.join(tmp, "vbytes_test_%s.exe" % march) + fs.build([cc] + fs.target_flags(cc, tmp) + ["-march=" + march] + fs.FLAGS + + ["-D_CRT_SECURE_NO_WARNINGS", "-I" + os.path.join(fs.KIT, "rt"), "-I" + tmp, + os.path.join(ROOT, "tests", "vbytes_test.c"), os.path.join(tmp, "a64_suyu.c"), "-o", exe], + "the driver at " + march) + r = fs.run([exe, str(a.random), str(SEED)], timeout=1200) + print("%-10s %s" % (march, r.stdout.rstrip())) + if r.returncode: + failed.append("%s (exit %d)" % (march, r.returncode)) + if a.time and not failed: # (at the game's own -O1: what each rewrite is worth by itself) + exe = os.path.join(tmp, "vbytes_time.exe") + fs.build([cc] + fs.target_flags(cc, tmp) + ["-march=native", "-ffp-contract=off", "-fno-strict-aliasing", "-O1", + "-D_CRT_SECURE_NO_WARNINGS", "-I" + os.path.join(fs.KIT, "rt"), "-I" + tmp, + os.path.join(ROOT, "tests", "vbytes_test.c"), os.path.join(tmp, "a64_suyu.c"), "-o", exe], "the timing driver") + print("timing at -O1 -march=native:") + print(fs.run([exe, "time"], timeout=1200).stdout.rstrip()) + if failed: + raise SystemExit("FAIL: vbytes: %s" % ", ".join(failed)) + print("PASS: %d guest words rewritten by nx2/fastvec.py leave the same registers as the kit's text (%.0f s)" % (n, time.time() - t0)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_vcore_wide.py b/tests/test_vcore_wide.py new file mode 100644 index 0000000..b39a40d --- /dev/null +++ b/tests/test_vcore_wide.py @@ -0,0 +1,339 @@ +"""Virtual cores: HOS_VCORE_WIDE=1 lets a game thread whose mask names two or more cores take any free core (host/ +hos_thread.c vc_allowed; no game). + +(Triforce, 11-22, the performance audit's GAME-22, after Source B's.) Off unless set. A thread with a one-core mask, +which guards the game's lock-free per-core data, and a thread with no mask stay where they were, with the switch or +without it. + +host/hos_thread.c's virtual cores (vc_lock to vc_yield) are cut out unchanged and run on real threads with tests/ +vcore_spin_test.c's stand-ins (pthreads for Windows' lock and condition variable, its clock, its hold() that fails when a +core is run by two threads). The scripted cases stand on the test's own clock and wait for states, never for a time: + +- two cores held, the third free: a thread whose mask names the two held ones waits for one of them without the switch + (and gets it when its holder lets go: the wake-up is not lost), and takes the free third at once with it; the core + number it then answers is the core it holds; +- all three held, the third let go: with the switch the release grants it to that waiter; without it the waiter goes on + waiting for one of its own two, and gets it at that one's release; +- a thread with a one-core mask and a thread with no mask (its ideal core) wait for their own core beside two free + ones, with the switch too. +Two broken rules (a one-core mask widened, a thread with no mask widened) must fail the script with the switch on. + +A storm, with the switch and without: eight threads of three priorities and masks of one, two and three cores hand three +cores to each other tens of thousands of times; no core is ever run by two threads, none runs without a core, a +one-core thread only ever holds its own core, without the switch every thread only cores of its mask, and nothing is +left owned or queued at the end. + + python -I -B tests/test_vcore_wide.py [--cc clang] +""" +import argparse +import os +import shutil +import subprocess +import tempfile +from pathlib import Path + +RUN_ALL_ARGS = ["--cc", "clang"] + +ROOT = Path(__file__).resolve().parents[1] +FAR = "60000000" # (60 s: no escape and no time slice in these cases) + + +def fail(message): + raise SystemExit("FAIL: " + message) + + +def block(source, marker): + start = source.index(marker) + end = source.index("{", start) + 1 + depth = 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[start:end] + + +CASES = r''' +/* ---- (Triforce, GAME-22) HOS_VCORE_WIDE ---- */ +typedef struct { + HosThread t; + int got, go; /* got: the core it holds + 1 once it has one */ +} Taker; +static void *take(void *arg) { + Taker *k = (Taker *)arg; + vc_acquire(&k->t); + __atomic_store_n(&k->got, k->t.vc_held, __ATOMIC_RELEASE); + hold(&k->t); /* (fails when another thread runs on that core, or when it has none) */ + wait_for(&k->go, 1); + vc_release(&k->t); + return NULL; +} +static void taker(Taker *k, const char *name, int priority, uint64_t mask, int ideal, pthread_t *th) { + memset(k, 0, sizeof *k); + thread_init(&k->t, name, priority, mask); + k->t.ideal_core = ideal; + pthread_create(th, NULL, take, k); +} +static int64_t waits_now(void) { + AcquireSRWLockExclusive(&vc_lock); + const int64_t v = vc_st.waits; + ReleaseSRWLockExclusive(&vc_lock); + return v; +} +static void done(Taker *k, pthread_t th) { + __atomic_store_n(&k->go, 1, __ATOMIC_RELEASE); + pthread_join(th, NULL); +} +/* waits until k waits for a core; a taker that took one instead ends the case at once (it would never queue) */ +static void must_wait(Taker *k) { + while (!__atomic_load_n(&k->t.vc_queued, __ATOMIC_ACQUIRE)) { + const int got = __atomic_load_n(&k->got, __ATOMIC_ACQUIRE); + if (got) { + printf("FAIL: %s took core %d instead of waiting for a core of its own\n", k->t.name, got - 1); + fflush(stdout); + _Exit(1); + } + sleep_us(200); + } +} +static void script(int wide) { + static HosThread h0, h1, h2; + static Taker a, b, n; + pthread_t ta, tb, tn; + cores = 3; + test_clock = 1000000000000LL; /* (standing still: no time slice, no escape) */ + thread_init(&h0, "holder 0", 10, 1); + thread_init(&h1, "holder 1", 10, 2); + thread_init(&h2, "holder 2", 10, 4); + + /* 1. cores 0 and 1 held, core 2 free: a thread of cores 0 and 1 (worse than the holders: it asks for nothing) */ + vc_acquire(&h0), vc_acquire(&h1); + CHECK(h0.vc_held == 1 && h1.vc_held == 2); + taker(&a, "two cores", 20, 3, -2, &ta); + if (wide) { + wait_for(&a.got, 1); + CHECK(a.got == 3); /* the free third core, at once */ + CHECK(waits_now() == 0); + CHECK(hos_thread_core(&a.t) == 2); /* GetCurrentCoreNumber: the core it holds, not one of its mask */ + } else { + must_wait(&a); + wait_for(&sleeps_begun, 1); + CHECK(a.got == 0 && a.t.vc_held == 0); /* it waits beside a free core, as before */ + vc_release(&h0); /* the waker: one of its own cores let go */ + wait_for(&a.got, 1); /* (and the wake-up is not lost) */ + CHECK(a.got == 1); + vc_acquire(&h2); /* (core 2 was free all along) */ + CHECK(h2.vc_held == 3); + vc_release(&h2); + } + done(&a, ta); + if (wide) vc_release(&h0); + vc_release(&h1); + end_state(); + + /* 2. all three held; the waiter of cores 0 and 1 is asleep; core 2 is let go, then core 0 */ + vc_acquire(&h0), vc_acquire(&h1), vc_acquire(&h2); + int slept = __atomic_load_n(&sleeps_begun, __ATOMIC_ACQUIRE); + taker(&a, "two cores", 20, 3, -2, &ta); + must_wait(&a); + wait_for(&sleeps_begun, slept + 1); + vc_release(&h2); /* (the grant is made inside the release) */ + if (wide) { + wait_for(&a.got, 1); + CHECK(a.got == 3); /* granted the core outside its mask */ + } else { + CHECK(__atomic_load_n(&a.t.vc_held, __ATOMIC_ACQUIRE) == 0 && a.t.vc_queued == 1); /* not its core: it goes on waiting */ + vc_release(&h0); + wait_for(&a.got, 1); + CHECK(a.got == 1); + } + done(&a, ta); + if (wide) vc_release(&h0); + vc_release(&h1); + end_state(); + + /* 3. core 0 held, cores 1 and 2 free: a one-core thread and a thread with no mask wait for core 0, the switch or not */ + vc_acquire(&h0); + slept = __atomic_load_n(&sleeps_begun, __ATOMIC_ACQUIRE); + taker(&b, "one core", 20, 1, -2, &tb); + must_wait(&b); + clock_step(1000); /* (the one-core thread has waited longer) */ + taker(&n, "no mask", 20, 0, 0, &tn); + must_wait(&n); + wait_for(&sleeps_begun, slept + 2); + CHECK(b.got == 0 && n.got == 0); + AcquireSRWLockExclusive(&vc_lock); + CHECK(vc_owner[1] == NULL && vc_owner[2] == NULL); /* (two cores stood free the whole time) */ + ReleaseSRWLockExclusive(&vc_lock); + vc_release(&h0); /* the waker: core 0 goes to the one that waited longest */ + wait_for(&b.got, 1); + CHECK(b.got == 1 && n.got == 0); + done(&b, tb); /* and then to the other */ + wait_for(&n.got, 1); + CHECK(n.got == 1); + done(&n, tn); + end_state(); +} + +/* the storm: as vcore_spin_test.c's, with masks of one, two and three cores */ +static long outside; /* (holds of a core outside the thread's mask) */ +static void *wide_thread(void *arg) { + Worker *w = (Worker *)arg; + HosThread *t = &w->t; + __atomic_add_fetch(&storm_ready, 1, __ATOMIC_ACQ_REL); + while (__atomic_load_n(&storm_ready, __ATOMIC_ACQUIRE) < 8) sched_yield(); + vc_acquire(t); + for (int i = 0; i < w->iters && !__atomic_load_n(&failures, __ATOMIC_RELAXED); i++) { + hold(t); + if (t->vc_held && !(t->core_mask >> (t->vc_held - 1) & 1)) { + if (__builtin_popcountll(t->core_mask) == 1) FAIL("%s, a one-core thread, ran on core %d", t->name, t->vc_held - 1); + __atomic_add_fetch(&outside, 1, __ATOMIC_RELAXED); + } + const unsigned r = rnd(&w->seed) % 100; + if (r < 50) { + if (t->vc_kick) vc_yield(t, t->vc_kick == 1); + } else if (r < 65) { + vc_yield(t, 0); + } else { + vc_release(t); + if (r < 80) sched_yield(); + else if (r < 90) + for (unsigned k = rnd(&w->seed) % 2000; k; k--) _mm_pause(); + vc_acquire(t); + } + if (!t->vc_held) FAIL("%s came back without a core", t->name); + } + vc_release(t); + return NULL; +} +static void wide_storm(int wide) { + static Worker w[8]; + static const int prio[8] = {10, 10, 16, 16, 16, 28, 28, 28}; + static const uint64_t mask[8] = {3, 1, 6, 2, 4, 5, 3, 7}; + pthread_t th[8]; + for (int i = 0; i < 8; i++) { + char name[16]; + snprintf(name, sizeof name, "w%d", i); + thread_init(&w[i].t, name, prio[i], mask[i]); + w[i].iters = 25000; + w[i].seed = 4321u + (unsigned)i * 7919u; + pthread_create(&th[i], NULL, wide_thread, &w[i]); + } + for (int i = 0; i < 8; i++) pthread_join(th[i], NULL); + for (int i = 0; i < 8; i++) CHECK(!w[i].t.vc_held && !w[i].t.vc_queued); + end_state(); + if (!wide) CHECK(outside == 0); /* without the switch: only ever a core of its mask */ + AcquireSRWLockExclusive(&vc_lock); + printf("SEEN: %lld waits, %ld holds of a core outside the mask\n", (long long)vc_st.waits, outside); + ReleaseSRWLockExclusive(&vc_lock); +} +int main(int argc, char **argv) { + const char *c = argc > 1 ? argv[1] : ""; + const int wide = getenv("HOS_VCORE_WIDE") && atoi(getenv("HOS_VCORE_WIDE")) != 0; + { + pthread_t dog; + pthread_create(&dog, NULL, watchdog, NULL); + pthread_detach(dog); + } + if (!strcmp(c, "script")) script(wide); + else if (!strcmp(c, "storm")) wide_storm(wide); + else FAIL("unknown case %s", c); + if (failures) return 1; + printf("PASS %s\n", c); + return 0; +} +''' + +RULE = " if (wide && __builtin_popcount(m) > 1) return all;\n" +# (each: what the broken rule widens, what replaces the rule's line) +MUTANTS = [ + ("a one-core mask", " if (wide && __builtin_popcount(m) > 0) return all;\n"), + ("a thread with no mask", " if (wide && __builtin_popcount(m) != 1) return all;\n"), +] + + +def read_source(thread, cores, dev): + allowed = block(cores, "static unsigned vc_allowed(HosThread *t) {") + read = 'if (wide < 0) wide = getenv("HOS_VCORE_WIDE") && atoi(getenv("HOS_VCORE_WIDE")) != 0;' + if "static int wide = -1;" not in allowed or read not in allowed: + fail("vc_allowed does not read HOS_VCORE_WIDE once, off unless it is a number other than 0") + if not allowed.index("if (any) return all;") < allowed.index("unsigned m = (unsigned)(t->core_mask & all);") < allowed.index(RULE) \ + < allowed.index("if (!m) m = 1u << hos_thread_core(t);"): + fail("vc_allowed: the widening is not of the game's own mask, before a thread with no mask is given its core") + hle = block(thread, "HLE(_ZN2nn2os17SetThreadCoreMaskEPNS0_10ThreadTypeEim) {") + if "core_mask_note(t, (int)ARG(1), ARG(2));" not in hle: + fail("SetThreadCoreMask no longer notes what the game asks for") + note = block(thread, "static void core_mask_note(HosThread *t, int ideal, uint64_t mask) {") + if "if (named < 2 && !every && !hos_trace) return;" not in note or 'hos_log("vcores: %s (priority %d) asks for ideal core %d' not in note: + fail("core_mask_note does not log a mask of two or more cores always") + if "| `HOS_VCORE_WIDE=1` |" not in dev: + fail("docs/development.md: HOS_VCORE_WIDE is not in the switches table") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cc", default="clang") + args = parser.parse_args() + thread = (ROOT / "host/hos_thread.c").read_text(encoding="utf-8") + dev = (ROOT / "docs/development.md").read_text(encoding="utf-8") + try: + cores = thread[thread.index("static SRWLOCK vc_lock = SRWLOCK_INIT;"):thread.index("int hos_vcore_away(void) {")] + read_source(thread, cores, dev) + spin = (ROOT / "tests/vcore_spin_test.c").read_text(encoding="utf-8") + stand_ins = spin[:spin.index("/* ---- the scripted hand-off")] # (the stand-ins, the helpers, hold() and the storm's pieces) + except ValueError as e: + fail("a text this test reads is not there (%s)" % e) + if cores.count(RULE) != 1: + fail("the rule's line is not in the virtual cores once") + compiler = shutil.which(args.cc) + if not compiler: + raise SystemExit("C compiler not found: " + args.cc) + env = {k: v for k, v in os.environ.items() if not k.startswith("HOS_")} + env.update(HOS_VCORE_ESCAPE_US=FAR, HOS_VCORE_ESCAPE_EQ_US=FAR) + flags = (subprocess.CREATE_NO_WINDOW | subprocess.BELOW_NORMAL_PRIORITY_CLASS) if os.name == "nt" else 0 + with tempfile.TemporaryDirectory(prefix="vcore-wide-") as temporary: + top = Path(temporary) + + def build(name, block_text): + folder = top / name + folder.mkdir() + (folder / "vcore_block.inc").write_text(block_text, encoding="utf-8") + (folder / "wide.c").write_text(stand_ins + CASES, encoding="utf-8") + exe = folder / "wide.exe" + built = subprocess.run([compiler, "-std=gnu17", "-O2", "-Wall", "-Werror", "-Wno-unused-function", "-Wno-unused-variable", + "-Wno-unknown-warning-option", "-Wno-unused-but-set-variable", "-Wno-unused-but-set-global", + "-pthread", "-I" + str(folder), str(folder / "wide.c"), "-o", str(exe)], + capture_output=True, text=True, timeout=180) + if built.returncode: + print(built.stdout + built.stderr) + fail("the fixture did not build (%s)" % name) + return exe + + def run(exe, case, switches): + return subprocess.run([str(exe), case], env=dict(env, **switches), capture_output=True, text=True, encoding="utf-8", + errors="replace", timeout=120, creationflags=flags) + + exe = build("as-made", cores) + for what, switches in (("not set", {}), ("0", {"HOS_VCORE_WIDE": "0"}), ("1", {"HOS_VCORE_WIDE": "1"})): + done = run(exe, "script", dict(switches, HOS_VCORE_SLICE_US=FAR, HOS_VCORE_SPIN_US="0")) + if done.returncode or "PASS script" not in done.stdout: + print(done.stdout[-4000:]) + fail("the scripted cases, HOS_VCORE_WIDE %s" % what) + print("PASS script, HOS_VCORE_WIDE %s: %s" % (what, "a two-core thread takes any free core, one-core and no-mask threads stay" + if what == "1" else "every thread waits for a core of its own, as before")) + for what, switches in (("not set", {}), ("1", {"HOS_VCORE_WIDE": "1"})): + done = run(exe, "storm", switches) + seen = [line for line in done.stdout.splitlines() if line.startswith("SEEN: ")] + if done.returncode or "PASS storm" not in done.stdout or not seen: + print(done.stdout[-4000:]) + fail("the storm, HOS_VCORE_WIDE %s" % what) + print("PASS storm, HOS_VCORE_WIDE %s: no core run by two threads, none without a core; %s" % (what, seen[0][6:])) + for i, (what, new) in enumerate(MUTANTS): + done = run(build("mutant%d" % i, cores.replace(RULE, new)), "script", {"HOS_VCORE_WIDE": "1", "HOS_VCORE_SLICE_US": FAR, "HOS_VCORE_SPIN_US": "0"}) + if done.returncode == 0: + fail("a rule that widens %s too still passed the script" % what) + print("PASS mutation: a rule that widens %s too fails the script" % what) + print("PASS: HOS_VCORE_WIDE=1 widens only masks of two or more cores, no core is ever run by two threads, and it is off unless set") + + +if __name__ == "__main__": + main() diff --git a/tests/test_vecfp.py b/tests/test_vecfp.py new file mode 100644 index 0000000..3788239 --- /dev/null +++ b/tests/test_vecfp.py @@ -0,0 +1,291 @@ +"""The SSE fast paths nx2/vecfp.py puts before float forms lifted lane by lane (A64_VECFP), bit-exact with the kit's +text and with the exact bodies (game-free). + +(Triforce, 11-22, the performance audit's LIFT-03, after Source A's.) With A64_VECFP=1 at lift time (off by default) +FMUL .2S, FMLA and FMLS .2S and .4S, FDIV .2S and .4S, FMUL / FMLA / FMLS .2S by element, FMAX / FMIN / FMAXNM / FMINNM +(scalar S, .2S, .4S), FCVT S to D and D to S, FCVTL, FCVTL2 and FCVTN are lifted as +`if (!a64_vfpx_q_(c, ...)) { }`: the helper does the op in SSE and writes the register only when +the scalar fast paths' rules prove the result is Arm's with no new FPSR bit. + +The rule (nx2/vecfp.py groups, fast_call): A64_VECFP unset, empty or 0 wraps nothing; 1 wraps every form; a list wraps +its groups (elem, vec, minmax, fcvt) and anything else stops the lift. Forms the kit's own passes vectorize (FMUL .4S, +by element .4S, FADD, FSUB) and the double-precision and pairwise forms are never wrapped. + +The lifted text: 137 guest words (every form with Vd apart, Vd = Vn, Vd = Vm, one register for all and registers +above 15; by element with indices 0, 1 and 3) go through a64emit.translate under insn.install() and vecfp.install(), +and a64_lift.op_text, three times: with A64_VECFP=1, without it (the kit's text) and with the fast path off (the exact +body inline). The first must be the helper's test around the second's text and nothing else; no line of it may start +with `}` (nx2/oot_patches.py ends a function at the first such line); the kit's mark grouping must take the helpers as +plain (a64_lift.MG_PLAIN: no run of stores ends at one). + +The run: tests/vecfp_test.c runs all three from the same random Ctx (operands biased to zeros of both signs, +subnormals, infinities, NaNs of both kinds with payloads, the extreme normals, equal operands, tiny and huge products, +cancelling addends; FPCR 0, FZ, or other control bits; FPSR with the gate open and closed; every NZCV combination) and +compares the whole Ctx, FPSR in it. Built at -march=x86-64-v3 and -march=native (FMA3: the multiply-adds are one fused +op), -march=x86-64-v2 and -march=x86-64 (no FMA3: they must decline every state). Every other word's helper must do the +op in some states and decline in others, or the comparison would show nothing. + +And reads the source: nx2/lift.py installs it after insn's, A64_VECFP is an A64_ name (in nx2/make_game.py's lift key, +as nx2/vecfp.py is), and the switch is in the switches table. + + python -I -B tests/test_vecfp.py [--cc clang] [--cxx clang++] [--random 20000] + +Skipped (exit 0) on a host that is not x86-64 or has no C or C++ compiler. +""" +import argparse +import os +import platform +import re +import sys +import tempfile +import time + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(ROOT, "tests")) +sys.path.insert(0, os.path.join(ROOT, "nx2")) +import test_fpx_sticky as fs # noqa: E402 (the build helpers the kit's other lifted-code tests use) + +RUN_ALL_ARGS = ["--cc", "clang", "--cxx", "{cxx}"] +SEED = 20261122 +MARCHES = ("x86-64-v3", "x86-64-v2", "x86-64", "native") +SWITCH = "A64_VECFP" +REGS = ((0, 5, 1), (5, 5, 1), (1, 5, 1), (2, 2, 2), (3, 30, 17)) # (rd, rn, rm) + + +def three(base, rd, rn, rm): + return base | (rm << 16) | (rn << 5) | rd + + +def cases(): + """(name, guest word, rd, rn, rm, group, binary64 source, needs FMA3, helper)""" + out = [] + for name, base, fused, helper in (("FMUL .2s", 0x2E20DC00, 0, "a64_vfpx_q_mul("), ("FMLA .2s", 0x0E20CC00, 1, "a64_vfpx_q_fma("), + ("FMLA .4s", 0x4E20CC00, 1, "a64_vfpx_q_fma("), ("FMLS .2s", 0x0EA0CC00, 1, "a64_vfpx_q_fma("), + ("FMLS .4s", 0x4EA0CC00, 1, "a64_vfpx_q_fma("), ("FDIV .2s", 0x2E20FC00, 0, "a64_vfpx_q_div("), + ("FDIV .4s", 0x6E20FC00, 0, "a64_vfpx_q_div(")): + for r in REGS: + out.append(("%s v%d, v%d, v%d" % ((name,) + r), three(base, *r), r[0], r[1], r[2], "vec", 0, fused, helper)) + for name, base, fused, helper in (("FMUL .2s", 0x0F809000, 0, "a64_vfpx_q_mule("), ("FMLA .2s", 0x0F801000, 1, "a64_vfpx_q_fmae("), + ("FMLS .2s", 0x0F805000, 1, "a64_vfpx_q_fmae(")): + for index in (0, 1, 3): + for r in ((0, 5, 1), (5, 5, 1), (2, 2, 2)): + w = three(base, *r) | ((index >> 1) << 11) | ((index & 1) << 21) + out.append(("%s v%d, v%d, v%d.s[%d]" % ((name,) + r + (index,)), w, r[0], r[1], r[2], "elem", 0, fused, helper)) + for name, vec, scalar in (("FMAX", 0x0E20F400, 0x1E204800), ("FMIN", 0x0EA0F400, 0x1E205800), + ("FMAXNM", 0x0E20C400, 0x1E206800), ("FMINNM", 0x0EA0C400, 0x1E207800)): + for form, base in ((" s", scalar), (" .2s", vec), (" .4s", vec | 0x40000000)): + for r in REGS: + out.append(("%s%s v%d, v%d, v%d" % ((name, form) + r), three(base, *r), r[0], r[1], r[2], "minmax", 0, 0, + "a64_vfpx_q_minmax(")) + for name, base, dbl, helper in (("FCVT d, s", 0x1E22C000, 0, "a64_vfpx_q_cvtsd("), ("FCVT s, d", 0x1E624000, 1, "a64_vfpx_q_cvtds("), + ("FCVTL .2d, .2s", 0x0E617800, 0, "a64_vfpx_q_cvtl("), ("FCVTL2 .2d, .4s", 0x4E617800, 0, "a64_vfpx_q_cvtl("), + ("FCVTN .2s, .2d", 0x0E616800, 1, "a64_vfpx_q_cvtn(")): + for rd, rn in ((0, 5), (5, 5), (17, 30)): + out.append(("%s v%d, v%d" % (name, rd, rn), base | (rn << 5) | rd, rd, rn, rn, "fcvt", dbl, 0, helper)) + return out + + +# (never wrapped: what the kit's own passes vectorize, the double-precision and pairwise forms, FCVTN2, FMULX, FRECPS) +OTHERS = (("FMUL .4s", 0x6E22DC20), ("FMUL .4s by element", 0x4F829020), ("FMLA .4s by element", 0x4F821020), + ("FADD .4s", 0x4E22D420), ("FSUB .2s", 0x0EA2D420), ("FMUL .2d", 0x6E62DC20), ("FMLA .2d", 0x4E62CC20), + ("FDIV .2d", 0x6E62FC20), ("FMAX .2d", 0x4E62F420), ("FMAX d", 0x1E624820), ("FMAXP .4s", 0x6E22F420), + ("FMAXNMP .4s", 0x6E22C420), ("FCVTN2", 0x4E616820), ("FMULX .2s", 0x0E22DC20), ("FRECPS .2s", 0x0E22FC20), + ("FMUL s by element", 0x5F829020), ("FCVT half to single", 0x1EE24020), ("FCVTL half", 0x0E217820), ("FMUL s", 0x1E220820)) + + +def with_switch(value, fn): + old = os.environ.get(SWITCH) + if value is None: + os.environ.pop(SWITCH, None) + else: + os.environ[SWITCH] = value + try: + return fn() + finally: + if old is None: + os.environ.pop(SWITCH, None) + else: + os.environ[SWITCH] = old + + +def check_rule(vecfp): + for value in (None, "", "0", " 0 "): + if with_switch(value, vecfp.groups): + raise SystemExit("vecfp: %s=%r turns groups on" % (SWITCH, value)) + if with_switch("1", vecfp.groups) != frozenset(vecfp.GROUPS) or set(vecfp.GROUPS) != {"elem", "vec", "minmax", "fcvt"}: + raise SystemExit("vecfp: %s=1 is not the four groups" % SWITCH) + if with_switch("minmax, fcvt", vecfp.groups) != frozenset(("minmax", "fcvt")): + raise SystemExit("vecfp: a list of groups is not taken as written") + try: + with_switch("all", vecfp.groups) + except SystemExit: + pass + else: + raise SystemExit("vecfp: an unknown group does not stop the lift") + every = frozenset(vecfp.GROUPS) + for c in cases(): + call = vecfp.fast_call(c[1], every) + if not call or not call.startswith(c[8]): + raise SystemExit("vecfp: %s (%08x) does not get %s: %r" % (c[0], c[1], c[8], call)) + for group in vecfp.GROUPS: # (a word belongs to one group, and is wrapped only when that one is on) + if bool(vecfp.fast_call(c[1], frozenset((group,)))) != (group == c[5]): + raise SystemExit("vecfp: %s (%08x) is not in the group %s alone" % (c[0], c[1], c[5])) + if vecfp.fast_call(c[1], frozenset()): + raise SystemExit("vecfp: %s is wrapped with no group on" % c[0]) + for name, w in OTHERS: + if vecfp.fast_call(w, every): + raise SystemExit("vecfp: %s (%08x) is wrapped: it is not one of the forms" % (name, w)) + + +def lifted(a64emit, a64_lift, used): + """vecfp_lifted.h: each guest word lifted with A64_VECFP=1, as the kit writes it, and with the exact body inline""" + all_cases = cases() + words = [(0x1000 + 4 * i, c[1]) for i, c in enumerate(all_cases)] + others = [(0x8000 + 4 * i, w) for i, (_, w) in enumerate(OTHERS)] + + def lift(items, all_translate=True): + texts, flags = a64emit.translate(items) + if all_translate and any(flags): + raise SystemExit("vecfp: a64emit did not translate the guest words: %s" % flags) + return texts + raw_now, raw_old = with_switch("1", lambda: lift(words)), with_switch(None, lambda: lift(words)) + if with_switch("1", lambda: lift(others, False)) != with_switch(None, lambda: lift(others, False)): # (FMULX: no translation) + raise SystemExit("vecfp: a word that is not one of the forms changes with the switch") + a64emit.lib().a64_config(1, 0) # (the fast path off: the exact body inline, the reference) + try: + exact = with_switch(None, lambda: lift(words)) + finally: + a64emit.lib().a64_config(1, 1) + out = ["/* Generated by tests/test_vecfp.py: each guest word lifted with A64_VECFP=1, as the kit writes it, and exact. */"] + for i, (rn, ro, ex, c) in enumerate(zip(raw_now, raw_old, exact, all_cases)): + m = re.fullmatch(r"if \(!(a64_vfpx_q_\w+\([^{}]*?\))\) \{ (.*) \}\n", rn, re.S) + if not m or not m.group(1).startswith(c[8]) or m.group(2) != ro.strip(): + raise SystemExit("vecfp: %s is not its helper's test around the kit's text: %s" % (c[0], rn[:200])) + if "a64_vfpx_q_" in ro or "a64_vfpx_q_" in ex: + raise SystemExit("vecfp: %s has the helper without the switch" % c[0]) + now, old = a64_lift.op_text(rn, used), a64_lift.op_text(ro, used) + for text in (rn, now): + if any(line.startswith("}") for line in text.split("\n")): + raise SystemExit("vecfp: %s: a line of the wrapped text starts with }: %s" % (c[0], text[-200:])) + if not now.startswith("if (!%s) { " % m.group(1)) or "recomp_fpx_" in ex: + raise SystemExit("vecfp: %s: the helper's test does not survive the kit's passes, or the exact form has a fast path" % c[0]) + # (the kit's mark grouping: the helper is a plain call, as its own vector helpers are) + if a64_lift.mg_ends_run(" if (!%s) { c->x[0]=c->x[1]; }" % m.group(1)): + raise SystemExit("vecfp: mark_groups ends a run of stores at %s" % m.group(1)) + out.append("static void g_now%d(Ctx *__restrict c) { %s }" % (i, now)) + out.append("static void g_old%d(Ctx *__restrict c) { %s }" % (i, old)) + out.append("static void g_exact%d(Ctx *__restrict c) { %s }" % (i, ex)) + out.append("static const struct { const char *name; int rd, rn, rm, dbl; void (*now)(Ctx *), (*old)(Ctx *), (*exact)(Ctx *); } CASES[] = {%s};" + % ", ".join('{"%s", %d, %d, %d, %d, g_now%d, g_old%d, g_exact%d}' % (c[0], c[2], c[3], c[4], c[6], i, i, i) + for i, c in enumerate(all_cases))) + return "\n".join(out) + "\n", all_cases + + +def generate(cxx, tmp, flags): + """a64_fpx.h (with the helpers), the kit's runtime header and C, and vecfp_lifted.h in tmp. Returns (cases, the C file)""" + dll = os.path.join(tmp, "a64emit.dll" if os.name == "nt" else "liba64emit.so") + fs.build([cxx, "-std=c++20", "-O0", "-shared", "-Wno-everything", "-o", dll, os.path.join(fs.KIT, "lift", "a64emit.cpp")] + + flags + ([] if os.name == "nt" else ["-fPIC"]), "a64emit") + import kit # noqa: F401 (a64-kit on the path) + import build_emit + build_emit.build = lambda force=False: dll # (not a64emit.dll beside the source) + import a64_lift + import a64_rtgen + import a64emit + import insn + import vecfp + check_rule(vecfp) + insn.install() + kit_header = a64_lift.fpx_header(set()) + vecfp.install() # (the wrap under test, around insn's translate and the kit's fpx_header) + used = set() + try: + a64_rtgen.write(tmp) + text, all_cases = lifted(a64emit, a64_lift, used) + finally: + if a64emit._lib is not None: # (unloaded, so the folder can go) + import _ctypes + (_ctypes.FreeLibrary if os.name == "nt" else _ctypes.dlclose)(a64emit._lib._handle) + a64emit._lib = None + if a64_lift.fpx_header(set()) != kit_header + vecfp.HEADER: + raise SystemExit("vecfp: a64_fpx.h is not the kit's header with the helpers after it") + rt_c = [f for f in sorted(os.listdir(tmp)) if f.startswith("a64_") and f.endswith(".c")] # (what a64_rtgen wrote) + if len(rt_c) != 1: + raise SystemExit("vecfp: a64_rtgen wrote %s, not one C file" % rt_c) + with open(os.path.join(tmp, "vecfp_lifted.h"), "w", newline="\n") as f: + f.write(text) + with open(os.path.join(tmp, "a64_fpx.h"), "w", newline="\n") as f: + f.write(a64_lift.fpx_header(used)) + return all_cases, os.path.join(tmp, rt_c[0]) + + +def check_sources(): + with open(os.path.join(ROOT, "nx2", "lift.py"), encoding="utf-8") as f: + lift = f.read() + at = [lift.find(p) for p in ("import vecfp", "insn.install()", "vecfp.install()", "def main():")] + if min(at) < 0 or not at[1] < at[2] < at[3]: + raise SystemExit("vecfp: nx2/lift.py does not install nx2/vecfp.py at module level after insn's") + with open(os.path.join(ROOT, "nx2", "make_game.py"), encoding="utf-8") as f: + make_game = f.read() + if 'if k.startswith(("A64_", "OOT_", "RX_"))' not in make_game or not SWITCH.startswith("A64_"): + raise SystemExit("vecfp: nx2/make_game.py's lift_key no longer takes the A64_ variables: %s is not in the key" % SWITCH) + not_keyed = re.search(r"^LIFT_KEY_NOT = \((.*?)\) #", make_game, re.S | re.M) + if not not_keyed or '"vecfp.py"' in not_keyed.group(1): + raise SystemExit("vecfp: nx2/vecfp.py must stay a file of the lift key") + with open(os.path.join(ROOT, "docs", "development.md"), encoding="utf-8") as f: + if "| `A64_VECFP=1` |" not in f.read(): + raise SystemExit("vecfp: docs/development.md: A64_VECFP is not in the switches table") + + +def main(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--cc") + p.add_argument("--cxx") + p.add_argument("--random", type=int, default=20000) + p.add_argument("--verbose", action="store_true", help="print how many states each word's helper did") + a = p.parse_args() + check_sources() + if platform.machine().lower() not in ("x86_64", "amd64"): + fs.skip("not an x86-64 host: %s" % platform.machine()) + cc = fs.compiler(a.cc, "CC", ("clang", "gcc", "cc"), "clang") + cxx = fs.compiler(a.cxx, "CXX", ("clang++", "g++", "c++"), "clang++") + if not cc or not cxx: + fs.skip("no C compiler" if not cc else "no C++ compiler") + t0 = time.time() + failed, checks = [], 0 + with tempfile.TemporaryDirectory(prefix="triforce-vecfp-") as tmp: + all_cases, rt_c = generate(cxx, tmp, fs.target_flags(cxx, tmp)) + for march in MARCHES: + exe = os.path.join(tmp, "vecfp_test_%s.exe" % march) + fs.build([cc] + fs.target_flags(cc, tmp) + ["-march=" + march] + fs.FLAGS + + ["-D_CRT_SECURE_NO_WARNINGS", "-Wno-everything", "-I" + os.path.join(fs.KIT, "rt"), "-I" + tmp, + os.path.join(ROOT, "tests", "vecfp_test.c"), rt_c, "-o", exe], "the driver at " + march) + r = fs.run([exe, str(a.random), str(SEED)], timeout=3600) + lines = r.stdout.splitlines() + total = [l for l in lines if l.startswith("total ")] + print("%-10s %s" % (march, "\n ".join(l for l in lines if a.verbose or not l.startswith("case ")))) + if r.returncode == 3: + print("vecfp: %s skipped (this CPU cannot run it)" % march) + continue + if r.returncode or len(total) != 1: + failed.append("%s (exit %d)" % (march, r.returncode)) + continue + t = {k: int(v) for k, v in (kv.split("=") for kv in total[0].split()[1:])} + checks += t["checks"] + if march == "x86-64-v3" and not t["fma"] or march in ("x86-64-v2", "x86-64") and t["fma"]: + failed.append("%s: built with%s FMA3" % (march, "" if t["fma"] else "out")) + done = {int(m.group(1)): int(m.group(2)) for m in (re.match(r"case (\d+) done (\d+) of", l) for l in lines) if m} + for i, c in enumerate(all_cases): + if c[7] and not t["fma"]: # (a fused op without FMA3: never) + if done.get(i): + failed.append("%s: %s was done %d times without FMA3" % (march, c[0], done[i])) + elif not 0 < done.get(i, 0) < a.random: # (done in some states and declined in others) + failed.append("%s: %s: its helper did %d of %d states" % (march, c[0], done.get(i, 0), a.random)) + if failed: + raise SystemExit("FAIL: vecfp: %s" % "; ".join(failed[:12])) + print("PASS: %d guest words lifted with A64_VECFP=1 leave the same registers, flags and FPSR as the kit's text and " + "the exact bodies: %d states, each three ways (%.0f s)" % (len(all_cases), checks, time.time() - t0)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_video_acquire_wait.py b/tests/test_video_acquire_wait.py new file mode 100644 index 0000000..18d878c --- /dev/null +++ b/tests/test_video_acquire_wait.py @@ -0,0 +1,144 @@ +"""A movie's Acquire waits for a picture up to 100 ms by the clock, not at all after a Stop or Flush with nothing submitted +since, and decodes without the thread's virtual core (host/nn_video.c dec_acquire). + +The wait was 50 sleeps of 2 ms, each 4-5 ms long on Windows: ~250 ms after a movie's Stop, a frame that long at every load +from the title, while the decode held the game thread's core. host/nn_video.c's decoder record, Submit, Stop/Flush and +Acquire are cut out unchanged and run by tests/video_acquire_test.c over a clock that moves 5 ms at each sleep, a decoder +whose one picture comes out at a set time, and a watched virtual core. Checked: the wait ends at 100 ms; none after a Stop; +a picture still due is waited for, after a Stop and a new submission too; the decode runs without the core, which is +taken back at every exit and never under the decoder's lock; HOS_VIDEO_ACQUIRE_TRIES=1 and HOS_VIDEO_KEEP_CORE=1 as +before. + + python -I -B tests/test_video_acquire_wait.py [--cc gcc] [--source host/nn_video.c] +""" +import argparse +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile + +RUN_ALL_ARGS = ["--cc", "gcc"] + +ROOT = Path(__file__).resolve().parents[1] +RESULT = re.compile(r"^RESULT (\w+) elapsed_ms=(\d+) ret=(\S+) index=(-?\d+) decodes=(\d+) decoded_with_core=(\d+) " + r"back_under_lock=(\d+) slept_under_lock=(\d+) core=(\d+) locks=(\d+) delivered=(\d+)$", re.M) + + +def fail(message): + raise SystemExit("FAIL: " + message) + + +def block(source, marker): + start = source.index(marker) + end = source.index("{", start) + 1 + depth = 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[start:end] + + +def typedef(source, first, last): + start = source.rindex("typedef struct", 0, source.index(first)) + return source[start:source.index(last, start) + len(last)] + + +def cut(source): + types = "\n".join([ + typedef(source, "int w, h, format, count;", "} Pool;"), + typedef(source, "uint8_t *rgba, *nv12; int w, h; int64_t pts; int last; }", "} Picture;"), + typedef(source, "AVPacket *pkt; struct Pending *next; }", "} Pending;"), + typedef(source, "AVCodecContext *ctx;", "} Dec;"), + ]) + calls = "\n".join(block(source, marker) for marker in ( + "static void dec_submit(Ctx *c) {", "static void dec_flush(Ctx *c, const char *what) {", + "static void dec_acquire(Ctx *c) {")) + return types, calls + + +def run_case(binary, case, extra): + env = {k: v for k, v in os.environ.items() if not k.startswith("HOS_VIDEO")} + env.update(extra) + done = subprocess.run([str(binary), case], env=env, capture_output=True, text=True, timeout=60) + m = RESULT.search(done.stdout) + if done.returncode or not m or m.group(1) != case: + print(done.stdout + done.stderr) + fail("%s: the fixture failed (exit %d)" % (case, done.returncode)) + keys = ("elapsed", "ret", "index", "decodes", "with_core", "back_under_lock", "slept_under_lock", "core", "locks", + "delivered") + values = dict(zip(keys, m.groups()[1:])) + return {k: int(v, 0) for k, v in values.items()} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cc", default="gcc") + parser.add_argument("--source", type=Path, default=ROOT / "host/nn_video.c") + args = parser.parse_args() + compiler = shutil.which(args.cc) + if not compiler: + raise SystemExit("C compiler not found: " + args.cc) + try: + types, calls = cut(args.source.read_text(encoding="utf-8")) + except ValueError as e: + fail("a text this test reads moved (%s)" % e) + problems = [] + with tempfile.TemporaryDirectory(prefix="video-acquire-") as temporary: + folder = Path(temporary) + (folder / "video_acquire_types.inc").write_text(types, encoding="utf-8") + (folder / "video_acquire_block.inc").write_text(calls, encoding="utf-8") + binary = folder / "fixture.exe" + built = subprocess.run([compiler, "-std=gnu17", "-O1", "-Wall", "-Wno-unused-function", "-I" + str(folder), + str(ROOT / "tests/video_acquire_test.c"), "-o", str(binary)], + capture_output=True, text=True, timeout=120) + if built.returncode: + print(built.stdout + built.stderr) + fail("the fixture did not build") + + def check(case, extra, want, label): + r = run_case(binary, case, extra) + for what, ok in want(r): + if not ok: + problems.append("%s (%s): %s; %s" % (case, label, what, r)) + return r + + def common(r): + return [("the core is held again at the end", r["core"] == 1), ("no lock left held", r["locks"] == 0), + ("the core never taken back under the lock", r["back_under_lock"] == 0), + ("no sleep under the lock", r["slept_under_lock"] == 0)] + + default = {} + check("empty", default, lambda r: common(r) + [ + ("the wait ends at 100 ms (was ~250)", 100 <= r["elapsed"] < 110), + ("a buffer as it was, not refused", r["ret"] == 0 and r["index"] == 0), + ("the decode ran without the core", r["with_core"] == 0)], "default") + check("stopped", default, lambda r: common(r) + [ + ("no wait after a Stop", r["elapsed"] == 0), + ("a buffer as it was, not refused", r["ret"] == 0 and r["index"] == 0)], "default") + check("resubmit", default, lambda r: common(r) + [ + ("a submission after the Stop is waited for", r["delivered"] == 1 and 30 <= r["elapsed"] < 40), + ("the decode ran without the core", r["with_core"] == 0)], "default") + check("pending", default, lambda r: common(r) + [ + ("a picture 60 ms away is waited for", r["delivered"] == 1 and 60 <= r["elapsed"] < 70), + ("the decode ran without the core", r["with_core"] == 0)], "default") + check("noslot", default, lambda r: common(r) + [ + ("no buffer: refused", r["ret"] == 0x2a1ca)], "default") + tries = {"HOS_VIDEO_ACQUIRE_TRIES": "1"} + check("empty", tries, lambda r: common(r) + [("the 50 sleeps as before", r["elapsed"] == 250)], + "HOS_VIDEO_ACQUIRE_TRIES=1") + check("stopped", tries, lambda r: common(r) + [("the 50 sleeps as before", r["elapsed"] == 250)], + "HOS_VIDEO_ACQUIRE_TRIES=1") + keep = {"HOS_VIDEO_KEEP_CORE": "1"} + check("pending", keep, lambda r: common(r) + [ + ("the core held while decoding, as before", r["with_core"] == r["decodes"] > 0), + ("the picture still handed out", r["delivered"] == 1)], "HOS_VIDEO_KEEP_CORE=1") + if problems: + fail("\n ".join([""] + problems)) + print("PASS: the movie's wait ends at 100 ms, none after a Stop, a due picture still waited for, the decode without the " + "core and the core never taken under the lock; both switches as before (8 cases)") + + +if __name__ == "__main__": + main() diff --git a/tests/test_video_av1_order.py b/tests/test_video_av1_order.py new file mode 100644 index 0000000..bc86daa --- /dev/null +++ b/tests/test_video_av1_order.py @@ -0,0 +1,228 @@ +"""AV1 movies on Windows: dav1d first where the FFmpeg has it, and one line a movie with what it took to decode +(host/video_av1.h, host/video_movie.h, host/nn_video.c dec_init_sized, decode_next, dec_flush and dec_finalize; no game, +no FFmpeg, no Media Foundation). + +(Triforce, 11-22, the performance audit's IO-32, after Source A's.) Windows decoded the game's 1080p AV1 movies with +Media Foundation first, in software on every GPU, and with dav1d only on a PC without Microsoft's AV1 extension. Nothing +timed either. Now libdav1d goes first where the FFmpeg the build links has it, Media Foundation where it has not or +where libdav1d does not open, HOS_AV1=mf brings the old order back, and each movie logs its pictures, the mean and the +worst decode time and the pictures skipped. + +On every system this compiles the two rule headers alone (the order for each HOS_AV1 with and without libdav1d; the +movie's counts and its line) and reads the source: the order is the rule's, by HOS_AV1 read once; the notice for a PC +with no AV1 decoder stays last; each submission is timed; a Stop, a Flush and a finalize end the movie's line; the +switch is in the switches table. + +On Windows it also compiles nn_video.c whole over tests/video_standins.h and opens the AV1 decoder as the game does, +for each case of HOS_AV1, an FFmpeg with and without libdav1d, a libdav1d that does not open and a Media Foundation +with and without a decoder: which decoder the movie got, what was asked for in which order, the notice once and only +with no decoder at all. Then two movies are played: the first says its line once at its Stop (not again at the Flush +after it), the second at the decoder's end, with the pictures skipped while the game took them late. + + python -I -B tests/test_video_av1_order.py [--cc clang] +""" +import argparse +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import video_fixture as vf # noqa: E402 + +RUN_ALL_ARGS = ["--cc", "clang"] + +RULES = r""" +#include +#include +#include "video_av1.h" +#include "video_movie.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +int main(void) { + /* libdav1d in the FFmpeg: first, unless HOS_AV1=mf */ + CHECK(!video_av1_mf_first(NULL, 1)); + CHECK(!video_av1_mf_first("dav1d", 1)); /* (the old way to ask for it: the default now) */ + CHECK(video_av1_mf_first("mf", 1)); + CHECK(!video_av1_mf_first("", 1) && !video_av1_mf_first("MF", 1) && !video_av1_mf_first("mfx", 1)); + /* none in it: Media Foundation, whatever is asked */ + CHECK(video_av1_mf_first(NULL, 0) && video_av1_mf_first("dav1d", 0) && video_av1_mf_first("mf", 0)); + + VideoMovie m; + char line[200] = "untouched"; + memset(&m, 0, sizeof m); + CHECK(!video_movie_line(&m, "libdav1d", 0, line, sizeof line) && !strcmp(line, "untouched")); /* (no picture: no line) */ + video_movie_decoded(&m, 7000000), m.pictures++; + video_movie_decoded(&m, 2000000), m.pictures++; + video_movie_decoded(&m, 0); /* (the stream's end: a submission with no picture) */ + video_movie_decoded(&m, -5); /* (a clock that went back: nothing taken off) */ + video_movie_decoded(&m, 3000000), m.pictures++; + m.skipped = 4; + CHECK(m.decode_ns == 12000000 && m.worst_ns == 7000000); + CHECK(video_movie_line(&m, "libdav1d", 0, line, sizeof line)); /* (the gap between pictures not known: left out) */ + CHECK(!strcmp(line, "a movie of 3 pictures on libdav1d: 4.00 ms to decode each (worst 7.00); 4 skipped to catch up")); + /* (THREAD-82) with the gap: a decoder faster than the movie, then one slower than it plays */ + CHECK(video_movie_line(&m, "libdav1d", 16683333, line, sizeof line)); + CHECK(!strcmp(line, "a movie of 3 pictures on libdav1d: 4.00 ms to decode each (worst 7.00), one shown every 16.68 ms; " + "4 skipped to catch up")); + CHECK(video_movie_line(&m, "libdav1d", 3900000, line, sizeof line)); + CHECK(strstr(line, "(worst 7.00), one shown every 3.90 ms (slower than it plays); 4 skipped to catch up") != NULL); + char small[16]; + CHECK(video_movie_line(&m, "Media Foundation", 16683333, small, sizeof small) && strlen(small) == 15); /* (cut, never past) */ + return fails ? 1 : 0; +} +""" + +MOVIES = r""" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +static int env_int(const char *name, int otherwise) { return getenv(name) ? atoi(getenv(name)) : otherwise; } +static const char *decoder(void) { + Dec *d = si_dec(); + return !d ? "no Dec" : d->ctx ? d->ctx->codec->name : d->mf ? "Media Foundation" : "none"; +} +int main(void) { + const int64_t gap = 16683333; /* (a picture every 16.7 ms) */ + si.have_dav1d = env_int("SI_DAV1D", 1), si.mf_ok = env_int("SI_MF", 1), si.open_fails = env_int("SI_OPEN_FAILS", 0); + si.decode_ns = 2000000, si.first_decode_ns = 7000000; + si_open(64, 36, 2, 0); + printf("decoder %s\norder %s\nthreads %d\n", decoder(), si.order, si_threads); + si_finalize(); + si.order[0] = 0; + si_open(64, 36, 2, 0); /* (a second decoder: the same choice, and the notice not a second time) */ + printf("again %s\norder-again %s\nthreads-again %d\nboxes %d\n", decoder(), si.order, si_threads, si_boxes); + Dec *d = si_dec(); + if (d->ctx || d->mf) { + /* the first movie: 12 pictures, each taken in time; a Stop, then a Flush */ + for (int i = 0; i < 12; i++) CHECK(si_submit(i * gap, i == 11) == 0); + for (int i = 0; i < 12; i++) { + const int b = si_acquire(); + CHECK(b >= 0 && si_pts(b, NULL) == i * gap); + si_release(b); + si_now += gap - 2000000; + } + printf("lines-playing %d\n", si_logged("video: a movie of ")); + si_stop(); + printf("lines-stop %d\n", si_logged("video: a movie of ")); + si_flush(); + printf("lines-flush %d\n", si_logged("video: a movie of ")); + /* the second: 40 pictures, the game taking one every three pictures' time; then the decoder's end */ + int next = 0; + for (int i = 0; i < 9; i++) { + CHECK(si_feed(&next, 40, gap) >= 0); /* (as many as the decoder takes: IO-34's input limit) */ + const int b = si_acquire(); + CHECK(b >= 0); + si_release(b); + si_now += 3 * gap; + } + printf("second %lld pictures, %lld skipped\n", (long long)d->movie.pictures, (long long)d->movie.skipped); + si_finalize(); + printf("lines-end %d\n", si_logged("video: a movie of ")); + } + fputs(si_log, stdout); + return fails ? 1 : 0; +} +""" + +# (HOS_AV1, libdav1d in the FFmpeg, Media Foundation has a decoder, libdav1d fails to open) -> the decoder, what was +# asked in order (d libdav1d by name, o its open, M Media Foundation quietly, T Media Foundation telling why), notices +CASES = [ + ((None, 1, 1, 0), "libdav1d", "do", 0), # the default: dav1d, Media Foundation never asked + ((None, 1, 0, 0), "libdav1d", "do", 0), # ... with or without the AV1 extension + (("dav1d", 1, 1, 0), "libdav1d", "do", 0), # the old switch for it: the same + (("mf", 1, 1, 0), "Media Foundation", "dM", 0), # HOS_AV1=mf: as before + (("mf", 1, 0, 0), "libdav1d", "dMo", 0), # ... and dav1d where the extension is missing, as before + ((None, 0, 1, 0), "Media Foundation", "dM", 0), # an FFmpeg without libdav1d: Media Foundation + ((None, 0, 0, 0), "none", "dMT", 1), # ... and neither: the notice, once + ((None, 1, 1, 1), "Media Foundation", "doT", 0), # a libdav1d that does not open: Media Foundation after it + ((None, 1, 0, 1), "none", "doT", 1), # ... and the notice when that has none either +] + + +def value(out, name): + m = re.search(r"^%s (.*)$" % re.escape(name), out, re.M) + vf.check(m is not None, "the fixture printed no %r line:\n%s" % (name, out)) + return m.group(1) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cc", default="clang") + args = parser.parse_args() + video = vf.video_source() + + vf.check('#include "video_av1.h"' in video and '#include "video_movie.h"' in video, "nn_video.c does not take the two rule headers") + first = vf.block(video, "static const char *av1_first(void) {") + vf.check('if (!read) value = getenv("HOS_AV1"), read = 1;' in first, "HOS_AV1 is not read once") + init = vf.block(video, "static void dec_init_sized(Ctx *c, enum AVCodecID id, const char *what, int w, int h) {") + by_name = 'codec = avcodec_find_decoder_by_name("libdav1d");' + rule = "if (video_av1_mf_first(av1_first(), codec != NULL)) d->mf = mf_av1_open(w, h, 0);" + told = "d->mf = mf_av1_open(w, h, 1);" + windows = init[init.index("#ifdef _WIN32"):] + vf.check(by_name in windows and rule in windows and windows.index(by_name) < windows.index(rule), + "dec_init_sized does not ask the rule, with libdav1d looked up first") + vf.check('getenv("HOS_AV1")' not in init, "dec_init_sized reads HOS_AV1 itself") + vf.check(told in init and init.index(rule) < init.index("avcodec_open2(") < init.index(told) < init.index("av1_notice"), + "Media Foundation is no longer the fallback of a libdav1d that does not open, with the notice last") + nxt = vf.block(video, "static void decode_next(Dec *d) {") + vf.check("const int64_t began = hos_now_ns();" in nxt and "const int64_t took = hos_now_ns() - began;" in nxt + and "video_movie_decoded(&d->movie, took);" in nxt + and nxt.index("began = hos_now_ns()") < nxt.index("mf_av1_decode(") < nxt.index("took = hos_now_ns() - began"), + "decode_next does not time each submission") + vf.check("d->movie.pictures++;" in vf.block(video, "static void queue_frame(Dec *d, const AVFrame *f) {"), + "queue_frame does not count the movie's pictures") + vf.check("d->movie.skipped += skipped;" in vf.block(video, "static void dec_acquire(Ctx *c) {"), + "the catch-up does not count what it skipped for the movie's line") + end = vf.block(video, "static void movie_end(Dec *d) {") + vf.check("video_movie_line(&d->movie," in end and "video_pace_spacing(&d->pace)" in end and 'hos_log("video: %s", line);' in end + and "memset(&d->movie, 0, sizeof d->movie);" in end, + "movie_end does not log the line and start the counts again") + flush = vf.block(video, "static void dec_flush(Ctx *c, const char *what) {") + vf.check("movie_end(d);" in flush and + flush.index("AcquireSRWLockExclusive(&d->lock);") < flush.index("movie_end(d);") < flush.index("pending_free(d);"), + "a Stop or a Flush does not end the movie's line under the decoder's lock") + vf.check("movie_end(d);" in vf.block(video, "static void dec_finalize(Ctx *c) {"), "the decoder's end does not end the movie's line") + dev = (vf.ROOT / "docs" / "development.md").read_text(encoding="utf-8") + vf.check("`HOS_AV1=mf`" in dev, "docs/development.md: HOS_AV1=mf is not in the switches table") + + with tempfile.TemporaryDirectory(prefix="video-av1-order-") as tmp: + src, exe = Path(tmp) / "rules.c", Path(tmp) / "rules.exe" + src.write_text(RULES, encoding="utf-8") + subprocess.run([args.cc, "-std=gnu11", "-O1", "-Wall", "-Wextra", "-Werror", "-I", str(vf.HOST), str(src), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + vf.check(r.returncode == 0, "the order rule or the movie's line:\n" + r.stdout) + if not vf.WINDOWS: + print(vf.WINDOWS_ONLY) + print("PASS: AV1 movies ask for libdav1d first unless HOS_AV1=mf, and a movie's line is written once (the rules and the source)") + return + exe = vf.build(tmp, args.cc, MOVIES) + for (hos_av1, dav1d, mf, open_fails), decoder, order, notices in CASES: + out = vf.run(exe, HOS_AV1=hos_av1, SI_DAV1D=dav1d, SI_MF=mf, SI_OPEN_FAILS=open_fails) + case = "HOS_AV1=%s, libdav1d %d, Media Foundation %d, open fails %d" % (hos_av1, dav1d, mf, open_fails) + vf.check("FAIL:" not in out, case + ":\n" + out) + vf.check(value(out, "decoder") == decoder and value(out, "again") == decoder, + "%s: the movie got %s, not %s" % (case, value(out, "decoder"), decoder)) + vf.check(value(out, "order") == order and value(out, "order-again") == order, + "%s: asked in the order %s, not %s" % (case, value(out, "order"), order)) + vf.check(value(out, "threads") == str(notices) and value(out, "threads-again") == str(notices) and value(out, "boxes") == "0", + "%s: %s notices after one decoder and %s after two, not %d once" % (case, value(out, "threads"), value(out, "threads-again"), notices)) + vf.check(out.count("video: no AV1 decoder: install") == notices, case + ": the log's line for no decoder") + if decoder == "none": + continue + vf.check("video: AV1 decoder ready" + (": libdav1d" if decoder == "libdav1d" else " on Media Foundation") in out, + case + ": the log does not name the decoder") + vf.check([value(out, "lines-" + k) for k in ("playing", "stop", "flush", "end")] == ["0", "1", "1", "2"], + case + ": a movie's line is not written once at its Stop and once at the decoder's end:\n" + out) + lines = re.findall(r"^video: a movie of (\d+) pictures on (.+?): ([\d.]+) ms to decode each \(worst ([\d.]+)\).*; (\d+) skipped to catch up(?:, \d+ submissions refused while its input was full)?$", out, re.M) + vf.check(len(lines) == 2, case + ": the two movie lines:\n" + out) + # (12 submissions: 7 ms the first, 2 ms the others, and the stream's end, which costs nothing) + vf.check(lines[0] == ("12", decoder, "2.42", "7.00", "0"), "%s: the first movie's line is %r" % (case, lines[0])) + m = re.search(r"^second (\d+) pictures, (\d+) skipped$", out, re.M) + vf.check(m and int(m.group(2)) > 0 and lines[1][0] == m.group(1) and lines[1][4] == m.group(2) and lines[1][1] == decoder, + "%s: the second movie's line %r against its counts %r" % (case, lines[1], m and m.groups())) + print("PASS: AV1 movies get libdav1d first on Windows, Media Foundation with HOS_AV1=mf or without it, and say what they took") + + +if __name__ == "__main__": + main() diff --git a/tests/test_video_catchup.py b/tests/test_video_catchup.py new file mode 100644 index 0000000..71a1060 --- /dev/null +++ b/tests/test_video_catchup.py @@ -0,0 +1,208 @@ +"""The movie decoder's catch-up is bounded by what a decode takes lately: an acquire stops decoding ahead when a decode +takes longer than the gap between two pictures, or once it has decoded for one gap (host/video_catchup.h, host/nn_video.c +decode_next, queue_frame and dec_acquire; no game, no FFmpeg). + +(Triforce, 11-22, the performance audit's THREAD-82, after Source A's.) dec_acquire skips the pictures behind the +movie's clock and decodes the next ones to skip them too, with no bound but a guard of 600 steps. With a decoder slower +than the movie each picture decoded to catch up puts the clock further ahead: an acquire decoded hundreds of pictures +with its lock held and threw them away, and the next one started further behind. HOS_VIDEO_UNBOUNDED_CATCHUP=1 keeps +that. + +On every system this compiles video_catchup.h alone (the gap from the pictures' own times, counted again from a movie +that loops; the decode time as seven parts old and one new; when the catch-up stops) and reads the source: each picture +gives its time to the gap, each decode that gave a picture its duration; the catch-up asks the rule before it decodes +ahead, with the time this acquire has spent; the switch is read once and is in the switches table. + +On Windows it also compiles nn_video.c whole over tests/video_standins.h and plays a movie of 600 pictures 16.68 ms +apart on a clock a decode moves, the game taking a picture a frame and stalling for a second after the twelfth: +- a decoder of 19.5 ms a picture: no acquire decodes more than one picture ahead, and none at all once the decode time + is known to be over the gap; the movie's line says it is slower than the movie plays. With the switch, the acquire + after the stall decodes more than 50, as before; +- a decoder of 4 ms: it catches up with the clock within 40 acquires of the stall, a few pictures an acquire; +- a first decode of 200 ms, then 2 ms: the same (one slow decode does not stop the catch-up for good); +- HOS_VIDEO_NO_CATCHUP=1: nothing skipped, as before. +The whole movie is submitted at once, as the game did before the decoder's input limit (IO-34), which the runs turn off +with HOS_VIDEO_INPUT_LIMIT=0: with it an acquire has 16 submissions at most to decode ahead, and this rule is the only +bound under test here. + + python -I -B tests/test_video_catchup.py [--cc clang] +""" +import argparse +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import video_fixture as vf # noqa: E402 + +RUN_ALL_ARGS = ["--cc", "clang"] + +RULES = r""" +#include +#include +#include "video_catchup.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +int main(void) { + const int64_t gap = 16683333, ms = 1000000; + VideoPace p; + memset(&p, 0, sizeof p); + CHECK(video_pace_spacing(&p) == 0); + video_pace_picture(&p, 5 * gap); + CHECK(video_pace_spacing(&p) == 0); /* (one picture: no gap yet) */ + for (int i = 6; i < 30; i++) video_pace_picture(&p, i * gap); + CHECK(p.n == 25 && video_pace_spacing(&p) == gap); + video_pace_picture(&p, 0); /* a loop: counted again from it */ + CHECK(p.n == 1 && video_pace_spacing(&p) == 0); + video_pace_picture(&p, 2 * gap), video_pace_picture(&p, 4 * gap); + CHECK(video_pace_spacing(&p) == 2 * gap); + video_pace_picture(&p, 4 * gap); /* (the same time twice: not a loop) */ + CHECK(p.n == 4 && video_pace_spacing(&p) == 4 * gap / 3); + + /* the decode time: seven parts old, one new. 19.5 ms decodes pass a 16.68 ms gap at the 15th, and stay */ + memset(&p, 0, sizeof p); + int over = 0; + for (int i = 1; i <= 40 && !over; i++) { + video_pace_decoded(&p, 19500000); + if (p.recent_ns > gap) over = i; + } + CHECK(over == 15); + for (int i = 0; i < 100; i++) video_pace_decoded(&p, 19500000); + CHECK(p.recent_ns > 19 * ms && p.recent_ns <= 19500000); + /* one slow decode is forgotten: 200 ms, then 2 ms ones */ + memset(&p, 0, sizeof p); + video_pace_decoded(&p, 200 * ms); + CHECK(p.recent_ns == 25 * ms); + for (int i = 0; i < 4; i++) video_pace_decoded(&p, 2 * ms); + CHECK(p.recent_ns < gap); + video_pace_decoded(&p, -7); /* (a clock that went back: as 0) */ + CHECK(p.recent_ns >= 0); + + /* video_catchup_stop(unbounded, gap, recent, spent in this acquire) */ + CHECK(!video_catchup_stop(0, gap, 4 * ms, 0)); /* a fast decoder: on */ + CHECK(!video_catchup_stop(0, gap, 4 * ms, 16 * ms)); /* ... while this acquire has decoded for less than a gap */ + CHECK(video_catchup_stop(0, gap, 4 * ms, gap)); /* ... and not past it */ + CHECK(video_catchup_stop(0, gap, 19500000, 0)); /* a decoder slower than the movie: not at all */ + CHECK(!video_catchup_stop(0, gap, gap, 0)); /* (as long as the gap exactly: on) */ + CHECK(!video_catchup_stop(0, 0, 19500000, 5000 * ms)); /* the gap not known yet: as before */ + CHECK(!video_catchup_stop(0, -gap, 19500000, 5000 * ms)); + CHECK(!video_catchup_stop(1, gap, 19500000, 5000 * ms)); /* the switch: never */ + return fails ? 1 : 0; +} +""" + +MOVIE = r""" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +int main(int argc, char **argv) { + CHECK(argc == 3); + const int64_t gap = 16683333; + si.decode_ns = atoll(argv[1]), si.first_decode_ns = atoll(argv[2]); + si_open(64, 36, 2, 0); + for (int i = 0; i < 600; i++) CHECK(si_submit(i * gap, i == 599) == 0); /* (the whole movie at once, as the game does) */ + Dec *d = si_dec(); + for (int a = 0; a < 200; a++) { + const int64_t before = si.decodes, skipped = d->movie.skipped; + const int b = si_acquire(); + CHECK(b >= 0); + int last = 0; + const int64_t pts = si_pts(b, &last); + printf("acquire %d: %lld decodes, %lld skipped, behind %lld ms\n", a, (long long)(si.decodes - before), + (long long)(d->movie.skipped - skipped), (long long)((si_now - d->clock_ns - (pts - d->clock_pts)) / 1000000)); + si_release(b); + if (last) break; + si_now += gap; /* the game's frame */ + if (a == 11) si_now += 1000000000; /* ... and one of a second */ + } + si_stop(); + fputs(si_log, stdout); + return fails ? 1 : 0; +} +""" + + +def acquires(out, case, at_least=100): + rows = [tuple(int(x) for x in m) for m in re.findall(r"^acquire (\d+): (\d+) decodes, (\d+) skipped, behind (-?\d+) ms$", out, re.M)] + vf.check("FAIL:" not in out and len(rows) >= at_least, case + ":\n" + out[-3000:]) + return rows + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cc", default="clang") + args = parser.parse_args() + video = vf.video_source() + + vf.check('#include "video_catchup.h"' in video and "VideoPace pace;" in vf.block(video, "typedef struct {\n AVCodecContext *ctx;"), + "nn_video.c does not keep the movie's pace in its decoder") + vf.check("video_pace_picture(&d->pace, f->pts);" in vf.block(video, "static void queue_frame(Dec *d, const AVFrame *f) {"), + "queue_frame does not give each picture's time to the gap") + nxt = vf.block(video, "static void decode_next(Dec *d) {") + vf.check("const int64_t pictures = d->movie.pictures;" in nxt + and "if (d->movie.pictures != pictures) video_pace_decoded(&d->pace, took);" in nxt, + "decode_next does not keep what a decode that gave a picture took") + vf.check("memset(&d->pace, 0, sizeof d->pace);" in vf.block(video, "static void movie_end(Dec *d) {"), + "a movie's end does not start the pace again") + acquire = vf.block(video, "static void dec_acquire(Ctx *c) {") + read_once = 'unbounded = getenv("HOS_VIDEO_UNBOUNDED_CATCHUP") != NULL && atoi(getenv("HOS_VIDEO_UNBOUNDED_CATCHUP")) != 0;' + stop = "if (video_catchup_stop(unbounded, video_pace_spacing(&d->pace), d->pace.recent_ns, hos_now_ns() - now)) {" + vf.check("static int off = -1, unbounded = -1;" in acquire and "if (unbounded < 0)" in acquire and read_once in acquire, + "HOS_VIDEO_UNBOUNDED_CATCHUP is not read once") + vf.check(stop in acquire, "the catch-up does not ask the rule with the time this acquire has spent") + loop = acquire[acquire.index("for (int guard = 0;"):] + loop = loop[:loop.index("if (skipped && video_log())")] + vf.check(loop.count("decode_next(d);") == 1 and loop.index("if (!d->phead) break;") < loop.index(stop) < loop.index("decode_next(d);"), + "the catch-up decodes ahead before it asks the rule") + vf.check('getenv("HOS_VIDEO_NO_CATCHUP")' in acquire and "guard < 600" in acquire, "the catch-up lost its switch or its guard") + dev = (vf.ROOT / "docs" / "development.md").read_text(encoding="utf-8") + vf.check("`HOS_VIDEO_UNBOUNDED_CATCHUP=1`" in dev, "docs/development.md: HOS_VIDEO_UNBOUNDED_CATCHUP=1 is not in the switches table") + + with tempfile.TemporaryDirectory(prefix="video-catchup-") as tmp: + src, exe = Path(tmp) / "rules.c", Path(tmp) / "rules.exe" + src.write_text(RULES, encoding="utf-8") + subprocess.run([args.cc, "-std=gnu11", "-O1", "-Wall", "-Wextra", "-Werror", "-I", str(vf.HOST), str(src), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + vf.check(r.returncode == 0, "the pace or the catch-up's rule:\n" + r.stdout) + if not vf.WINDOWS: + print(vf.WINDOWS_ONLY) + print("PASS: the movie catch-up is bounded by the recent decode time (the rule and the source)") + return + exe = vf.build(tmp, args.cc, MOVIE) + + # a decoder slower than the movie: 19.5 ms a picture against 16.68 + out = vf.run(exe, "19500000", "0", HOS_VIDEO_INPUT_LIMIT="0") + rows = acquires(out, "19.5 ms decodes") + # (one decode for the picture shown, and at most one ahead) + vf.check(max(r[1] for r in rows) <= 2, "19.5 ms decodes: an acquire decoded %d pictures" % max(r[1] for r in rows)) + vf.check(all(r[1] == 1 and r[2] == 0 for r in rows[40:]), + "19.5 ms decodes: the catch-up still decodes ahead once a decode is known to take longer than the gap") + vf.check(re.search(r"^video: a movie of \d+ pictures on libdav1d: 19\.50 ms to decode each \(worst 19\.50\), one shown every " + r"16\.68 ms \(slower than it plays\); \d+ skipped to catch up$", out, re.M) is not None, + "19.5 ms decodes: the movie's line does not say the decoder is slower than the movie plays:\n" + out[-600:]) + out = vf.run(exe, "19500000", "0", HOS_VIDEO_UNBOUNDED_CATCHUP="1", HOS_VIDEO_INPUT_LIMIT="0") + rows = acquires(out, "19.5 ms decodes, unbounded", 13) # (it skips to the movie's end long before 100 acquires) + vf.check(rows[12][1] > 50 and rows[12][2] > 50, + "HOS_VIDEO_UNBOUNDED_CATCHUP=1: the acquire after the stall decoded %d pictures, not all it was behind by" % rows[12][1]) + out = vf.run(exe, "19500000", "0", HOS_VIDEO_UNBOUNDED_CATCHUP="0", HOS_VIDEO_INPUT_LIMIT="0") + vf.check(max(r[1] for r in acquires(out, "19.5 ms decodes, =0")) <= 2, "HOS_VIDEO_UNBOUNDED_CATCHUP=0 is taken as the switch") + + # decoders faster than the movie catch up: a few pictures an acquire, until the picture shown is in time again + for case, first in (("4 ms decodes", "0"), ("a first decode of 200 ms, then 2 ms", "200000000")): + out = vf.run(exe, "4000000" if first == "0" else "2000000", first, HOS_VIDEO_INPUT_LIMIT="0") + rows = acquires(out, case) + vf.check(rows[12][3] > 700, "%s: the stall did not leave the movie most of a second behind (%d ms)" % (case, rows[12][3])) + vf.check(max(r[3] for r in rows[52:]) <= 45, "%s: still %d ms behind 40 acquires after the stall" % (case, max(r[3] for r in rows[52:]))) + vf.check(sum(r[2] for r in rows[12:52]) >= 50, "%s: the catch-up skipped %d pictures" % (case, sum(r[2] for r in rows[12:52]))) + vf.check(2 <= max(r[1] for r in rows) <= 12, "%s: an acquire decoded %d pictures" % (case, max(r[1] for r in rows))) + vf.check("(slower than it plays)" not in out and "one shown every 16.68 ms; " in out, case + ": the movie's line:\n" + out[-600:]) + + out = vf.run(exe, "4000000", "0", HOS_VIDEO_NO_CATCHUP="1", HOS_VIDEO_INPUT_LIMIT="0") + rows = acquires(out, "no catch-up") + vf.check(all(r[1] == 1 and r[2] == 0 for r in rows), "HOS_VIDEO_NO_CATCHUP=1: pictures were decoded ahead or skipped") + print("PASS: the movie catch-up stops when a decode takes longer than a picture's gap, or after one gap of decoding") + + +if __name__ == "__main__": + main() diff --git a/tests/test_video_input_limit.py b/tests/test_video_input_limit.py new file mode 100644 index 0000000..a463a51 --- /dev/null +++ b/tests/test_video_input_limit.py @@ -0,0 +1,251 @@ +"""The movie decoder refuses a submission while 16 wait, and its ready event is clear until there is room +(host/video_input.h, host/nn_video.c dec_submit, input_room and ready_event, host/nn_os_sync.c; no game, no FFmpeg). + +(Triforce, 11-22, the performance audit's IO-34, after Source A's.) The game submits a whole movie at once, on the +thread that also takes the pictures, and every submission was taken: the pictures that came due while it submitted +were skipped, a stutter at each movie's start. Now a submission is refused at once with the console's result for a +full decoder (0x1c266f) while 16 wait, not decoded yet or decoded and not taken; the game submits it again later. The +stream's last submission is never refused. Each decoder has its own "ready for a submission" event, clear from a +refusal until there is room. HOS_VIDEO_INPUT_LIMIT= sets the limit; 0 takes every submission, as before. + +A decoder left full with its event clear, or a refused last submission, would hang the movie on a black screen, so on +every system this compiles video_input.h alone and reads the source: the refusal comes before the submission is +counted, under the decoder's lock, with the event cleared there; every call that ends with the count possibly lower +(submit, Stop and Flush, the three exits of the acquire) sets the event again before it lets the lock go; a finalize +leaves it set; the event's clear and its set under a lock take no virtual-core checkpoint; the switch is read once and +is in the switches table. + +On Windows it also compiles nn_video.c whole over tests/video_standins.h and, from one thread as the game does: +- submits to a fresh decoder: 16 are taken and the 17th is refused with 0x1c266f, its id not written and the count of + submissions not moved; the event, set until then, is clear; an acquire sets it and one more submission fits; a Stop, + a Flush and a finalize each set it again after a refusal; a last submission is taken while full; +- replays a movie of 600 pictures, submitting until refused before each picture it takes: never more than 16 wait before the last, + every picture comes out in order with none skipped, the last one marked last, the event is set whenever there is + room, and the movie's line counts the refusals; +- with HOS_VIDEO_INPUT_LIMIT=0 all 600 are taken at once and the event is never cleared; with =4 the 5th is refused; +- a decoder with no AV1 decoder behind it refuses nothing. + + python -I -B tests/test_video_input_limit.py [--cc clang] +""" +import argparse +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import video_fixture as vf # noqa: E402 + +RUN_ALL_ARGS = ["--cc", "clang"] + +RULES = r""" +#include +#include "video_input.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +int main(void) { + CHECK(VIDEO_INPUT_LIMIT == 16 && VIDEO_RESULT_INPUT_FULL == 0x1c266f); + CHECK(video_input_limit(NULL) == 16 && video_input_limit("") == 16); + CHECK(video_input_limit("0") == 0 && video_input_limit("4") == 4 && video_input_limit("64") == 64); + CHECK(video_input_limit("-3") == 0 && video_input_limit("none") == 0); /* (not a positive number: no limit) */ + /* video_input_full(limit, pending, queued) */ + CHECK(!video_input_full(16, 15, 0) && video_input_full(16, 16, 0) && video_input_full(16, 8, 8) && video_input_full(16, 9, 8)); + CHECK(!video_input_full(16, 7, 8) && !video_input_full(16, 0, 0)); + CHECK(!video_input_full(0, 1000, 8)); /* no limit: never full */ + /* video_input_refuse(limit, pending, queued, last) */ + CHECK(video_input_refuse(16, 16, 0, 0) && video_input_refuse(16, 10, 6, 0)); + CHECK(!video_input_refuse(16, 16, 0, 1) && !video_input_refuse(16, 400, 8, 1)); /* the stream's last: never */ + CHECK(!video_input_refuse(16, 15, 0, 0) && !video_input_refuse(0, 1000, 8, 0)); + return fails ? 1 : 0; +} +""" + +MOVIE = r""" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +static int env_int(const char *name, int otherwise) { return getenv(name) ? atoi(getenv(name)) : otherwise; } +static int waiting(void) { return si_dec()->npending + si_dec()->qn; } +/* submissions until one is refused: how many were taken (never more than `most`) */ +static int fill(int64_t *pts, int most) { + int taken = 0; + while (taken < most && si_submit(*pts, 0) == 0) taken++, *pts += 16683333; + return taken; +} +int main(void) { + const int64_t gap = 16683333; + const int limit = env_int("SI_LIMIT", 16); /* (what the run's HOS_VIDEO_INPUT_LIMIT means: 0 none) */ + si.have_dav1d = env_int("SI_DAV1D", 1), si.mf_ok = env_int("SI_MF", 1); + si.decode_ns = 2000000; + si_open(64, 36, 2, 0); + Dec *d = si_dec(); + uint64_t ev = si_ready_event(); + CHECK(ev != 0 && ev == d->ready && si_event_set(ev)); + if (!d->ctx && !d->mf) { /* no decoder behind it: every submission answered 0, as before, and nothing cleared */ + for (int i = 0; i < 40; i++) CHECK(si_submit(i * gap, 0) == 0); + CHECK(si_event_set(ev) && si_event_clears(ev) == 0); + printf("no decoder: 40 taken\n"); + return fails ? 1 : 0; + } + int64_t pts = 0; + if (limit) { + /* the limit's worth is taken, the next refused: no id, no submission counted, the event clear */ + CHECK(fill(&pts, limit) == limit && waiting() == limit && si_event_set(ev)); + const LONG64 counted = submissions; + G64(SI_OUT) = 0x1234; + CHECK(si_submit(pts, 0) == SI_FULL); + CHECK(G64(SI_OUT) == 0x1234 && submissions == counted && waiting() == limit && d->submits == limit); + CHECK(!si_event_set(ev) && d->ready_clear); + CHECK(si_submit(pts, 0) == SI_FULL && si_event_clears(ev) == 1); /* (again: refused, cleared once) */ + CHECK(si_ready_event() == ev); + /* an acquire takes a picture: room, the event set, one more fits and the next is refused again */ + int b = si_acquire(); + CHECK(b >= 0 && si_pts(b, NULL) == 0); + si_release(b); + CHECK(waiting() == limit - 1 && si_event_set(ev) && !d->ready_clear); + CHECK(fill(&pts, 100) == 1 && !si_event_set(ev)); + /* a Stop empties it: set; and a Flush after another refusal */ + si_stop(); + CHECK(waiting() == 0 && si_event_set(ev)); + CHECK(fill(&pts, 100) == limit && !si_event_set(ev)); + si_flush(); + CHECK(waiting() == 0 && si_event_set(ev)); + /* the stream's last submission is taken while full, and the refusals after it stay refusals */ + CHECK(fill(&pts, 100) == limit && !si_event_set(ev)); + CHECK(si_submit(pts, 1) == 0 && d->npending == limit + 2); /* (it and the end's mark) */ + CHECK(si_submit(pts + gap, 0) == SI_FULL); + /* a finalize leaves the event set, for a thread that still has it */ + si_finalize(); + CHECK(si_event_set(ev)); + si_open(64, 36, 2, 0); + d = si_dec(); + CHECK(si_ready_event() != ev); /* (a new decoder: an event of its own) */ + ev = si_ready_event(); + CHECK(si_event_set(ev)); + printf("first part: the refusals and the event\n"); + } + /* a movie of 600 pictures: the game submits until it is refused, then takes a picture, every frame */ + int next = 0, most = 0, refusals_seen = 0, taken_at_once = 0, acquired = 0, last = 0; + int64_t expect = 0; + for (int frame = 0; frame < 2000 && !last; frame++) { + const int taken = si_feed(&next, 600, gap); + CHECK(taken >= 0); + if (frame == 0) taken_at_once = taken; + if (waiting() > most) most = waiting(); + if (next < 600) { /* (refused: full, the event clear) */ + refusals_seen++; + CHECK(waiting() >= limit && !si_event_set(ev)); + } else { + CHECK(frame == 0 || !limit || si_event_set(ev) || waiting() >= limit); + } + const int b = si_acquire(); + CHECK(b >= 0); + CHECK(si_pts(b, &last) == expect); /* (in order, none skipped: the game keeps up here) */ + expect += gap, acquired++; + si_release(b); + CHECK(waiting() < limit || !limit ? si_event_set(ev) : 1); /* (room: never left clear) */ + si_now += gap - si.decode_ns; + } + printf("movie: %d taken at once, %d acquired, last %d, most waiting %d, frames refused %d, event cleared %d times\n", + taken_at_once, acquired, last, most, refusals_seen, si_event_clears(ev)); + si_stop(); + CHECK(si_event_set(ev)); + fputs(si_log, stdout); + return fails ? 1 : 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cc", default="clang") + args = parser.parse_args() + video = vf.video_source() + sync = (vf.HOST / "nn_os_sync.c").read_text(encoding="utf-8") + + vf.check("void hos_system_event_clear(uint64_t key) { event_clear(obj_nockpt(key, K_EVENT)); }" in sync + and "void hos_system_event_signal_held(uint64_t key) { event_signal(obj_nockpt(key, K_EVENT)); }" in sync, + "nn_os_sync.c: the event's clear and its set under a lock are missing, or take the virtual-core checkpoint") + vf.check('#include "video_input.h"' in video, "nn_video.c does not take video_input.h") + limit = vf.block(video, "static int input_limit(void) {") + vf.check('if (limit < 0) limit = video_input_limit(getenv("HOS_VIDEO_INPUT_LIMIT"));' in limit, "HOS_VIDEO_INPUT_LIMIT is not read once") + room = vf.block(video, "static void input_room(Dec *d) {") + vf.check("if (d->ready_clear && !input_full(d)) {" in room and "d->ready_clear = 0;" in room + and "hos_system_event_signal_held(d->ready);" in room and "hos_system_event_signal(" not in room, + "input_room does not set a cleared event once there is room, without a checkpoint") + vf.check("d->npending++;" in vf.block(video, "static void pending_push(Dec *d, AVPacket *pkt) {") + and "d->npending--;" in vf.block(video, "static void decode_next(Dec *d) {") + and "d->npending = 0;" in vf.block(video, "static void pending_free(Dec *d) {"), + "the count of pending submissions is not kept where the list changes") + + submit = vf.block(video, "static void dec_submit(Ctx *c) {") + refuse = "const int refuse = video_input_refuse(input_limit(), d->npending, d->qn, (int)(ARG(5) & 1));" + clear = "if (!d->ready_clear) d->ready_clear = 1, hos_system_event_clear(d->ready);" + out = "if (refuse) { RET(VIDEO_RESULT_INPUT_FULL); return; }" + vf.check("if (d && (d->ctx || d->mf) && input_limit()) {" in submit and refuse in submit and clear in submit and out in submit, + "dec_submit does not refuse by the rule, only for a decoder that decodes") + first_lock, first_unlock = submit.index("AcquireSRWLockExclusive(&d->lock);"), submit.index("ReleaseSRWLockExclusive(&d->lock);") + vf.check(first_lock < submit.index(refuse) < submit.index(clear) < first_unlock < submit.index(out) < submit.index("submitted();"), + "the refusal is not decided and the event cleared under the lock, before the submission is counted") + vf.check(re.search(r"input_room\(d\);[^\n]*\n\s+ReleaseSRWLockExclusive\(&d->lock\);\n\s+RET\(0\);\n}$", submit) is not None, + "dec_submit does not look for room before its last unlock") + flush = vf.block(video, "static void dec_flush(Ctx *c, const char *what) {") + vf.check(re.search(r"input_room\(d\);[^\n]*\n\s+ReleaseSRWLockExclusive\(&d->lock\);", flush) is not None + and flush.index("pending_free(d);") < flush.index("input_room(d);"), + "a Stop or a Flush does not set the event again before it lets the lock go") + acquire = vf.block(video, "static void dec_acquire(Ctx *c) {") + exits = re.findall(r"input_room\(d\);[^\n]*\n\s+ReleaseSRWLockExclusive\(&d->lock\);\n\s+if \(away\) hos_vcore_back\(\);", acquire) + vf.check(len(exits) == 3 and acquire.count("if (away) hos_vcore_back();") == 3, + "dec_acquire does not look for room before each of its three last unlocks (found %d)" % len(exits)) + final = vf.block(video, "static void dec_finalize(Ctx *c) {") + vf.check("if (d->ready_clear) hos_system_event_signal(d->ready);" in final, "a finalized decoder can leave its event clear") + init = vf.block(video, "static void dec_init_sized(Ctx *c, enum AVCodecID id, const char *what, int w, int h) {") + vf.check("d->ready = ready_new();" in init and init.index("d->ready = ready_new();") < init.index("set_host(ARG(0), DEC_MAGIC, d);"), + "a decoder does not have its ready event before the game can reach it") + ready = vf.block(video, "static uint64_t ready_new(void) {") + vf.check(ready.index("hos_system_event_init(event, 0);") < ready.index("hos_system_event_signal(event);"), "a new ready event is not set") + vf.check(video.count("{ RET(ready_event(c)); }") == 3 and "if (d && d->ready) return d->ready;" in vf.block(video, "static uint64_t ready_event(Ctx *c) {"), + "GetXDecoderSubmissionReadyEvent does not give the decoder's own event") + dev = (vf.ROOT / "docs" / "development.md").read_text(encoding="utf-8") + vf.check("`HOS_VIDEO_INPUT_LIMIT=`" in dev, "docs/development.md: HOS_VIDEO_INPUT_LIMIT is not in the switches table") + + with tempfile.TemporaryDirectory(prefix="video-input-limit-") as tmp: + src, exe = Path(tmp) / "rules.c", Path(tmp) / "rules.exe" + src.write_text(RULES, encoding="utf-8") + subprocess.run([args.cc, "-std=gnu11", "-O1", "-Wall", "-Wextra", "-Werror", "-I", str(vf.HOST), str(src), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + vf.check(r.returncode == 0, "the input limit's rule:\n" + r.stdout) + if not vf.WINDOWS: + print(vf.WINDOWS_ONLY) + print("PASS: the movie decoder refuses a submission while 16 wait, never the last (the rule and the source)") + return + exe = vf.build(tmp, args.cc, MOVIE) + pattern = (r"^movie: (\d+) taken at once, (\d+) acquired, last (\d), most waiting (\d+), frames refused (\d+), " + r"event cleared (\d+) times$") + # (HOS_VIDEO_INPUT_LIMIT, the limit it means, Media Foundation in place of libdav1d) + for env, limit, mf in ((None, 16, 0), ("16", 16, 1), ("4", 4, 0), ("0", 0, 0), ("0", 0, 1)): + out = vf.run(exe, HOS_VIDEO_INPUT_LIMIT=env, SI_LIMIT=limit, HOS_AV1="mf" if mf else None) + case = "HOS_VIDEO_INPUT_LIMIT=%s%s" % (env, " on Media Foundation" if mf else "") + vf.check("FAIL:" not in out, case + ":\n" + out[:3000]) + m = re.search(pattern, out, re.M) + vf.check(m is not None, case + ":\n" + out[:3000]) + at_once, acquired, last, most, refused_frames, clears = (int(x) for x in m.groups()) + vf.check(acquired == 600 and last == 1, "%s: %d of 600 pictures came out, the last marked %d" % (case, acquired, last)) + line = re.search(r"^video: a movie of 600 pictures on .*; 0 skipped to catch up(?:, (\d+) submissions refused while its input was full)?$", out, re.M) + vf.check(line is not None, case + ": the movie's line:\n" + out[-800:]) + if limit: + vf.check("first part: the refusals and the event" in out, case + ": the first part did not run") + # (the stream's last submission is taken though the decoder is full, and the end's mark with it: two over) + vf.check(at_once == limit and limit <= most <= limit + 2, "%s: %d taken at once and %d waiting at most, not %d" % (case, at_once, most, limit)) + vf.check(refused_frames > 500 and clears == refused_frames and line.group(1) == str(refused_frames), + "%s: %d frames ended in a refusal, the event was cleared %d times and the line counts %s" % (case, refused_frames, clears, line.group(1))) + else: + vf.check(at_once == 600 and refused_frames == 0 and clears == 0 and line.group(1) is None, + "%s: %d taken at once, %d refusals, %d clears: not every submission taken as before" % (case, at_once, refused_frames, clears)) + out = vf.run(exe, SI_DAV1D=0, SI_MF=0) + vf.check("FAIL:" not in out and "no decoder: 40 taken" in out, "a decoder with nothing behind it refuses submissions:\n" + out) + print("PASS: the movie decoder refuses a submission while 16 wait, never the last, with its ready event clear until there is room") + + +if __name__ == "__main__": + main() diff --git a/tests/test_video_lazy_rgba.py b/tests/test_video_lazy_rgba.py new file mode 100644 index 0000000..59692aa --- /dev/null +++ b/tests/test_video_lazy_rgba.py @@ -0,0 +1,213 @@ +"""A decoded movie picture is kept as NV12 alone, from libdav1d's planar 4:2:0 as from Media Foundation's NV12: its RGBA +is made only when the game reads RGBA (host/video_nv12.h, host/nn_video.c queue_frame; no game, no FFmpeg). + +(Triforce, 11-22, the performance audit's IO-36, after Source A's.) queue_frame converted every planar picture to RGBA +on the movie thread (a swscale pass, an 8 MB allocation and an 8 MB row copy at 1080p), zeroed its NV12 buffer and +interleaved U and V one byte at a time, for a game that reads only the NV12. Now planar 4:2:0 without alpha is kept as +NV12 alone, the buffer is zeroed only where a copy leaves bytes out, and the interleave does 16 pairs a step. +HOS_VIDEO_RGBA=1 makes the RGBA of every planar picture, as before. + +On every system this compiles video_nv12.h alone (which pictures are lazy, when the buffer is zeroed, the interleave +against one byte at a time for every length up to 70 with guard bytes around it) and reads the source: queue_frame asks +the rule, builds its swscale context and its RGBA only for a picture that is not lazy, zeroes by the rule, interleaves +with the header's loop; the switch is read once and is in the switches table. + +On Windows it also compiles nn_video.c whole over tests/video_standins.h and decodes pictures of each format: planar +4:2:0 at 1920x1080, 70x10 and 35x7 has no swscale context and no conversion, no RGBA, and an NV12 equal to a +reference built one byte at a time (the bytes an odd size leaves out are 0); with HOS_VIDEO_RGBA=1 there is one +conversion a picture and the same NV12; planar 4:2:0 with alpha is converted either way; a format queue_frame does +not copy is converted and its NV12 is zeros; Media Foundation's NV12 is lazy with or without the switch. And a lazy +picture taken by the game leaves its pool's RGBA to be made when ReadDecoderOutputImageRgba asks: one conversion then, +none before. + + python -I -B tests/test_video_lazy_rgba.py [--cc clang] +""" +import argparse +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import video_fixture as vf # noqa: E402 + +RUN_ALL_ARGS = ["--cc", "clang"] + +RULES = r""" +#include +#include +#include "video_nv12.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +int main(void) { + /* video_rgba_lazy(eager, nv12, planar 4:2:0 without alpha) */ + CHECK(video_rgba_lazy(0, 1, 0) && video_rgba_lazy(1, 1, 0)); /* NV12: always, as before */ + CHECK(video_rgba_lazy(0, 0, 1) && !video_rgba_lazy(1, 0, 1)); /* planar: unless HOS_VIDEO_RGBA=1 */ + CHECK(!video_rgba_lazy(0, 0, 0) && !video_rgba_lazy(1, 0, 0)); /* with alpha, or any other format: never */ + /* video_nv12_zero(w, h, a format queue_frame copies) */ + CHECK(!video_nv12_zero(1920, 1080, 1) && !video_nv12_zero(64, 36, 1)); + CHECK(video_nv12_zero(35, 8, 1) && video_nv12_zero(36, 7, 1) && video_nv12_zero(35, 7, 1)); + CHECK(video_nv12_zero(1920, 1080, 0)); + /* the interleave: every length to 70, at every offset of 0 to 3 (unaligned rows), nothing written past 2n */ + static uint8_t u[80], v[80], out[4 + 160 + 32], want[4 + 160 + 32]; + for (int i = 0; i < 80; i++) u[i] = (uint8_t)(i * 5 + 1), v[i] = (uint8_t)(200 - i * 3); + for (int n = 0; n <= 70; n++) + for (int at = 0; at < 4; at++) { + memset(out, 0xC3, sizeof out), memset(want, 0xC3, sizeof want); + for (int x = 0; x < n; x++) want[at + 2 * x] = u[at + x], want[at + 2 * x + 1] = v[at + x]; + video_uv_interleave(out + at, u + at, v + at, n); + CHECK(!memcmp(out, want, sizeof out)); + } +#if defined(__x86_64__) || defined(_M_X64) + CHECK(VIDEO_NV12_SSE2); /* (the 16-pair step is compiled here) */ +#endif + return fails ? 1 : 0; +} +""" + +PICTURES = r""" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +/* picture n as NV12, one byte at a time from the stand-in decoder's planes; what no copy reaches is 0 */ +static uint8_t *reference(int w, int h, int64_t n, int copied) { + uint8_t *r = (uint8_t *)calloc(1, (size_t)w * h * 3 / 2); + if (!copied) return r; + for (int y = 0; y < h; y++) + for (int x = 0; x < w; x++) r[(size_t)y * w + x] = si_byte(0, x, y, n); + for (int y = 0; y < h / 2; y++) + for (int x = 0; x < w / 2; x++) { + r[(size_t)w * h + (size_t)y * w + 2 * x] = si_byte(1, x, y, n); + r[(size_t)w * h + (size_t)y * w + 2 * x + 1] = si_byte(2, x, y, n); + } + return r; +} +int main(int argc, char **argv) { + CHECK(argc == 4); + const char *name = argv[1]; + const int w = atoi(argv[2]), h = atoi(argv[3]); + const int copied = strcmp(name, "other") != 0; + si.format = !strcmp(name, "yuv420p") ? AV_PIX_FMT_YUV420P : !strcmp(name, "yuva420p") ? AV_PIX_FMT_YUVA420P : + !strcmp(name, "nv12") ? AV_PIX_FMT_NV12 : AV_PIX_FMT_YUV420P10LE; + si.w = w, si.h = h; + si_open(w, h, 2, 3); /* (pools of format 3, as the game's) */ + Dec *d = si_dec(); + CHECK(d && (d->ctx || d->mf)); + /* three pictures decoded into the queue, as dec_acquire's first lines do */ + int rgba = 0, equal = 0, filled = 0; + for (int n = 0; n < 3; n++) { + CHECK(si_submit(n * 16683333ll, 0) == 0); + const int before = d->qn; + decode_next(d); + CHECK(d->qn == before + 1); + const Picture *p = &d->queue[(d->qhead + d->qn - 1) % MAX_QUEUE]; + uint8_t *want = reference(w, h, n, copied); + CHECK(p->w == w && p->h == h && p->nv12 != NULL); + equal += !memcmp(p->nv12, want, (size_t)w * h * 3 / 2); + rgba += p->rgba != NULL; + filled += p->rgba && p->rgba[0] == 0x5A && p->rgba[(size_t)w * h * 4 - 1] == 0x5A; /* (the stand-in's conversion) */ + free(want); + } + printf("decoder %s\n", d->ctx ? d->ctx->codec->name : "Media Foundation"); + printf("queued: rgba %d, nv12-equal %d, contexts %d, conversions %d, rgba-filled %d\n", rgba, equal, si.sws_contexts, + si.sws_scales, filled); + /* the game takes the first: its NV12 in the pool; the pool's RGBA only once it is read */ + const int b = si_acquire(); + CHECK(b >= 0); + Pool *o = si_pool(b); + uint8_t *want = reference(w, h, 0, copied); + int pool_equal = 1; /* (the pool's rows, at the pool's width: the same size here) */ + for (int y = 0; y < h + h / 2; y++) pool_equal &= !memcmp(o->nv12 + (size_t)y * w, want + (size_t)y * w, (size_t)w); + const int before = si.sws_scales, stale = o->rgba_stale; + if (w <= 64 && h <= 36) { + Ctx c; + memset(&c, 0, sizeof c); + c.x[0] = SI_POOLS + (uint64_t)b * 0x40, c.x[4] = (uint64_t)w, c.x[5] = (uint64_t)h, c.x[6] = 0x4000, c.x[7] = (uint64_t)w * 4; + imp__ZN2nn5codec26ReadDecoderOutputImageRgbaEPKNS0_25VideoDecoderImagePoolTypeEjttttPvmm(&c); + imp__ZN2nn5codec26ReadDecoderOutputImageRgbaEPKNS0_25VideoDecoderImagePoolTypeEjttttPvmm(&c); /* (twice: made once) */ + } + printf("taken: pool-nv12-equal %d, rgba-left-for-the-read %d, conversions-at-the-read %d, read-is-0x5A %d\n", pool_equal, + stale, si.sws_scales - before, si_guest[0x4000] == 0x5A && si_guest[0x4000 + (size_t)w * h * 4 - 1] == 0x5A); + free(want); + return fails ? 1 : 0; +} +""" + +# (format, w, h, HOS_VIDEO_RGBA, HOS_AV1) -> pictures with an RGBA of 3, swscale contexts, conversions +CASES = [ + (("yuv420p", 1920, 1080, None, None), 0, 0, 0), # libdav1d's pictures: NV12 alone + (("yuv420p", 70, 10, None, None), 0, 0, 0), # (35 pairs a row: two steps of 16 and three single) + (("yuv420p", 35, 7, None, None), 0, 0, 0), # (odd both ways: 17 pairs, a byte and rows no copy reaches) + (("yuv420p", 64, 36, None, None), 0, 0, 0), + (("yuv420p", 1920, 1080, "1", None), 3, 1, 3), # HOS_VIDEO_RGBA=1: one conversion a picture, one context + (("yuv420p", 35, 7, "1", None), 3, 1, 3), + (("yuv420p", 64, 36, "1", None), 3, 1, 3), + (("yuv420p", 64, 36, "0", None), 0, 0, 0), # (=0 is not the switch) + (("yuva420p", 64, 36, None, None), 3, 1, 3), # with alpha: made at once, either way + (("yuva420p", 64, 36, "1", None), 3, 1, 3), + (("other", 64, 36, None, None), 3, 1, 3), # a format that is not copied: converted, its NV12 zeros + (("nv12", 64, 36, None, "mf"), 0, 0, 0), # Media Foundation's NV12: as before + (("nv12", 64, 36, "1", "mf"), 0, 0, 0), # ... with the switch too, as before +] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cc", default="clang") + args = parser.parse_args() + video = vf.video_source() + + vf.check('#include "video_nv12.h"' in video, "nn_video.c does not take video_nv12.h") + eager = vf.block(video, "static int rgba_eager(void) {") + vf.check('if (on < 0) on = getenv("HOS_VIDEO_RGBA") != NULL && atoi(getenv("HOS_VIDEO_RGBA")) != 0;' in eager, + "HOS_VIDEO_RGBA is not read once") + queue = vf.block(video, "static void queue_frame(Dec *d, const AVFrame *f) {") + lazy = "const int lazy = video_rgba_lazy(rgba_eager(), f->format == AV_PIX_FMT_NV12, f->format == AV_PIX_FMT_YUV420P);" + vf.check(lazy in queue, "queue_frame does not ask the rule whether the picture is kept as NV12 alone") + vf.check("if (!lazy && (!d->sws || d->sws_w != w || d->sws_h != h || d->sws_fmt != f->format)) {" in queue + and queue.index(lazy) < queue.index("sws_getContext("), "queue_frame builds a swscale context for a lazy picture") + vf.check("p.rgba = lazy ? NULL : (uint8_t *)malloc((size_t)w * h * 4);" in queue, "queue_frame allocates RGBA for a lazy picture") + vf.check("if (d->sws && p.rgba) sws_scale(" in queue, "queue_frame converts a picture that has no RGBA buffer") + vf.check("calloc(" not in queue and "p.nv12 = (uint8_t *)malloc((size_t)w * h * 3 / 2);" in queue + and "if (video_nv12_zero(w, h, nv12 || planar)) memset(p.nv12, 0, (size_t)w * h * 3 / 2);" in queue, + "queue_frame zeroes every picture's NV12 buffer, or none") + vf.check("video_uv_interleave(uv + (size_t)y * w, f->data[1] + (size_t)y * f->linesize[1]," in queue + and "uv[(size_t)y * w + 2 * x]" not in queue, "queue_frame interleaves U and V one byte at a time") + vf.check("if (p->rgba && o->rgba) memcpy(" in video and "o->rgba_stale = !p->rgba;" in video, + "dec_acquire no longer leaves a lazy picture's RGBA to its pool") + dev = (vf.ROOT / "docs" / "development.md").read_text(encoding="utf-8") + vf.check("`HOS_VIDEO_RGBA=1`" in dev, "docs/development.md: HOS_VIDEO_RGBA=1 is not in the switches table") + + with tempfile.TemporaryDirectory(prefix="video-lazy-rgba-") as tmp: + src, exe = Path(tmp) / "rules.c", Path(tmp) / "rules.exe" + src.write_text(RULES, encoding="utf-8") + subprocess.run([args.cc, "-std=gnu11", "-O2", "-Wall", "-Wextra", "-Werror", "-I", str(vf.HOST), str(src), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + vf.check(r.returncode == 0, "the lazy rule, the zeroing rule or the interleave:\n" + r.stdout) + if not vf.WINDOWS: + print(vf.WINDOWS_ONLY) + print("PASS: planar 4:2:0 pictures are kept as NV12 alone unless HOS_VIDEO_RGBA=1 (the rules and the source)") + return + exe = vf.build(tmp, args.cc, PICTURES) + for (fmt, w, h, rgba_env, av1_env), rgba, contexts, conversions in CASES: + out = vf.run(exe, fmt, str(w), str(h), HOS_VIDEO_RGBA=rgba_env, HOS_AV1=av1_env) + case = "%s %dx%d, HOS_VIDEO_RGBA=%s, HOS_AV1=%s" % (fmt, w, h, rgba_env, av1_env) + vf.check("FAIL:" not in out, case + ":\n" + out) + vf.check(("decoder Media Foundation" in out) == (av1_env == "mf"), case + ": the wrong decoder:\n" + out) + m = re.search(r"^queued: rgba (\d+), nv12-equal (\d+), contexts (\d+), conversions (\d+), rgba-filled (\d+)$", out, re.M) + vf.check(m is not None, case + ":\n" + out) + got = tuple(int(x) for x in m.groups()) + vf.check(got == (rgba, 3, contexts, conversions, rgba), + "%s: of 3 pictures %d have an RGBA, %d an NV12 equal to the reference, with %d swscale contexts and %d " + "conversions (%d RGBA filled); expected %d, 3, %d, %d" % ((case,) + got + (rgba, contexts, conversions))) + m = re.search(r"^taken: pool-nv12-equal (\d), rgba-left-for-the-read (\d), conversions-at-the-read (\d+), read-is-0x5A (\d)$", out, re.M) + vf.check(m is not None and m.group(1) == "1", case + ": the pool's NV12 is not the picture's:\n" + out) + vf.check(m.group(2) == ("0" if rgba else "1"), case + ": whether the pool's RGBA waits for a read:\n" + out) + if w <= 64 and h <= 36: # (the read into the stand-in's guest window) + vf.check(m.group(3) == ("0" if rgba else "1") and m.group(4) == "1", + case + ": ReadDecoderOutputImageRgba does not make a lazy picture's RGBA once, then give it:\n" + out) + print("PASS: planar 4:2:0 pictures are kept as NV12 alone, their RGBA made when it is read; HOS_VIDEO_RGBA=1 as before") + + +if __name__ == "__main__": + main() diff --git a/tests/test_vk_pgo.py b/tests/test_vk_pgo.py new file mode 100644 index 0000000..e430186 --- /dev/null +++ b/tests/test_vk_pgo.py @@ -0,0 +1,102 @@ +"""The renderer's own code can be built with a play profile (nx2/build.py vk_pgo_flags, flag_text and the renderer's +jobs; nx2/make_game.py; host/pgo_rt.c; the packers; no compiler, no game). + +(Triforce, 11-21, the performance audit's LIFT-10, after Source A's.) The back end runs the replay on the game's render +thread and was built at plain -O2 whatever it does most. RX_VK_PGO=gen builds it counting its branches, a run writes the +counts, llvm-profdata merges them, and RX_VK_PGO= builds with that profile. + +Runs build.py's own vk_pgo_flags and flag_text, cut out through the AST: +- not set, empty or 0: no flags; gen: the counting flags; a file: built with it, its contents' SHA-1 in the flags; +- two copies of one profile at different paths give the same object key text, another profile another; +- the include folders and the sysroot are still left out of the key text. +And reads the sources: only the renderer's own files get the flags, the vendored recompiler does not; gen adds the +counters' writer to the DLL, compiled for RX_VK_PGO_FILE; the writer reads PGO_ENV, HOS_PGO unless told; make_game +copies the shipped profile, passes it only when RX_VK_PGO is not set and the compiler can read it, and keeps both +variables out of the lift's key; both packers carry the file; the switches are in the switches table. + + python -I -B tests/test_vk_pgo.py +""" +import ast +import hashlib +import os +import tempfile +from pathlib import Path +from types import SimpleNamespace + +ROOT = Path(__file__).resolve().parents[1] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def functions(source, names, **scope): + """the named top-level functions of a file, compiled alone into scope""" + tree = ast.parse(source) + body = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in names] + check(len(body) == len(names), "build.py no longer has %s" % ", ".join(names)) + exec(compile(ast.fix_missing_locations(ast.Module(body=body, type_ignores=[])), "build.py", "exec"), scope) + return scope + + +def main(): + build = (ROOT / "nx2" / "build.py").read_text(encoding="utf-8") + check('VK_PGO = os.environ.get("RX_VK_PGO", "")' in build, "build.py does not read RX_VK_PGO") + with tempfile.TemporaryDirectory(prefix="vk-pgo-") as tmp: + a, b, c = (os.path.join(tmp, n) for n in ("a.profdata", os.path.join("other", "a.profdata"), "c.profdata")) + os.makedirs(os.path.dirname(b)) + for path, data in ((a, b"one profile"), (b, b"one profile"), (c, b"another profile")): + with open(path, "wb") as f: + f.write(data) + scope = functions(build, ("vk_pgo_flags", "flag_text"), os=os, hashlib=hashlib, VK_PGO="", kit=SimpleNamespace(ROOT=tmp)) + flags, text = scope["vk_pgo_flags"], scope["flag_text"] + check(flags("") == [] and flags("0") == [], "no profile asked for still adds flags") + check(flags() == [], "the environment's value is not the default") + check(flags("gen") == ["-fprofile-instr-generate", "-mllvm", "-disable-vp=true"], "gen does not count the branches") + sha = hashlib.sha1(b"one profile").hexdigest()[:16] + use = flags(a) + check(use[:2] == ["-fprofile-instr-use=" + a, "-DTRIFORCE_VK_PROFILE_SHA1=" + sha], "a profile is not used, or its contents are not in the flags") + check("-Wno-profile-instr-out-of-date" in use, "a profile of an older renderer would stop the build with a warning as an error") + keys = [] + for path in (a, b, c): + scope["VK_PGO"] = path + keys.append(text(["-O2", "-Iinclude/here", "--sysroot=/tmp/.mount_x"] + flags(path))) + check(keys[0] == keys[1], "the same profile at another path is another object key: a second work folder compiles everything again") + check(keys[0] != keys[2], "another profile is the same object key: an update's new profile would not be compiled in") + check("" in keys[0] and a not in keys[0] and "-Iinclude" not in keys[0] and "--sysroot=" in keys[0], + "the key text names the profile by its path, or lost its other rules") + scope["VK_PGO"] = "" + check(text(["-fprofile-instr-use=" + a]) == "-fprofile-instr-use=" + a.replace(tmp, ""), + "the hot code's own profile flag is renamed too: its key would change") + + jobs = build[build.index("vk_pgo = vk_pgo_flags()"):build.index("if NGX_SDK and not LINUX and toolchain.mingw():")] + check("vflags + ([] if lib else vk_pgo) +" in jobs and 'lib = os.path.relpath(src, vk).split(os.sep)[0] in ("eden", "sirit")' in jobs, + "the profile's flags are not on the renderer's own files alone") + check('if VK_PGO == "gen" and not LINUX:' in jobs and 'os.path.join(HOST, "pgo_rt.c"), os.path.join(obj_dir, "vk_pgo_rt.o")' in jobs and + "'-DPGO_ENV=\"RX_VK_PGO_FILE\"'" in jobs, "a counting renderer has nothing to write its counters, or writes them for another variable") + rt = (ROOT / "host" / "pgo_rt.c").read_text(encoding="utf-8") + check('#define PGO_ENV "HOS_PGO"' in rt and "getenv(PGO_ENV)" in rt and "#ifndef PGO_ENV" in rt, + "the counters' writer does not read PGO_ENV, HOS_PGO unless told") + + make = (ROOT / "nx2" / "make_game.py").read_text(encoding="utf-8") + check('"hookshot_pieces.prof", "triforce_vk.profdata", "hookshot.res"' in make, "make_game does not copy the shipped renderer profile") + check('if "RX_VK_PGO" not in os.environ and os.path.exists(vk_profile) and profile_readable(vk_profile):' in make and + 'env["RX_VK_PGO"] = vk_profile' in make, + "the shipped profile is passed over a set RX_VK_PGO, or without asking whether the compiler can read it") + check('COMPILE_ONLY_ENV = ("RX_VK_PGO", "RX_VK_PGO_FILE")' in make, "the renderer profile's variables are not named as the compile's alone") + key = make[make.index("def lift_key(exefs, work, env, chunks=True):"):make.index("def key_after_lift(")] + check(key.index("lift_env = dict(env,") < key.index("for name in COMPILE_ONLY_ENV:") < key.index("lift_env.pop(name, None)") < + key.index("h.update(repr(sorted(lift_env.items())).encode())"), + "a renderer profile changes the lift's key: the whole game would be translated again for it") + check("'hookshot_pieces.prof', 'triforce_vk.profdata'" in (ROOT / "setup" / "pack_tools.ps1").read_text(encoding="utf-8"), + "setup/pack_tools.ps1 does not carry the renderer's profile") + check("hookshot_pieces.prof triforce_vk.profdata" in (ROOT / "scripts" / "linux" / "appimage.sh").read_text(encoding="utf-8"), + "scripts/linux/appimage.sh does not carry the renderer's profile") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`RX_VK_PGO=gen`, `=`, `=0`; `RX_VK_PGO_FILE=`" in dev, "docs/development.md: RX_VK_PGO is not in the switches table") + print("PASS: the renderer's own files take a play profile by its contents; the lift's key does not change") + + +if __name__ == "__main__": + main() diff --git a/tests/test_vram_compact.cpp b/tests/test_vram_compact.cpp index f1cc120..1b53aed 100644 --- a/tests/test_vram_compact.cpp +++ b/tests/test_vram_compact.cpp @@ -21,6 +21,10 @@ int main() { CHECK(run_from(512 * MB, false) == 512 * MB); CHECK(run_from(512 * MB, true) == 192 * MB); CHECK(run_from(64 * MB, true) == 64 * MB); + // ((Triforce, 11-22, TEX-38) unset, that free space by the card: allowance/16, 64-512 MB, 512 unknown; + // tests/test_compact_start.py has the cards from 2 to 32 GB) + CHECK(default_min_free(0) == 512 * MB && default_min_free(512 * MB) == 64 * MB); + CHECK(default_min_free(5233 * MB) >> 20 == 327 && default_min_free(16384 * MB) == 512 * MB); CHECK(rest(false) == 600 && rest(true) == 120); // only the textures' pool's free space comes off the count CHECK(reclaimable(850 * MB, 215 * MB) == 215 * MB); // (the rest is in the render targets' and the buffers' blocks) @@ -56,6 +60,18 @@ int main() { CHECK(rule_target(5600 * MB, 6144 * MB) == UINT64_MAX); // (under the budget: they ask nothing) CHECK(rule_target(5600 * MB, 5600 * MB) == UINT64_MAX); CHECK(rule_target(5600 * MB, 5300 * MB) == 5300 * MB / 8 * 7); // (over it: cut to 7/8 of it, as ever) + // ((Triforce) RX_VK_EVICT_EXCESS: the budget's size apart from what the 90% and 97% rules left. The size's part is 7/8 of + // it as above; the rules' part their budget less 64 MB. The Market on an 8 GB card: 2,873 MB held, a size of 3,285, the + // 90% rule's budget 2,811: 126 MB go, not 413) + CHECK(rule_target(2873 * MB, 3285 * MB, 2811 * MB) == 2811 * MB - kExcessSlack); + CHECK(2873 * MB - rule_target(2873 * MB, 3285 * MB, 2811 * MB) == 126 * MB); + CHECK(2873 * MB - rule_target(2873 * MB, 2811 * MB) > 413 * MB); // (the old cut: 7/8 of the rules' budget) + CHECK(rule_target(2800 * MB, 3285 * MB, 2811 * MB) == UINT64_MAX); // (under both: nothing) + CHECK(rule_target(5600 * MB, 6144 * MB, 6144 * MB) == UINT64_MAX); + CHECK(rule_target(5600 * MB, 5300 * MB, 5300 * MB) == 5300 * MB / 8 * 7); // (the size alone: as the old form) + CHECK(rule_target(5600 * MB, 5300 * MB, 5000 * MB) == 5300 * MB / 8 * 7); // (both over: the lower, here the size's) + CHECK(rule_target(5600 * MB, 5300 * MB, 4600 * MB) == 4600 * MB - kExcessSlack); // (and here the rules') + CHECK(rule_target(100 * MB, 3285 * MB, 40 * MB) == 0); // (never under nothing) CHECK(hold_target(100, 60, 4637 * MB, 5000 * MB, 4500 * MB) == 4637 * MB); // (the count over 90%: one unused 100 presents still goes) CHECK(hold_target(59, 60, 4637 * MB, 5000 * MB, 4500 * MB) == UINT64_MAX); // (one used in the last 60: never) CHECK(hold_target(100, 120, 4637 * MB, 5000 * MB, 4500 * MB) == UINT64_MAX); // (nor before the pass's own time, whatever that is) diff --git a/tests/test_vram_compact.py b/tests/test_vram_compact.py index b8300a1..6eb85d1 100644 --- a/tests/test_vram_compact.py +++ b/tests/test_vram_compact.py @@ -57,7 +57,8 @@ def check_source(): # (and what the other rules ask on their own is kept from before the hold's lines, the 97% rule's with it, for the # pass's third tier: a texture unused for the eviction's own time goes for that part) "uint64_t rule_budget = budget;", "rule_budget = std::min(rule_budget, total_bytes > over ? total_bytes - over : 0);", - "const uint64_t rule_to = vram_compact::rule_target(total_bytes, rule_budget);", + # ((Triforce) RX_VK_EVICT_EXCESS: the size apart, tests/test_evict_excess.py; =0 the old form) + "? vram_compact::rule_target(total_bytes, size_budget, rule_budget)", ": vram_compact::rule_target(total_bytes, rule_budget);", # (the reserve: a lower line that sets the budget too, the hold's own line kept for the textures not long unused; # the driver's limit, when it asks for more, clears both) "hold_to = evict_to;", "vram_compact::reserve_pct(pct));", "if (reserve < budget) budget = evict_to = reserve;", @@ -68,14 +69,14 @@ def check_source(): sweep_fn = block(tex[tex.index("void texture_frame(uint64_t completed)"):], "void texture_sweep(uint64_t completed") need("vk_texture.cpp: texture_sweep()", sweep_fn, ["const bool by_hold = evict_to != 0 && !sweep;", - "if (shifted || drawn_over(tex)) hold_dead += tex->bytes + tex->pad;", + "if (shifted || drawn_over(tex) || obsolete(tex)) hold_dead += tex->bytes + tex->pad;", # ((Triforce) the pass's own end, the hold's pass apart: 7/8 of a size, the budget itself by live use, tex_pressure.h) "if ((!sweep && !by_hold && tex_pressure::met(total_bytes, pending_bytes, budget, press.live)) || too_young(last))", "const uint64_t to = vram_compact::hold_target(age60(frame_no - last), need_unused, rule_to, hold_to, evict_to);", "if (to == UINT64_MAX || vram_compact::hold_met(total_bytes, hold_live, hold_dead, to)) continue;", "if (by_hold && !overwritten) hold_live += tex->bytes + tex->pad;"]) # (and a live texture is judged by the hold only once it is known whether a render target drew over it) - if not sweep_fn.index("const bool overwritten = shifted || drawn_over(tex);") < sweep_fn.index("vram_compact::hold_target(age60(frame_no - last)"): + if not sweep_fn.index("const bool overwritten = shifted || drawn_over(tex) || stale;") < sweep_fn.index("vram_compact::hold_target(age60(frame_no - last)"): raise SystemExit("vk_texture.cpp: the hold's pass judges a texture before it knows whether it is a dead render target") need("vk_texture.cpp: compact_step()", block(tex, "void compact_step(uint64_t completed)"), ["if (!compact_on()) return;", "vram_compact::run_from(min_free, compact.pressed)", "vram_compact::rest(compact.pressed)", diff --git a/tests/test_wait_hold_pause.py b/tests/test_wait_hold_pause.py new file mode 100644 index 0000000..19b91e4 --- /dev/null +++ b/tests/test_wait_hold_pause.py @@ -0,0 +1,434 @@ +"""A fence wait's hold for stopped work: counted, and with RX_VK_WAIT_HOLD_PAUSE_US paced by PAUSE gaps, the order lock taken +only when it can say something new (gpu/vk/wait_hold.h, vk_replay.cpp wait(); no game, no GPU). + +(Triforce, 11-22, the performance audit's THREAD-11, after Source B's.) Off (0) unless set: a yield a look and the lock at +each look, as before. + +The header's rules are compiled and run (the switch's values, when a look takes the lock, the gap's PAUSEs, the order line). +Then the hold loop itself, cut out of vk_replay.cpp's wait() with the real OrderMutex, runs against stand-ins: the fence's +value, the oldest unfinished stopped submission (which, as in the renderer, changes only inside a section of the order +lock), the flush, the forced drain, and a clock that only the loop's own yields and PAUSEs move, so nothing depends on this +PC's speed. The wait modelled: the waiter is the hold; the waker is another thread's section of the order lock that lets +the stopped work on, or a signal of the CPU's (ev_cpu_any) queued without a section, which a section then acts on; the +condition is "the oldest unfinished stopped submission is past this fence". + +- 0: every look takes the lock and yields; nothing changing, the hold ends forced at its bound with one drain; +- 2: no yield, a gap of PAUSEs (16 at a time, 2 us on the clock) between looks, the lock at the first look only while no + section leaves; a section that leaves without changing the answer costs one more take and no more; a signal of the + CPU's queued: the lock at every look until a section has taken it, then left alone again; where the waiter's own + section acts on a queued signal (the audit's SYNC-59 gives the hold such a look; it is not in this tree) the work it + lets on is seen at the first look after the signal, not at the bound; a section that lets the work on is seen at the + first look after it leaves and the hold ends there, no drain; a section in progress on another thread (the lock held + across many looks) never blocks the waiter and is seen when it leaves; the fence done ends the hold at that look; the + wait's own time ends it; the bound still forces the drain; +- the hold is skipped as before with no time to wait, with the replay on its own thread, and with nothing stopped; +- on real threads, 300 rounds: a signaller makes sections at its own pace and then lets the work on, by a section of its + own or (every other round) by a signal of the CPU's that an order thread's section acts on when it gets the lock (as + sync_watch), while the waiter holds with the bound far away: every round ends by "let on" (a lost section or signal + would hang the round: the test's time limit). +Each rule taken out of the cut loop in turn (a mutation) must make a case fail. + + python -I -B tests/test_wait_hold_pause.py [--cxx clang++] +""" +import argparse +import os +import subprocess +import tempfile +from pathlib import Path + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +HOLD = " if (timeout_ns && !async_on() && blocked_n.load()) {" + + +def fail(message): + raise SystemExit("FAIL: " + message) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +FIXTURE = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include "lock_census.h" +#include "wait_hold.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +namespace mkwvk { +void logf(const char*, ...) {} +static std::atomic order_waiters{0}; +static std::mutex order_wait_mu; +static std::condition_variable order_wait_cv; +/* ORDER */; +/* ORDER_UNLOCK */ +static OrderMutex sync_order_mu; + +// ---- the stand-ins ---- +// the clock: moved by the loop's own yields (1 us each) and PAUSEs (50 ns each) alone +static int64_t clock_ns = 1000000000; +struct test_clock { + typedef std::chrono::nanoseconds duration; + typedef duration::rep rep; + typedef duration::period period; + typedef std::chrono::time_point time_point; + static time_point now() { return time_point(duration(clock_ns)); } +}; +static int64_t ticks, yields, pauses, flushes, drains, looks_now, asked_unlocked; +static void (*at_tick)(int64_t) = nullptr; // (what the rest of the program does meanwhile, by the loop's ticks) +static void tick() { + ticks++; + if (at_tick) at_tick(ticks); +} +static void test_yield() { + yields++, clock_ns += 1000; + tick(); +} +namespace gate_wait { +static void pause() { + pauses++, clock_ns += 50; + if (pauses % wait_hold::kPauses == 0) tick(); +} +} // namespace gate_wait +static bool async = false; +static bool async_on() { return async; } +static std::atomic blocked_n{1}; +enum { FR_WAIT = 2 }; +static void queue_flush_for(int why) { CHECK(why == FR_WAIT); flushes++; } +static std::atomic done_value{0}, oldest{0}; +static std::atomic ev_cpu_any{false}; // (a signal of the CPU's queued, which no section has acted on yet) +static std::atomic signal_oldest{0}; // (the oldest unfinished stopped submission once a section has acted on it) +static bool own_look_acts = false; // (the waiter's own section acts on a queued signal, as the audit's SYNC-59 adds) +// (SYNC-59's own look in the hold's lock section, RX_VK_ORDER_WAKE=1: off here, as by default. own_look_acts above plays it) +static bool order_wake_on() { return false; } +static int64_t st_resumed, st_wake_looks, st_wake_resumes; +static void burst_begin(int) {} +static void schedule() {} +static void burst_end() {} +static void update_blocked() {} +static uint64_t completed() { + looks_now++; + return done_value.load(); +} +// (inside a section: the queued signal taken in, and what its event lets on let on, as schedule() does) +static void apply_signal() { + if (sync_order_mu.entered.load() - sync_order_mu.left.load() != 1) asked_unlocked++; + oldest = signal_oldest.load(); + ev_cpu_any = false; +} +// (asked only inside a section of the order lock, as the renderer's state it stands for is only changed in one) +static uint64_t oldest_unfinished() { + if (sync_order_mu.entered.load() - sync_order_mu.left.load() != 1) asked_unlocked++; + if (own_look_acts && ev_cpu_any.load()) apply_signal(); + return oldest.load(); +} +static void sync_drain(int why) { CHECK(why == 2); drains++; } +static wait_hold::Stats hold_stats; +static int64_t pause_now; +static int64_t wait_hold_pause_ns() { return pause_now; } + +// 1: the fence was done in the hold; 0: the hold is over (or there was none) and the timeline's own wait follows +static int hold(uint64_t f, uint64_t timeout_ns) { +/* HOLD */ + return 0; +} +} // namespace mkwvk +using namespace mkwvk; + +static const uint64_t F = 100; +// another thread's section of the order lock, run between two of the loop's steps +static void section(uint64_t new_oldest) { + std::lock_guard lk(sync_order_mu); + oldest = new_oldest; +} +static int64_t held0; +static void fresh(int64_t pause_us) { + pause_now = pause_us * 1000; + ticks = yields = pauses = flushes = drains = looks_now = asked_unlocked = 0; + at_tick = nullptr, async = false, blocked_n = 1, done_value = 0, oldest = F; // (stopped work at this fence: needed) + ev_cpu_any = false, signal_oldest = F, own_look_acts = false; + for (auto* a : {&hold_stats.holds, &hold_stats.looks, &hold_stats.locks, &hold_stats.held_ns}) a->store(0); + for (auto& e : hold_stats.ended) e.store(0); + held0 = clock_ns; +} +static int64_t ended(int how) { return hold_stats.ended[how].load(); } +static int64_t section_looks; // (the looks made when a scripted section left) + +int main(int argc, char** argv) { + const std::string c = argc > 1 ? argv[1] : ""; + const uint64_t far = 60000000000ull; + if (c == "script") { + // ---- 0: as before ---- + fresh(0); + CHECK(hold(F, far) == 0); + CHECK(drains == 1 && ended(wait_hold::kForced) == 1 && hold_stats.holds.load() == 1); + CHECK(hold_stats.looks.load() == looks_now && looks_now == 3001); // (a look a microsecond for the bound's 3 ms) + CHECK(hold_stats.locks.load() == looks_now && asked_unlocked == 0); // (the lock at every look) + CHECK(yields == looks_now - 1 && pauses == 0 && flushes == looks_now); // (a yield after every look but the last) + CHECK(hold_stats.held_ns.load() == clock_ns - held0 && clock_ns - held0 == 3000000); + // ---- 2 us: the gap, and the lock at the first look alone ---- + fresh(2); + CHECK(hold(F, far) == 0); + CHECK(drains == 1 && ended(wait_hold::kForced) == 1); // (the bound and the forced drain stay) + CHECK(yields == 0 && pauses == (looks_now - 1) * 48); // (three rounds of 16 PAUSEs make 2 us here) + CHECK(hold_stats.locks.load() == 1 && asked_unlocked == 0 && hold_stats.looks.load() == looks_now); + CHECK(looks_now == 1251 && flushes == looks_now); // (a look every 2.4 us) + // a section that leaves and changes nothing: one more take, then none + fresh(2); + at_tick = [](int64_t t) { if (t == 300) section(F); }; + CHECK(hold(F, far) == 0); + CHECK(hold_stats.locks.load() == 2 && ended(wait_hold::kForced) == 1 && drains == 1); + // a signal of the CPU's queued for a while (it acts in no section until the order thread's): the lock at every look + // meanwhile, as before, and left alone again once a section has taken the signal + fresh(2); + at_tick = [](int64_t t) { + if (t == 300) ev_cpu_any = true; + if (t == 600) { std::lock_guard lk(sync_order_mu); ev_cpu_any = false; } + }; + CHECK(hold(F, far) == 0); + CHECK(hold_stats.locks.load() == 1 + 100 + 1 && ended(wait_hold::kForced) == 1 && drains == 1); + // the waiter's own section acting on a queued signal (SYNC-59's look in the hold): no other section leaves, and the work + // the signal lets on is seen at the first look after it, not forced at the bound + fresh(2); + own_look_acts = true, signal_oldest = F + 1; + at_tick = [](int64_t t) { if (t == 300) ev_cpu_any = true, section_looks = looks_now; }; + CHECK(hold(F, far) == 0); + CHECK(ended(wait_hold::kLetOn) == 1 && drains == 0 && hold_stats.locks.load() == 2 && looks_now == section_looks + 1); + CHECK(!ev_cpu_any.load() && asked_unlocked == 0); + // a section that lets the work on: seen at the first look after it leaves + fresh(2); + at_tick = [](int64_t t) { if (t == 600) section(F + 1), section_looks = looks_now; }; + CHECK(hold(F, far) == 0); + CHECK(ended(wait_hold::kLetOn) == 1 && drains == 0 && hold_stats.locks.load() == 2); + CHECK(looks_now == section_looks + 1 && looks_now < 300); + // the same at 0 + fresh(0); + at_tick = [](int64_t t) { if (t == 200) section(F + 1), section_looks = looks_now; }; + CHECK(hold(F, far) == 0); + CHECK(ended(wait_hold::kLetOn) == 1 && drains == 0 && looks_now == section_looks + 1); + // the fence done: the hold ends at that look, with the lock not asked again + for (int64_t us : {0, 2}) { + fresh(us); + at_tick = [](int64_t t) { if (t == 150) done_value = F, section_looks = looks_now; }; + CHECK(hold(F, far) == 1); + CHECK(ended(wait_hold::kMet) == 1 && drains == 0 && looks_now == section_looks + 1); + CHECK(hold_stats.locks.load() == (us ? 1 : looks_now - 1)); + } + // the wait's own time: 1 ms, before the bound + for (int64_t us : {0, 2}) { + fresh(us); + CHECK(hold(F, 1000000) == 0); + CHECK(ended(wait_hold::kTimedOut) == 1 && drains == 0 && clock_ns - held0 >= 1000000 && clock_ns - held0 < 1010000); + } + // a fence done already, and work older than the stopped ones: one look + fresh(2), done_value = F + 5; + CHECK(hold(F, far) == 1 && looks_now == 1 && hold_stats.locks.load() == 0); + fresh(2), oldest = F + 1; + CHECK(hold(F, far) == 0 && looks_now == 1 && ended(wait_hold::kLetOn) == 1 && hold_stats.locks.load() == 1); + // no hold at all: no time to wait, the replay on its own thread, nothing stopped + fresh(2); + CHECK(hold(F, 0) == 0); + async = true; + CHECK(hold(F, far) == 0); + async = false, blocked_n = 0; + CHECK(hold(F, far) == 0); + CHECK(looks_now == 0 && hold_stats.holds.load() == 0 && flushes == 0); + // a section in progress on another thread: the lock held across many looks, which never block on it + if (fails) return 1; // (a loop that already takes the lock at every look would wait here for good) + { + fresh(2); + static std::atomic stage{0}; + stage = 0; + std::thread other([] { + while (stage.load() < 1) std::this_thread::yield(); + sync_order_mu.lock(); + oldest = F + 1; // (the work let on, the section not left yet) + stage = 2; + while (stage.load() < 3) std::this_thread::yield(); + sync_order_mu.unlock(); + stage = 4; + }); + at_tick = [](int64_t t) { + if (t == 100) { stage = 1; while (stage.load() < 2) std::this_thread::yield(); } + if (t == 400) { stage = 3; while (stage.load() < 4) std::this_thread::yield(); section_looks = looks_now; } + }; + CHECK(hold(F, far) == 0); + other.join(); + CHECK(ended(wait_hold::kLetOn) == 1 && drains == 0 && hold_stats.locks.load() == 2 && looks_now == section_looks + 1); + } + // the order line + fresh(2); + hold_stats.done(wait_hold::kMet, 10, 1, 1500000), hold_stats.done(wait_hold::kForced, 1251, 1, 3000000); + hold_stats.done(wait_hold::kLetOn, 40, 2, 100000), hold_stats.done(wait_hold::kTimedOut, 400, 1, 1000000); + char line[512]; + wait_hold::describe(hold_stats, 2000, line, sizeof line); + if (strcmp(line, "vk: fence waits held for stopped work: 4 holds, 1701 looks, 5.6 ms held, the order lock taken 5 times; ended by " + "the fence done 1, the work let on 1, the wait's time up 1, forced on at the bound 1 (RX_VK_WAIT_HOLD_PAUSE_US=2)")) { + std::printf("FAIL: the line is\n %s\n", line); + fails++; + } + // the rules alone + using namespace wait_hold; + CHECK(pause_ns(nullptr) == 0 && pause_ns("") == 0 && pause_ns("0") == 0 && pause_ns("x") == 0 && pause_ns("-3") == 0); + CHECK(pause_ns("2") == 2000 && pause_ns("1") == 1000 && pause_ns("1000") == 1000000 && pause_ns("5000") == 1000000); + CHECK(takes_lock(0, false, 0, 0, false) && takes_lock(0, true, 7, 8, false) && takes_lock(0, true, 7, 9, false)); // (0: always) + CHECK(takes_lock(2000, false, 0, 0, false) && takes_lock(2000, false, 7, 8, false)); // (not asked yet) + CHECK(!takes_lock(2000, true, 7, 8, false)); // (only its own section left) + CHECK(takes_lock(2000, true, 7, 9, false) && takes_lock(2000, true, 7, 100, false) && takes_lock(2000, true, 7, 7, false)); + CHECK(takes_lock(2000, true, 7, 8, true)); // (a signal of the CPU's queued) + int64_t t = 0, n = 0; + gap(2000, [&] { return t; }, [&] { n++, t += 50; }); + CHECK(n == 48 && t == 2400); + t = 0, n = 0; + gap(100, [&] { return t; }, [&] { n++, t += 50; }); + CHECK(n == 16); // (a round at least) + } else if (c == "storm") { // RX_VK_WAIT_HOLD_US far away: only a section that lets the work on ends a round + for (int round = 0; round < 300 && !fails; round++) { + fresh(2); + const bool by_signal = round % 2 == 1; // (the work let on by a signal of the CPU's rather than by a section) + signal_oldest = F + 1; + std::atomic begun{false}, over{false}; + // the order thread, as sync_watch: a section when it can get the lock, which acts on a queued signal + std::thread order([&] { + while (!over.load()) { + if (!ev_cpu_any.load()) { + std::this_thread::yield(); + continue; + } + std::unique_lock lk(sync_order_mu, std::try_to_lock); + if (lk && ev_cpu_any.load()) apply_signal(); + } + }); + std::thread signaller([&] { + begun = true; + unsigned seed = 77u + (unsigned)round * 7919u; + const int sections = 1 + round % 7; + for (int i = 0; i < sections; i++) { + for (unsigned spin = (seed = seed * 1103515245u + 12345u) >> 20; spin; spin--) gate_wait_spin(); + section(F); // (a section that changes nothing) + } + if (by_signal) ev_cpu_any.store(true, std::memory_order_release); // (queued without a section) + else section(F + 1); // (the one that lets the work on) + }); + while (!begun.load()) std::this_thread::yield(); + CHECK(hold(F, 1ull << 62) == 0); + signaller.join(); + over = true; + order.join(); + CHECK(ended(wait_hold::kLetOn) == 1 && drains == 0 && asked_unlocked == 0 && oldest.load() == F + 1 && !ev_cpu_any.load()); + } + } else { + std::printf("FAIL: unknown case\n"); + fails++; + } + if (!fails) std::printf("PASS %s\n", c.c_str()); + return fails ? 1 : 0; +} +''' + +# (each: what is taken out, the text in the cut loop as the fixture has it, what replaces it) +MUTATIONS = [ + ("the lock taken again once another section has left", + "wait_hold::takes_lock(pause_ns, asked, seen_left, sync_order_mu.left.load(std::memory_order_acquire),\n" + " ev_cpu_any.load(std::memory_order_acquire))", "(!pause_ns || !asked)"), + ("the lock taken while a signal of the CPU's is queued", "ev_cpu_any.load(std::memory_order_acquire))", "false)"), + ("the lock left alone while no section has left", + "seen_left = sync_order_mu.left.load(std::memory_order_relaxed);", "seen_left = sync_order_mu.left.load(std::memory_order_relaxed) + 1;"), + ("the gap in place of the yield", "if (!pause_ns) test_yield();", "if (true) test_yield();"), + ("the yield with the switch at 0", "if (!pause_ns) test_yield();", "if (false) test_yield();"), + ("the forced drain at the bound", " sync_drain(2);\n", ""), + ("the hold's count", " hold_stats.done(end, looks, locks, since());\n", ""), + ("the fence's value at every look", " if (completed() >= f) {", " if (looks == 1 && completed() >= f) {"), +] + + +def read_sources(replay, dev): + if '#include "wait_hold.h"' not in replay: + fail("vk_replay.cpp does not include wait_hold.h") + pause = block(replay, "int64_t wait_hold_pause_ns() {") + if 'static const int64_t v = wait_hold::pause_ns(getenv("RX_VK_WAIT_HOLD_PAUSE_US"));' not in pause: + fail("vk_replay.cpp: the switch is not the header's rule over RX_VK_WAIT_HOLD_PAUSE_US, read once") + stats = block(replay, "void log_order_stats() {") + if "wait_hold::describe(hold_stats, wait_hold_pause_ns(), line, sizeof line);" not in stats: + fail("vk_replay.cpp log_order_stats: the holds are not in the order's report") + wait = block(replay, "int wait(uint64_t f, uint64_t timeout_ns) {") + if wait.count(HOLD) != 1: + fail("vk_replay.cpp wait(): the hold for stopped work is not there once") + hold = block(wait, HOLD) + for need in ("std::chrono::steady_clock", "std::this_thread::yield();", "gate_wait::pause"): + if need not in hold: + fail("vk_replay.cpp wait(): %r is not in the hold loop (the fixture stands in for it)" % need) + if not hold.index("if (completed() >= f) {") < hold.index("wait_hold::takes_lock("): + fail("vk_replay.cpp wait(): the fence's value is not read before the lock is thought of") + if "| `RX_VK_WAIT_HOLD_PAUSE_US=` |" not in dev: + fail("docs/development.md: RX_VK_WAIT_HOLD_PAUSE_US is not in the switches table") + # (the loop as the fixture runs it: the clock, the yield and the PAUSE are the fixture's) + return hold.replace("std::chrono::steady_clock", "test_clock").replace("std::this_thread::yield();", "test_yield();") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + try: + replay = (VK / "vk_replay.cpp").read_text(encoding="utf-8") + hold = read_sources(replay, (ROOT / "docs/development.md").read_text(encoding="utf-8")) + fixture = FIXTURE.replace("/* ORDER */", block(replay, "struct OrderMutex {")) \ + .replace("/* ORDER_UNLOCK */", block(replay, "void OrderMutex::unlock() {")) \ + .replace("gate_wait_spin();", "_mm_pause();") + except (ValueError, OSError) as e: + fail("a text this test reads is not there (%s)" % e) + env = {k: v for k, v in os.environ.items() if k not in ("RX_VK_WAIT_HOLD_US", "RX_VK_WAIT_HOLD_PAUSE_US", "RX_VK_LOCK_CENSUS")} + with tempfile.TemporaryDirectory(prefix="wait-hold-") as tmp: + def build(name, loop): + src, exe = Path(tmp) / (name + ".cpp"), Path(tmp) / (name + ".exe") + src.write_text("#include \n" + fixture.replace("/* HOLD */", loop), encoding="utf-8") + built = subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", "-Wno-unused-function", "-Wno-unused-variable", + "-Wno-unused-but-set-variable", "-pthread", "-I" + str(VK), str(src), "-o", str(exe)], + capture_output=True, text=True) + if built.returncode: + print(built.stdout + built.stderr) + fail("the fixture did not build (%s)" % name) + return exe + + def run(exe, case, **switches): + try: + return subprocess.run([str(exe), case], env=dict(env, **switches), capture_output=True, text=True, timeout=120) + except subprocess.TimeoutExpired: + return subprocess.CompletedProcess([], 1, "no end in 120 s (a section never seen, or a look that blocked)", "") + + exe = build("hold", hold) + for case, switches in (("script", {}), ("storm", {"RX_VK_WAIT_HOLD_US": "60000000000"})): + done = run(exe, case, **switches) + if done.returncode or "PASS " + case not in done.stdout: + print(done.stdout[-3000:] + done.stderr[-1000:]) + fail("the hold loop: " + case) + print("PASS " + case) + for i, (what, old, new) in enumerate(MUTATIONS): + if hold.count(old) != 1: + fail("mutation %r: %r is not in the cut loop once (update the test with the code)" % (what, old)) + done = run(build("mutant%d" % i, hold.replace(old, new)), "script") + if done.returncode == 0: + fail("mutation %r: the script still passed without it" % what) + first = [line for line in done.stdout.splitlines() if line.startswith("FAIL") or "no end" in line] + print("PASS mutation: without %s the script fails (%s)" % (what, first[0][:110] if first else "exit %d" % done.returncode)) + print("PASS: the hold is counted; at 0 it is as before; with the gap the lock is taken only after a section has left, and no section is missed") + + +if __name__ == "__main__": + main() diff --git a/tests/test_warm_computes.cpp b/tests/test_warm_computes.cpp new file mode 100644 index 0000000..4ad3e06 --- /dev/null +++ b/tests/test_warm_computes.cpp @@ -0,0 +1,32 @@ +// tests/test_warm_computes.py: gpu/vk/warm_computes.h's rule, no game or GPU +#include "../gpu/vk/warm_computes.h" + +#include +#include + +static int failures = 0; +static void check(bool ok, const char* what) { + if (!ok) std::printf("FAIL: %s\n", what), failures++; +} + +int main() { + using namespace warm_computes; + // (unset: listed on Intel, every one on the others) + check(listed_only(nullptr, 0x8086), "Intel, unset: listed"); + check(!listed_only(nullptr, 0x10DE), "NVIDIA, unset: all"); + check(!listed_only(nullptr, 0x1002), "AMD, unset: all"); + // (the switch decides on any GPU; anything else is as unset) + check(listed_only("listed", 0x10DE), "NVIDIA, listed"); + check(listed_only("listed", 0x1002), "AMD, listed"); + check(!listed_only("all", 0x8086), "Intel, all"); + check(listed_only("", 0x8086) && !listed_only("", 0x10DE), "empty: as unset"); + check(listed_only("1", 0x8086) && !listed_only("1", 0x1002), "other text: as unset"); + // (built: without listed_only every one; with it the listed ones, and every one while there is no list) + check(build(false, false, false) && build(false, true, false) && build(false, false, true), "all: every one"); + check(build(true, true, false), "listed, no list yet: every one"); + check(build(true, false, true), "listed: a listed program"); + check(!build(true, false, false), "listed: a program play has not dispatched is left"); + if (failures) return 1; + std::printf("warm_computes: all checks passed\n"); + return 0; +} diff --git a/tests/test_warm_computes.py b/tests/test_warm_computes.py new file mode 100644 index 0000000..c52e5a6 --- /dev/null +++ b/tests/test_warm_computes.py @@ -0,0 +1,65 @@ +"""On Intel's GPUs the shader preparation's translation pass builds only the compute pipelines play dispatches. + +The pass (gpu/vk/vk_draw.cpp prepare_run) had the driver compile every compute program's texture variants, 9681 pipelines +on an Arc A750: 5175 s of its 5858 s pass (an RTX 4070 SUPER takes 170 s for the whole pass), where play dispatched 166 +programs (920 pipelines), which the precompile builds from triforce_computes.bin anyway. + +What must hold: gpu/vk/warm_computes.h's rule (compiled and run here, no game or GPU): listed only on Intel unless +RX_VK_WARM_COMPUTES says listed or all; with listed, every one while there is no list, else the listed ones. And +vk_draw.cpp: prepare_run's warm_compute asks the rule with the list computes_read gives, before the carry's note and the +build, and says what it left; the switch is in docs/development.md. + +python -I -B tests/test_warm_computes.py [--cxx g++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + +ROOT = Path(__file__).resolve().parents[1] + + +def need(where, text, needles): + for needle in needles: + if needle not in text: + raise SystemExit(where + " no longer has: " + needle) + + +def check_source(): + draw = (ROOT / "gpu/vk/vk_draw.cpp").read_text(encoding="utf-8") + need("vk_draw.cpp", draw, ['#include "warm_computes.h"']) + start = draw.index("bool prepare_run(void (*progress)(size_t done, size_t total), bool background) {") + prepare = draw[start:draw.index("\n}\n", start)] + need("vk_draw.cpp: prepare_run()", prepare, + ['warm_computes::listed_only(getenv("RX_VK_WARM_COMPUTES"), dev.props.vendorID)', + "cs_listed_only ? computes_read() : std::unordered_set{}", + "warm_computes::build(cs_listed_only, cs_listed.empty(), cs_listed.count(p.key) != 0)", + "vk: the preparation left %zu compute variants"]) + warm = prepare[prepare.index("const auto warm_compute = [&](Program& p) {"):] + warm = warm[:warm.index("};")] + # (left before the carry's note, so that the carry's rebuild builds the same ones, and before the build) + order = [warm.index(s) for s in ("warm_computes::build(", "prepare_cs_keys.insert(p.key);", "prepare_compute_pipeline(p)")] + if order != sorted(order): + raise SystemExit("vk_draw.cpp: warm_compute asks the rule after the carry's note or the build") + if "const bool cs_listed_only = warm_cs &&" not in prepare: + raise SystemExit("vk_draw.cpp: the list is read with no compute pipelines to build (behind the game, RX_VK_NO_WARM)") + need("docs/development.md", (ROOT / "docs/development.md").read_text(encoding="utf-8"), + ["`RX_VK_WARM_COMPUTES=listed` or `all`"]) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="g++") + args = parser.parse_args() + check_source() + with tempfile.TemporaryDirectory(prefix="warm-computes-") as tmp: + exe = Path(tmp) / "warm-computes" + subprocess.run([args.cxx, "-std=c++20", "-Wall", "-Wextra", "-Werror", + str(ROOT / "tests/test_warm_computes.cpp"), "-o", str(exe)], check=True) + subprocess.run([str(exe)], check=True) + + +if __name__ == "__main__": + main() diff --git a/tests/test_wb_diff_stale_shadow.py b/tests/test_wb_diff_stale_shadow.py index 9843b68..3082a58 100644 --- a/tests/test_wb_diff_stale_shadow.py +++ b/tests/test_wb_diff_stale_shadow.py @@ -85,6 +85,7 @@ Host the_host; Host* host = &the_host; bool diff_on = true; bool wb_diff_on() { return diff_on; } +bool wb_notes_off() { return false; } // ((Triforce, 11-22, MEM-51) the mirrors are on here: tests/test_mirror_bookkeeping.py has the case without std::mutex shadow_written_mu; std::vector shadow_written_pages, shadow_written_collected_list, shadow_written_by_replay; bool shadow_written_all = false; diff --git a/tests/test_wb_excl_clip.cpp b/tests/test_wb_excl_clip.cpp new file mode 100644 index 0000000..9d2b88a --- /dev/null +++ b/tests/test_wb_excl_clip.cpp @@ -0,0 +1,111 @@ +// gpu/vk/wb_excl_clip.h without the game or a GPU: which parts of a written binding are copied back, and the 0.6.0 heap +// corruption it is for, played on a page of bytes (tests/test_wb_excl_clip.py) +#include "../gpu/vk/wb_excl_clip.h" + +#include +#include +#include + +using wb_excl_clip::Range; + +static int fails = 0; +#define CHECK(c) \ + do { \ + if (!(c)) { \ + std::fprintf(stderr, "%s:%d: CHECK(%s)\n", __FILE__, __LINE__, #c); \ + fails++; \ + } \ + } while (0) + +static std::vector kept(uint64_t lo, uint64_t hi, const std::vector& ex, uint64_t* dropped = nullptr) { + std::vector out; + const uint64_t d = wb_excl_clip::kept(lo, hi, ex, out); + if (dropped) *dropped = d; + return out; +} + +// A write-back as the diff does it: the mirror's words that differ from the shadow (guest memory) copied over guest memory +static void write_back(uint8_t* guest, const uint8_t* mirror, uint64_t base, const std::vector& parts) { + for (const auto& [a, b] : parts) + for (uint64_t i = a; i < b; i++) + if (mirror[i - base] != guest[i - base]) guest[i - base] = mirror[i - base]; +} + +int main() { + const std::vector none; + const std::vector ex = {{0x2000, 0x3000}, {0x5000, 0x6000}}; + uint64_t d = 0; + + // nothing left out: the whole binding, as before + CHECK((kept(0x1000, 0x1040, none, &d) == std::vector{{0x1000, 0x1040}}) && d == 0); + // outside every left-out range: whole + CHECK((kept(0x1000, 0x2000, ex, &d) == std::vector{{0x1000, 0x2000}}) && d == 0); + CHECK((kept(0x3000, 0x5000, ex, &d) == std::vector{{0x3000, 0x5000}}) && d == 0); + CHECK((kept(0x6000, 0x7000, ex, &d) == std::vector{{0x6000, 0x7000}}) && d == 0); + // a bare entry's 64-byte head inside one: nothing copied back + CHECK(kept(0x2000, 0x2040, ex, &d).empty() && d == 0x40); + CHECK(kept(0x2fc0, 0x3000, ex, &d).empty() && d == 0x40); + // across one edge, either way + CHECK((kept(0x1f00, 0x2100, ex, &d) == std::vector{{0x1f00, 0x2000}}) && d == 0x100); + CHECK((kept(0x2f00, 0x3100, ex, &d) == std::vector{{0x3000, 0x3100}}) && d == 0x100); + // across both: the parts between and beside them + CHECK((kept(0x1000, 0x8000, ex, &d) == std::vector{{0x1000, 0x2000}, {0x3000, 0x5000}, {0x6000, 0x8000}}) && + d == 0x2000); + // over a whole range exactly, and an empty binding + CHECK(kept(0x5000, 0x6000, ex, &d).empty() && d == 0x1000); + CHECK(kept(0x5000, 0x5000, ex, &d).empty() && d == 0); + // what was kept and what was left out always make the binding + for (uint64_t lo = 0x1800; lo < 0x6800; lo += 0x1c0) + for (uint64_t n = 0x40; n < 0x3000; n += 0x3c0) { + uint64_t sum = 0, at = lo; + bool inside = true; + for (const auto& [a, b] : kept(lo, lo + n, ex, &d)) { + inside = inside && a >= at && b > a && b <= lo + n; + for (const auto& [ea, eb] : ex) inside = inside && (b <= ea || a >= eb); + sum += b - a, at = b; + } + CHECK(inside && sum + d == n); + } + + // the switch: on unless "0", and only while the capture leaves memory out (RX_VK_CAP_ONLY_GPU=1) + CHECK(wb_excl_clip::on(nullptr, 1) && wb_excl_clip::on("", 1) && wb_excl_clip::on("1", 1)); + CHECK(!wb_excl_clip::on("0", 1)); + CHECK(!wb_excl_clip::on(nullptr, 0) && !wb_excl_clip::on(nullptr, 2) && !wb_excl_clip::on("1", 0)); + + // The crash, on a page: a pool at 0x10000, the heap's 12-byte block at +0x18 (its header at +0x10) written by the CPU + // after the mirror was made; the page is left out, so the mirror still holds what was there before (the counts the logs + // show: 0x9ed00000001). A bare entry's head [base, base + 0x40) is the written binding + const uint64_t base = 0x10000; + uint8_t guest[0x1000], mirror[0x1000]; + std::memset(guest, 0, sizeof guest); + std::memset(mirror, 0, sizeof mirror); + const uint64_t header = 0x0000000014a40005ull, stale = 0x000009ed00000001ull; + std::memcpy(guest + 0x10, &header, 8); // (the CPU's block header, guest memory only) + std::memcpy(mirror + 0x10, &stale, 8); // (the mirror's stale copy) + const std::vector left_out = {{base, base + 0x1000}}; + uint8_t before[0x1000]; + std::memcpy(before, guest, sizeof guest); + // as 0.6.0 did: the whole head written back, and the header is the stale value (0x803b44e0 then reads 0x9ed00000012) + write_back(guest, mirror, base, {{base, base + 0x40}}); + uint64_t got = 0; + std::memcpy(&got, guest + 0x10, 8); + CHECK(got == stale); + // clipped: guest memory as the CPU left it + std::memcpy(guest, before, sizeof guest); + write_back(guest, mirror, base, kept(base, base + 0x40, left_out)); + std::memcpy(&got, guest + 0x10, 8); + CHECK(got == header && std::memcmp(guest, before, sizeof guest) == 0); + // a pool's own page is still written back (the GPU's results the CPU reads) + const uint64_t result = 0x1234; + std::memcpy(mirror + 0x20, &result, 8); + write_back(guest, mirror, base, kept(base, base + 0x40, none)); + std::memcpy(&got, guest + 0x20, 8); + CHECK(got == result); + + if (fails) { + std::fprintf(stderr, "%d checks failed\n", fails); + return 1; + } + std::puts("wb_excl_clip: all checks passed"); + return 0; +} diff --git a/tests/test_wb_excl_clip.py b/tests/test_wb_excl_clip.py new file mode 100644 index 0000000..b7182d5 --- /dev/null +++ b/tests/test_wb_excl_clip.py @@ -0,0 +1,46 @@ +"""Compile and run the rule for which parts of a written binding go back to guest memory while the capture leaves memory out +(gpu/vk/wb_excl_clip.h, RX_VK_WB_EXCL_CLIP; no game/GPU), and check that gpu_buffer_bind still uses it.""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def check_source(): + """gpu_buffer_bind clips its write-backs through the rule, behind the switch, and the docs name the switch""" + src = (ROOT / "gpu/vk/vk_memory.cpp").read_text(encoding="utf-8") + for needle in ('#include "wb_excl_clip.h"', 'wb_excl_clip::on(getenv("RX_VK_WB_EXCL_CLIP"), cap_only_gpu())', + "return wb_excl_clip::kept(lo, hi, mine, out);", + "if (excl_clip && cap_excl_hit(p->lo + off, p->lo + wb_end)) {", + "for (const auto& [a, b] : parts) writebacks.push_back({p, a - p->lo, b - p->lo, tl_event_write});"): + if needle not in src: + raise SystemExit("vk_memory.cpp no longer has: " + needle) + bind = src[src.index("BufferRef gpu_buffer_bind(uint64_t gpu, uint64_t size, bool written) {"):] + bind = bind[:bind.index("\n}\n")] + # (the one write-back of a written binding: the clipped parts, or the whole range when nothing of it is left out) + if bind.count("writebacks.push_back(") != 2: + raise SystemExit("gpu_buffer_bind queues a write-back outside the clip") + # (the collection still leaves the same memory unstamped: the reason the mirror's copy there is stale) + if "if (only_gpu && cap_excl_page(guest)) continue;" not in src: + raise SystemExit("memory_collect_written's left-out pages changed: recheck wb_excl_clip.h") + docs = (ROOT / "docs/development.md").read_text(encoding="utf-8") + if "RX_VK_WB_EXCL_CLIP=0" not in docs: + raise SystemExit("docs/development.md does not name RX_VK_WB_EXCL_CLIP") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="g++") + args = parser.parse_args() + check_source() + with tempfile.TemporaryDirectory(prefix="wb-excl-clip-") as tmp: + exe = Path(tmp) / "wb-excl-clip" + subprocess.run([args.cxx, "-std=c++20", "-Wall", "-Wextra", "-Werror", + str(ROOT / "tests/test_wb_excl_clip.cpp"), "-o", str(exe)], check=True) + subprocess.run([str(exe)], check=True) + + +if __name__ == "__main__": + main() diff --git a/tests/test_wb_forget_order.py b/tests/test_wb_forget_order.py new file mode 100644 index 0000000..8b22ba2 --- /dev/null +++ b/tests/test_wb_forget_order.py @@ -0,0 +1,152 @@ +"""The write-back diff forgets the written pages without sorting their lists, to the same result (gpu/vk/vk_memory.cpp +shadows_forget_written; no game, no GPU). + +(Triforce, 11-21, the performance audit's MEM-61, after Source A's.) At the end of each part with write-backs the diff +clears, for every page written since, the bit that says its pool's shadow knows the page. Its three lists were sorted +first, only so that the pool found for one page serves the next. + +This cuts shadows_forget_written out of the source as it is and runs it twice over the same lists, unsorted (as it is +now) and sorted first (RX_VK_WB_FORGET_SORT=1's arm, set in the copy): two pools with a shadow and one without, lists in +no order with repeats, pages in no pool and pages in two lists. The shadows' bits and the three counts come out the +same. + +And reads the source: the sort is made only under the switch; the call is timed for the 600-submit line; the switch is +in the switches table. + + python -I -B tests/test_wb_forget_order.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +FIXTURE = r""" +#include +#include +#include +#include +#include +#include +#include +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +constexpr uint64_t kPageShift = 12; +struct Pool { + uint64_t lo = 0, hi = 0; + bool shadow = true; + std::vector shadow_known; +}; +std::map imported; +uint64_t shadow_bytes = 1; +std::mutex shadow_written_mu; +std::vector shadow_written_pages, shadow_written_collected_list, shadow_written_by_replay; +bool shadow_written_all = false; +int64_t st_wbd_known_game = 0, st_wbd_known_collected = 0, st_wbd_known_replay = 0; +namespace unsorted { +@UNSORTED@ +} +namespace sorted_first { +@SORTED@ +} +static uint32_t rnd_state = 99; +static uint32_t rnd() { return rnd_state = rnd_state * 1664525u + 1013904223u; } +static void pools() { // (two pools with a shadow, 200 and 300 pages, every page known; one of 100 pages without) + imported.clear(); + Pool a, b, c; + a.lo = 0x10000000, a.hi = a.lo + 200 * 4096, a.shadow_known.assign(4, ~0ull); + b.lo = 0x20000000, b.hi = b.lo + 300 * 4096, b.shadow_known.assign(5, ~0ull); + c.lo = 0x30000000, c.hi = c.lo + 100 * 4096, c.shadow = false, c.shadow_known.assign(2, ~0ull); + imported[a.lo] = a, imported[b.lo] = b, imported[c.lo] = c; + st_wbd_known_game = st_wbd_known_collected = st_wbd_known_replay = 0; +} +int main() { + for (int round = 0; round < 50; round++) { + std::vector lists[3]; + for (auto& l : lists) { + const int n = 20 + rnd() % 400; + for (int i = 0; i < n; i++) { + const uint32_t which = rnd() % 5; // (the two shadowed pools, the one without, below every pool, between two) + const uint32_t page = which == 0 ? 0x10000 + rnd() % 200 : which == 1 ? 0x20000 + rnd() % 300 + : which == 2 ? 0x30000 + rnd() % 100 : which == 3 ? rnd() % 0x10000 : 0x10000 + 200 + rnd() % 1000; + l.push_back(page); + if (rnd() % 4 == 0) l.push_back(page); // (a repeat) + if (rnd() % 6 == 0) lists[rnd() % 3].push_back(page); // (and in another list) + } + } + std::map> bits[2]; + int64_t counts[2][3]; + for (int arm = 0; arm < 2; arm++) { + pools(); + shadow_written_pages = lists[0], shadow_written_collected_list = lists[1], shadow_written_by_replay = lists[2]; + if (arm == 0) unsorted::shadows_forget_written(); + else sorted_first::shadows_forget_written(); + for (const auto& [lo, p] : imported) bits[arm][lo] = p.shadow_known; + counts[arm][0] = st_wbd_known_game, counts[arm][1] = st_wbd_known_collected, counts[arm][2] = st_wbd_known_replay; + CHECK(shadow_written_pages.empty() && shadow_written_collected_list.empty() && shadow_written_by_replay.empty()); + } + CHECK(bits[0] == bits[1]); + CHECK(counts[0][0] == counts[1][0] && counts[0][1] == counts[1][1] && counts[0][2] == counts[1][2]); + CHECK(counts[0][0] + counts[0][1] + counts[0][2] > 0); // (the lists did reach known pages) + CHECK(bits[0][0x30000000] == std::vector(2, ~0ull)); // (a pool without a shadow is left alone) + } + return fails ? 1 : 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + mem = (ROOT / "gpu" / "vk" / "vk_memory.cpp").read_text(encoding="utf-8").replace("\r\n", "\n") + fn = block(mem, "void shadows_forget_written() {") + switch = 'static const bool forget_sort = getenv("RX_VK_WB_FORGET_SORT") && atoi(getenv("RX_VK_WB_FORGET_SORT")) != 0;' + check(switch in fn, "shadows_forget_written has no RX_VK_WB_FORGET_SORT switch") + check(fn.count("std::sort(") == 1 and "if (forget_sort) std::sort(pages.begin(), pages.end());" in fn, + "shadows_forget_written sorts its lists whatever the switch says: thousands of pages a part, three times, under pools_mu") + end = block(mem, "void memory_end_submission(VkCommandBuffer cmd) {") + at = 0 + for part in ("const auto f0 = std::chrono::steady_clock::now();", "shadows_forget_written();", "st_wbd_forget_ns +=", + "st_wbd_forget_calls++;", "shadows_reap(), shadows_check_room();"): + at = end.find(part, at) + check(at >= 0, "memory_end_submission does not time the forgetting, or no longer reaps and checks the shadows after it " + "(missing, or out of order: " + part + ")") + at += len(part) + check("forgetting the written pages took %.2f ms in %lld calls" in end, "the forgetting's time is not on the 600-submit line") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`RX_VK_WB_FORGET_SORT=1`" in dev, "docs/development.md: RX_VK_WB_FORGET_SORT is not in the switches table") + + # (the arm that sorts: the same function with its switch read as on) + sorted_fn = fn.replace(switch, "static const bool forget_sort = true;") + check(sorted_fn != fn, "the sorted arm could not be made of the function") + with tempfile.TemporaryDirectory(prefix="wb-forget-") as tmp: + c, exe = Path(tmp) / "forget.cpp", Path(tmp) / "forget.exe" + c.write_text(FIXTURE.replace("@UNSORTED@", fn).replace("@SORTED@", sorted_fn), encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O1", "-Wall", "-Wextra", "-Werror", str(c), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True, env={"PATH": __import__("os").environ.get("PATH", ""), + "SYSTEMROOT": __import__("os").environ.get("SYSTEMROOT", "")}) + check(r.returncode == 0, "unsorted against sorted:\n" + r.stdout) + print("PASS: the write-back diff forgets the written pages unsorted, to the same bits and counts") + + +if __name__ == "__main__": + main() diff --git a/tests/test_wb_lazy_state.cpp b/tests/test_wb_lazy_state.cpp index 3b5d423..ae66cdb 100644 --- a/tests/test_wb_lazy_state.cpp +++ b/tests/test_wb_lazy_state.cpp @@ -547,9 +547,56 @@ void choice() { } // namespace +// ((Triforce, 11-21, a player's report on 0.6.0) The fault handler's question, on_fault: (1) a fault that arrives after another +// thread's payment gave the page back (untracked by now, readable and writable) runs again, where tracked() alone sent it to the +// crash handler; untracked and without access is still not ours. (2) start_watch counts pages before it protects them: the model +// fails any protection of a page tracked() does not know yet +struct OrderModel : Model { + Table* t = nullptr; + bool set(uint64_t lo, uint64_t hi, Access a) { + if (a != RW && t) + for (uint64_t p = lo; p < hi; p++) + if (!t->tracked(p)) { fail("a page protected before the table tracks it", p - kBase, a); break; } + return Model::set(lo, hi, a); + } +}; +void late_fault() { + Model m; + Table t(m); + Epoch e; + Skip skip; + std::vector jobs; + const uint64_t now = 100000, a = kBase + 32, b = kBase + 96; + e.collects_begun = 1; + t.advance(a, b, kSrc, 1, e, now, true, skip); + e.collects = 2, e.collects_begun = 2; + t.advance(a, b, kSrc, 2, e, now, true, skip); + expect(t.state_of(a) == OWED && !m.accessible(a, false), "late fault: the run is owed, its pages without access"); + // a fault there now: this thread pays (as fault() did) + expect(t.on_fault(a + 5, true, 255, now, jobs) == PAY && !jobs.empty(), "late fault: a fault on an owed page pays (the whole run: grow 255)"); + t.end_pay(jobs, BY_FAULT, now); + jobs.clear(); + // another thread faulted on a page of that payment before end_pay gave the access back; its handler runs only now + expect(!t.tracked(a + 6), "late fault: the paid page is untracked by now"); + expect(t.on_fault(a + 6, true, 7, now, jobs) == RETRY && jobs.empty(), "late fault: untracked but writable again: run the access again"); + expect(t.late_retries.load() == 1, "late fault: counted"); + // an untracked page that still has no access is not this table's (the crash handler's, as before) + m.set(kBase + 200, kBase + 201, NONE); + expect(t.on_fault(kBase + 200, false, 7, now, jobs) == NOT_OURS, "late fault: untracked and without access: not ours"); + m.set(kBase + 200, kBase + 201, RW); + // (2) the order in start_watch + OrderModel om; + Table ot(om); + om.t = &ot; + Epoch e2; + e2.collects_begun = 1; + ot.advance(a, b, kSrc, 1, e2, now, true, skip); + expect(ot.state_of(a) == WATCH && ot.tracked(a), "late fault: watched, tracked"); +} + int main(int argc, char** argv) { if (argc > 1 && !std::strcmp(argv[1], "stress")) stress(argc > 2 ? std::atof(argv[2]) : 2.0, argc > 3 ? std::atoi(argv[3]) : kMaxPayers); - else rules(), choice(), payer_slots(); + else rules(), choice(), payer_slots(), late_fault(); if (failures) return std::printf("%d check(s) failed\n", failures), 1; std::printf("PASS: %s\n", argc > 1 ? argv[1] : "rules"); return 0; diff --git a/tests/test_wb_rec_wait.cpp b/tests/test_wb_rec_wait.cpp index 89e7b35..a1b0728 100644 --- a/tests/test_wb_rec_wait.cpp +++ b/tests/test_wb_rec_wait.cpp @@ -90,6 +90,116 @@ int main() { std::strstr(b, "; 1 waits for write-backs still on the GPU 39.4 ms (up to 4 submissions ahead of it, 1 of them 37.9 ms after " "their writer's recording was done (the timeline held by work stopped at a wait))") != nullptr); + // ---- (Triforce, 11-22, the performance audit's SYNC-122) the reads' side ---- + // the mode: 1 on, 2 the check, anything else off + CHECK("mode unset", wr::mode(nullptr) == wr::kOff); + CHECK("mode empty", wr::mode("") == wr::kOff); + CHECK("mode 0", wr::mode("0") == wr::kOff); + CHECK("mode 1", wr::mode("1") == wr::kOn); + CHECK("mode 2", wr::mode("2") == wr::kCheck); + CHECK("mode 3", wr::mode("3") == wr::kOff); + CHECK("mode 12", wr::mode("12") == wr::kOff); + CHECK("mode a word", wr::mode("yes") == wr::kOff); + CHECK("mode 1 with a tail", wr::mode("1x") == wr::kOff); + // RX_VK_TL_REC: 2 the old lookup; on otherwise it ends the wait as mode 1 does + CHECK("tl unset", !wr::tl_rec_first_part(nullptr)); + CHECK("tl 1", !wr::tl_rec_first_part("1")); + CHECK("tl 2", wr::tl_rec_first_part("2")); + CHECK("ends: all off", !wr::ends_by_switch(wr::kOff, false, false)); + CHECK("ends: the check never ends a wait", !wr::ends_by_switch(wr::kCheck, false, false)); + CHECK("ends: mode 1", wr::ends_by_switch(wr::kOn, false, false)); + CHECK("ends: RX_VK_TL_REC", wr::ends_by_switch(wr::kOff, true, false)); + CHECK("ends: RX_VK_TL_REC=2 has its own wait", !wr::ends_by_switch(wr::kOff, true, true)); + CHECK("ends: the check with RX_VK_TL_REC", wr::ends_by_switch(wr::kCheck, true, false)); + CHECK("callers", !std::strcmp(wr::caller_name(wr::kTake), "takes") && !std::strcmp(wr::caller_name(wr::kRead), "reads") && + !std::strcmp(wr::caller_name(wr::kBind), "binds") && !std::strcmp(wr::caller_name(wr::kReadback), "readbacks") && + wr::kCallers == 4); + CHECK("sliced only with a mode and a recordings' timeline", + !wr::fallback_sliced(wr::kOff, true) && !wr::fallback_sliced(wr::kOn, false) && !wr::fallback_sliced(wr::kCheck, false) && + wr::fallback_sliced(wr::kOn, true) && wr::fallback_sliced(wr::kCheck, true)); + + // the fallback wait against a made-up GPU: the writer's recording is known at K, done at R, and the submissions' timeline + // reaches the value at T (K <= R <= T: known at its submit, and the timeline cannot pass the recording that finished it) + struct Gpu { + int64_t K, R, T, t = 0; + int plains = 0, flushes = 0, lookups = 0, rec_waits = 0, slices = 0; + bool fail_slice = false; + void plain() { plains++, t = t > T ? t : T; } + void flush() { flushes++; } + uint64_t lookup() { lookups++; return t >= K ? 77 : 0; } + int rec_wait(uint64_t seq) { + if (seq != 77) return 2; + rec_waits++, t = t > R ? t : R; + return 0; + } + bool rec_done(uint64_t seq) { return seq == 77 && t >= R; } + int slice() { + slices++; + if (fail_slice) return 2; + if (T <= t + (int64_t)wr::kSliceNs) { + t = t > T ? t : T; + return 0; + } + t += (int64_t)wr::kSliceNs; + return 1; + } + int64_t now() { return t; } + }; + uint64_t rng = 0x9E3779B97F4A7C15ull; + const auto rnd = [&](uint64_t n) { + rng ^= rng << 13, rng ^= rng >> 7, rng ^= rng << 17; + return (int64_t)(rng % n); + }; + for (int i = 0; i < 20000; i++) { + const int64_t K = rnd(40000000), R = K + rnd(20000000), T = R + rnd(i % 3 ? 60000000 : 50000); + Gpu off{K, R, T}; + const wr::FallbackResult a = wr::fallback_wait(wr::kOff, true, off); + CHECK("off: the one old wait and nothing else", + off.plains == 1 && !off.flushes && !off.lookups && !off.rec_waits && !off.slices && !a.sliced && a.total_ns == T && + wr::fallback_held_ns(a) == 0); + Gpu none{K, R, T}; + const wr::FallbackResult n1 = wr::fallback_wait(wr::kOn, false, none); + CHECK("no recordings' timeline: the old wait in any mode", none.plains == 1 && !none.slices && !n1.sliced && n1.total_ns == T); + Gpu chk{K, R, T}; + const wr::FallbackResult c = wr::fallback_wait(wr::kCheck, true, chk); + CHECK("the check ends where the old wait ends", c.total_ns == a.total_ns && chk.t == off.t); + CHECK("the check makes neither the old wait nor the recording's", !chk.plains && !chk.rec_waits && c.sliced && !c.at_recording && !c.error); + CHECK("the check flushes before each look", chk.flushes == chk.slices); + const int64_t h = wr::fallback_held_ns(c); + CHECK("the check's held part is never more than the part after the recording", h <= T - R); + // (seen done at the first look at or after R: a slice late at most; under the latency's floor it is not named) + if (T - R >= (int64_t)wr::kSliceNs + wr::kHeldMinNs) CHECK("the check's held part, to a slice", h >= T - R - (int64_t)wr::kSliceNs); + if (T - R < wr::kHeldMinNs) CHECK("no held part under the latency", h == 0); + Gpu on{K, R, T}; + const wr::FallbackResult o = wr::fallback_wait(wr::kOn, true, on); + CHECK("on: never the old wait", !on.plains && o.sliced && !o.error); + CHECK("on: never past where the old wait ends", o.total_ns <= T); + if (o.at_recording) { + CHECK("on: not before the writer's recording is done", o.total_ns >= R && on.rec_waits == 1); + CHECK("on: at the recording, or a slice after it was known", o.total_ns <= (R > K + (int64_t)wr::kSliceNs ? R : K + (int64_t)wr::kSliceNs)); + CHECK("on: nothing held", wr::fallback_held_ns(o) == 0); + } else { + CHECK("on: ended by the timeline only when it got there first", o.total_ns == T && !on.rec_waits); + } + } + { + Gpu bad{1000000, 2000000, 900000000}; + bad.fail_slice = true; + const wr::FallbackResult e = wr::fallback_wait(wr::kCheck, true, bad); + CHECK("an error ends the wait", e.error && bad.slices == 1 && wr::fallback_held_ns(e) == 0); + } + + // the stall line names the held part by caller, only when there is one + sw::Frame g; + g.no = 1, g.interval = 60, g.replay = 50; + sw::stall_line(g, b, sizeof b); + CHECK("no held part by caller: not named", !std::strstr(b, "by caller")); + g.wb_held_by[wr::kRead] = 12.5, g.wb_held_by[wr::kReadback] = 3.5; + sw::stall_line(g, b, sizeof b); + CHECK("held part by caller named", + std::strstr(b, "; the timeline held after a writer's recording was done, by caller: takes 0.0, reads 12.5, binds 0.0, " + "readbacks 3.5 ms") != nullptr); + std::puts("PASS: write-back waits by the writer's recording, and the held part"); return 0; } diff --git a/tests/test_wb_rec_wait.py b/tests/test_wb_rec_wait.py index 7567649..e6fe36a 100644 --- a/tests/test_wb_rec_wait.py +++ b/tests/test_wb_rec_wait.py @@ -16,6 +16,11 @@ words), and reads the source for what needs the game: - vk_timeline_wait looks the value up under rec_map_mu, waits for the recording on rec_timeline first, waits for the timeline unless the switch ends the wait there, with what is left of the one bound, and counts the held part for the stall line; - status_tick gives the held part to the stall line, and the switch is in the development guide. +- (Triforce, 11-22, the performance audit's SYNC-122) the reads' side: the switch is a mode (2 the check); a readback whose + write-back is not recorded yet is counted, and under the mode waits in slices that look the finished table up (the header's + fallback_wait, run here against a made-up GPU on 20,000 random cases: the check ends exactly where the old wait ends, mode 1 + never before the writer's recording is done and never after the old wait); RX_VK_TL_REC ends a wait at the recording that + finished the value, its old first-part lookup kept to =2; the held part is counted by caller (take, read, bind, readback). python -I -B tests/test_wb_rec_wait.py [--cxx g++] """ @@ -57,8 +62,12 @@ def check_source(): par = replay[replay.index("memory_sync_mirrors_before_segments();"):] check(par[:par.index("\n", par.index("submit_recording("))].count("submit_recording(r, false, 0, true);") == 1, "a parallel replay's setup submit is not a part's") - check(sum(", 0, true)" in line for line in replay.splitlines() if "submit_recording(" in line) == 3, - "a submit other than the three parts' is a part's") + # ((Triforce, 11-22) four since v0.6.0: the parallel replay's early segments (RX_VK_SEG_EARLY) submit what came before them + # and the dry pass's uploads as a part too, ahead of every segment; this check still said three and failed on v0.6.0 itself) + check("submit_recording(r, false, 0, true); // (a part: what came before and the dry pass's uploads, ahead of every segment)" in replay, + "the early segments' setup submit is not a part's") + check(sum(", 0, true)" in line for line in replay.splitlines() if "submit_recording(" in line) == 4, + "a submit other than the four parts' is a part's") finish = function(replay, "void finish_recording(uint64_t value) {") check("submit_recording(r, signal, at);" in finish, "finish_recording's submit does not end the recording") # (Triforce-dev: a present that leaves work stopped (RX_VK_PRESENT_DRAIN=2) names its timeline signal as finish_recording @@ -71,7 +80,12 @@ def check_source(): ("std::lock_guard mk(rec_map_mu);", "the table is read without its lock"), ("seq = fin_rec.of(v);", "the writer's recording is not looked up"), ("ri.pSemaphores = &rec_timeline;", "the recording is not waited for on the recordings' timeline"), - ("if (!wr::ends_at_recording(wb_wait_rec_on(), seq != 0, rec_done)) {", "the switch does not decide the timeline's wait"), + ("if (!wr::ends_at_recording(ends_here, seq != 0, rec_done)) {", "the switch does not decide the timeline's wait"), + # ((Triforce, 11-22, the performance audit's SYNC-122) the mode, RX_VK_TL_REC through the finished table, by caller + ("const bool ends_here = wr::ends_by_switch(wb_wait_rec_mode(), rb_fix(1), tl_rec_first_part_on());", + "RX_VK_TL_REC does not end the wait at the recording that finished the value"), + ("if (rb_fix(1) && tl_rec_first_part_on()) {", "RX_VK_TL_REC's old lookup (a first part) is not kept to RX_VK_TL_REC=2"), + ("stall_ct.wb_held_by_ns[wb_wait_who].fetch_add(held, std::memory_order_relaxed);", "the held part is not counted by caller"), ("wr::budget_left(wr::kBoundNs, rec_ns)", "the timeline's wait does not keep to what is left of the bound"), ("wr::held_ns(seq != 0, rec_done, rec_ns, waited_ns)", "the held part is not counted"), ("stall_ct.wb_held_ns.fetch_add(", "the held part does not reach the stall line's counts"), @@ -85,8 +99,10 @@ def check_source(): "vk_timeline_wait waits for the recording before the held submits are flushed") check("wb_held_last_ns = 0;" in wait[:wait.index("if (!timeline) return;")], "the thread's last held part is not reset at a wait's start") - on = function(replay, "static bool wb_wait_rec_on() {") - check('::mkwvk::wb_rec_wait::on(getenv("RX_VK_WB_WAIT_REC"))' in on, "RX_VK_WB_WAIT_REC is not read") + on = function(replay, "static int wb_wait_rec_mode() {") + check('const char* e = getenv("RX_VK_WB_WAIT_REC");' in on and "const int m = wr::mode(e);" in on, "RX_VK_WB_WAIT_REC is not read as a mode") + check("wb_wait_rec_on" not in replay, "the switch is read as on / off somewhere: 2 would be on") + check_reads(replay) tick = function(replay, "void status_tick() {") for needle in ("stall_ct.wb_held.load(", "stall_ct.wb_held_ns.load(", @@ -98,6 +114,46 @@ def check_source(): check("`RX_VK_WB_WAIT_REC=1`" in dev, "docs/development.md: RX_VK_WB_WAIT_REC is not in the switches table") +def check_reads(replay): + """(Triforce, 11-22, the performance audit's SYNC-122) the reads' side: the readback's fallback, the callers, RX_VK_TL_REC""" + fb = function(replay, "void readback_fallback_wait(uint64_t v, bool by_recording) {") + for needle, why in ( + ("void plain() { wait_value(v); }", "off, the fallback is not the wait it was"), + ("return fin_rec.of(v);", "the slices do not look the finished table up"), + ("std::lock_guard mk(rec_map_mu);", "the table is read without its lock"), + ("const VkResult res = vkWaitSemaphores(dev.device, &wi, wr::kSliceNs);", "the slices are not 1 ms waits on the timeline"), + ("return vkWaitSemaphores(dev.device, &ri, UINT64_MAX) == VK_SUCCESS ? 0 : 2;", "the recording's wait has a bound"), + ("wr::fallback_wait(wb_wait_rec_mode(), by_recording && rec_timeline != VK_NULL_HANDLE, calls);", + "the fallback is not the header's, by the switch's mode"), + ("stall_ct.wb_held_by_ns[wr::kReadback].fetch_add(held, std::memory_order_relaxed);", "its held part does not reach the stall line"), + ("st_rb_fb.fetch_add(1, std::memory_order_relaxed)", "the fallbacks are not counted")): + check(needle in fb, "vk_replay.cpp readback_fallback_wait: " + why) + inv = function(replay, "void invalidate(uint64_t gpu, uint64_t size) {") + check(inv.count("readback_fallback_wait(v, unforced);") == 1, "invalidate's fallback is not the counted one") + # (the replay thread's own path keeps its waits; only the last else, the submitting thread's, is the fallback) + at = inv.index("readback_fallback_wait(v, unforced);") + check(inv.rindex("if (async_unforced()) {", 0, at) < inv.rindex("} else {", 0, at) and inv.count("wait_value(v);") == 1, + "invalidate: the fallback is not where the unbounded wait on the timeline was") + check("st_rb_fb.exchange(0)" in inv and "(counted with RX_VK_WB_WAIT_REC=2: what 1 would not have waited)" in inv, + "the 3000-readback line does not say the fallbacks") + check("void vk_timeline_wait_from(uint64_t v, int who) {" in replay, "vk_timeline_wait_from is missing") + tick = function(replay, "void status_tick() {") + for needle in ("wb_by[i] = stall_ct.wb_held_by_ns[i].load(std::memory_order_relaxed);", + "f.wb_held_by[i] = (wb_by[i] - s_wb_by[i]) / 1e6;", "s_wb_by[i] = wb_by[i];"): + check(needle in tick, "vk_replay.cpp status_tick: the held part by caller is not this frame's (%s)" % needle) + memory = (ROOT / "gpu/vk/vk_memory.cpp").read_text(encoding="utf-8") + read = function(memory, "void late_unresolved_read(uint64_t page0, uint64_t page1) {") + check("vk_timeline_wait_from(v, 1);" in read and "vk_timeline_wait(v)" not in read, "a read of a page put off does not wait as a read") + bind = memory[memory.index("deferred_late_n.store((int)deferred_late.size(), std::memory_order_relaxed);\n }\n for (uint32_t page : hit) {"):] + bind = bind[:bind.index("\n}\n")] + check("if (v > done0) vk_timeline_wait_from(v, 2);" in bind, "a binding of a page put off does not wait as a bind") + head = (ROOT / "gpu/vk/wb_rec_wait.h").read_text(encoding="utf-8") + check("kTake = 0, kRead = 1, kBind = 2, kReadback = 3, kCallers = 4" in head, "the callers' numbers moved: vk_memory.cpp passes 1 and 2") + dev = (ROOT / "docs/development.md").read_text(encoding="utf-8") + check("`RX_VK_WB_WAIT_REC=1`, `=2`" in dev and "`RX_VK_TL_REC=2` keeps the old lookup" in dev, + "docs/development.md: the mode and RX_VK_TL_REC are not in the switches table") + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--cxx", default="g++") diff --git a/tests/test_work_lock.py b/tests/test_work_lock.py new file mode 100644 index 0000000..2388a42 --- /dev/null +++ b/tests/test_work_lock.py @@ -0,0 +1,232 @@ +"""One build at a time per work folder: make_game.py's OS lock on /make_game.lock (no game, no tools). + +(Triforce, 11-22, the performance audit's IO-47, after Source A's.) Two builds on one work folder compiled the same +objects twice and raced on the staged game folder; on Linux nothing stopped the second. make_game.py now holds an OS +lock for as long as it runs, taken after stop_stray_compilers (so Windows still replaces a build left running) and +before drop_stale_objects. HOOKSHOT_WORK_LOCK=0 turns it off. + +With real processes on temporary folders: + - while another process holds a folder's lock, lock_work stops the build (exit 1, an @@error naming the folder and the + holder's process id), at once; + - a holder that is killed frees the lock: the next lock_work, told to wait as main does after a stop, takes it; + - asked again for the same folder it keeps the same open file; asked for another folder it lets the first go; + - release_work frees the lock and leaves the file; + - HOOKSHOT_WORK_LOCK=0 takes no lock and refuses nothing; + - an error that is not "held by another" is said and the build goes on without the lock; + - main() asks stop_stray_compilers, then lock_work (waiting 10 s only when something was stopped), then + drop_stale_objects; the source and the switches table say the same. + + python -I -B tests/test_work_lock.py +""" +import contextlib +import errno +import importlib.util +import io +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import time +import unittest +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +SOURCE = ROOT / "nx2" / "make_game.py" +spec = importlib.util.spec_from_file_location("make_game_work_lock", SOURCE) +make_game = importlib.util.module_from_spec(spec) +spec.loader.exec_module(make_game) + +# (another build: takes the folder's lock as make_game does, says so, and holds it until its input ends) +HOLDER = r''' +import importlib.util, sys +spec = importlib.util.spec_from_file_location("make_game_lock_holder", sys.argv[1]) +m = importlib.util.module_from_spec(spec) +spec.loader.exec_module(m) +m.lock_work(sys.argv[2]) +print("held", flush=True) +sys.stdin.read() +''' + + +class Stop(Exception): + """main() stopped by the test at drop_stale_objects""" + + +class WorkLock(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="triforce-work-lock-") + self.addCleanup(self.temporary.cleanup) + self.addCleanup(make_game.release_work) # (before the folder goes: Windows deletes no open file) + self.root = Path(self.temporary.name).resolve() + self.work = self.root / "work" + self.work.mkdir() + self.holders = [] + self.addCleanup(self.stop_holders) + self.env = {k: v for k, v in os.environ.items() if k != "HOOKSHOT_WORK_LOCK"} + patched = mock.patch.dict(os.environ, self.env, clear=True) + patched.start() + self.addCleanup(patched.stop) + + def stop_holders(self): + for p in self.holders: + if p.poll() is None: + p.kill() + p.communicate() + + def holder(self, folder=None, hold=True): + """Another process that locks the folder: (the process, whether it got the lock, what it said)""" + p = subprocess.Popen([sys.executable, "-I", "-B", "-c", HOLDER, str(SOURCE), str(folder or self.work)], + stdin=subprocess.PIPE if hold else subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, env=self.env) + self.holders.append(p) + if not hold: + out, err = p.communicate(timeout=60) + return p, p.returncode == 0 and out.strip() == "held", out + err + line = p.stdout.readline() + return p, line.strip() == "held", line + + def lock(self, folder=None, **kw): + """lock_work here: (what it returned or the exit code it stopped with, what it printed, the seconds it took)""" + said = io.StringIO() + began = time.monotonic() + try: + with contextlib.redirect_stdout(said): + result = make_game.lock_work(str(folder or self.work), **kw) + except SystemExit as e: + result = "exit %s" % e.code + return result, said.getvalue(), time.monotonic() - began + + def test_a_held_folder_stops_the_second_build_and_names_the_holder(self): + p, held, said = self.holder() + self.assertTrue(held, said) + result, text, seconds = self.lock() + self.assertEqual(result, "exit 1") + self.assertIn("@@error Another build is already working in %s (process %d)." % (self.work, p.pid), text) + self.assertLess(seconds, 2.0) + self.assertIsNone(make_game._work_lock) + self.assertEqual(make_game.work_lock_holder(str(self.work / make_game.WORK_LOCK)), p.pid) + # (and the holder still has it: a refused build took nothing away) + self.assertFalse(self.holder(hold=False)[1]) + + def test_a_killed_holder_frees_the_lock(self): + p, held, said = self.holder() + self.assertTrue(held, said) + p.kill() + p.wait() + result, text, seconds = self.lock(wait=10) # (as main does after stop_stray_compilers stopped a build) + self.assertIs(result, True, text) + self.assertLess(seconds, 10.0) + self.assertEqual(make_game.work_lock_holder(str(self.work / make_game.WORK_LOCK)), os.getpid()) + + def test_wait_is_bounded_while_the_holder_lives(self): + p, held, said = self.holder() + self.assertTrue(held, said) + result, text, seconds = self.lock(wait=1.0) + self.assertEqual(result, "exit 1") + self.assertGreaterEqual(seconds, 1.0) + self.assertLess(seconds, 5.0) + + def test_again_for_the_same_folder_keeps_the_file(self): + self.assertIs(self.lock()[0], True) + first = make_game._work_lock[0] + self.assertIs(self.lock(self.work / "." )[0], True) # (the same folder, spelled another way) + self.assertIs(make_game._work_lock[0], first) + self.assertFalse(first.closed) + self.assertFalse(self.holder(hold=False)[1]) # (another process is refused) + self.assertTrue((self.work / make_game.WORK_LOCK).is_file()) + + def test_another_folder_lets_the_first_go(self): + other = self.root / "other" + other.mkdir() + self.assertIs(self.lock()[0], True) + self.assertIs(self.lock(other)[0], True) + self.assertEqual(os.path.normcase(make_game._work_lock[1]), os.path.normcase(str(other))) + p, held, said = self.holder(self.work, hold=False) + self.assertTrue(held, said) # (the first folder is free) + p, held, said = self.holder(other, hold=False) + self.assertFalse(held, said) # (the second is this process's) + self.assertIn("@@error Another build is already working in", said) + self.assertEqual(p.returncode, 1) + + def test_release_frees_the_lock_and_leaves_the_file(self): + self.assertIs(self.lock()[0], True) + make_game.release_work() + self.assertIsNone(make_game._work_lock) + self.assertTrue((self.work / make_game.WORK_LOCK).is_file()) + self.assertTrue(self.holder(hold=False)[1]) + make_game.release_work() # (nothing held: nothing happens) + + def test_switch_off_takes_no_lock(self): + p, held, said = self.holder() + self.assertTrue(held, said) + with mock.patch.dict(os.environ, {"HOOKSHOT_WORK_LOCK": "0"}): + result, text, seconds = self.lock() + self.assertIs(result, False) + self.assertEqual(text, "") + self.assertIsNone(make_game._work_lock) + with mock.patch.dict(os.environ, {"HOOKSHOT_WORK_LOCK": "1"}): + self.assertEqual(self.lock()[0], "exit 1") + + def test_another_error_builds_without_the_lock(self): + for error in (OSError(errno.ENOSYS, "no locks on this file system"), OSError(errno.EIO, "a made-up error")): + with self.subTest(error=error.errno), mock.patch.object(make_game, "_lock_first_byte", side_effect=error): + result, text, seconds = self.lock() + self.assertIs(result, False) + self.assertIn("this build runs without the work folder's lock", text) + self.assertIsNone(make_game._work_lock) + # and a held error, however the OS words it, refuses + for number in make_game.WORK_LOCK_HELD: + with self.subTest(held=number), mock.patch.object(make_game, "_lock_first_byte", side_effect=OSError(number, "held")): + self.assertEqual(self.lock()[0], "exit 1") + + def test_main_locks_after_the_stop_and_before_the_stale_objects(self): + out = self.root / "out" + for stopped, wait in ((2, 10), (0, 0), (None, 0)): + calls = [] + argv = ["make_game.py", "--work", str(self.work), "--out", str(out), "--romfs", str(self.root / "r"), + "--exefs", str(self.root / "e")] + with contextlib.ExitStack() as stack: + stack.enter_context(mock.patch.object(sys, "argv", argv)) + stack.enter_context(mock.patch.object(make_game, "check_windows_sdk")) + stack.enter_context(mock.patch.object(make_game, "check_cpu")) + stack.enter_context(mock.patch.object(make_game, "no_throttle")) + stack.enter_context(mock.patch.object(make_game, "warn_slow_disk")) + stack.enter_context(mock.patch("ctypes.WinDLL", create=True)) + stack.enter_context(mock.patch.object( + make_game, "stop_stray_compilers", side_effect=lambda work, default: calls.append("stop") or stopped)) + stack.enter_context(mock.patch.object( + make_game, "lock_work", side_effect=lambda work, wait=0.0: calls.append(("lock", work, wait)))) + + def stale(work): + calls.append("stale") + raise Stop + + stack.enter_context(mock.patch.object(make_game, "drop_stale_objects", side_effect=stale)) + stack.enter_context(contextlib.redirect_stdout(io.StringIO())) + with self.assertRaises(Stop): + make_game.main() + self.assertEqual(calls, ["stop", ("lock", os.path.abspath(str(self.work)), wait), "stale"]) + + def test_source_and_docs(self): + source = SOURCE.read_text(encoding="utf-8") + main = source[source.index("def main(a=None):"):source.index("def cli():")] + stop = main.index("stopped = stop_stray_compilers(work, a.default_work)") + lock = main.index("lock_work(work, wait=10 if stopped else 0)") + stale = main.index("drop_stale_objects(work)") + self.assertTrue(stop < lock < stale) + self.assertEqual(main.count("lock_work("), 1) + self.assertIn('if os.environ.get("HOOKSHOT_WORK_LOCK") == "0":', source) + self.assertIn("fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)", source) + self.assertIn("msvcrt.locking(f.fileno(), msvcrt.LK_NBLCK, 1)", source) + self.assertIn("make_game.py", make_game.LIFT_KEY_NOT) # (the lock changes no lift key: nothing is translated again) + docs = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + rows = [line for line in docs.splitlines() if line.startswith("| `HOOKSHOT_WORK_LOCK=0` |")] + self.assertEqual(len(rows), 1) + self.assertIn("IO-47", rows[0]) + for name in ("test_make_game_cache.py", "test_missing_nup.py"): # (they run main() in temporary folders) + self.assertTrue("self.addCleanup(make_game.release_work)" in (ROOT / "tests" / name).read_text(encoding="utf-8"), name) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_wrote_regions.py b/tests/test_wrote_regions.py new file mode 100644 index 0000000..1774d1b --- /dev/null +++ b/tests/test_wrote_regions.py @@ -0,0 +1,122 @@ +"""A texture whose storage no GPU write ever reached is answered "not written" without the lock (gpu/vk/wrote_regions.h, +vk_texture.cpp texture_gpu_wrote and gpu_wrote_into; no game, no GPU). + +(Triforce, 11-21, the performance audit's TEX-02, after Source A's.) gpu_wrote_into took gpu_wrote_mu and walked the map +of written ranges for every sampled, uploaded texture once an epoch; nearly every answer is no. + +This compiles wrote_regions.h alone and checks the bits against a map kept beside them: over 20,000 random writes and +60,000 random questions a "no" from the bits is never a range the map has a write in (a set bit may be a false "ask the +map", never the other way); a write marks every region it touches, across a region's edge too; an empty range marks and +answers nothing; a range longer than the bits wraps onto all of them; addresses past 40 bits share the bits below. + +And reads the source: a write's regions are marked before the map's lock is taken, whatever the switch says; the +question is asked of the bits before the lock and answers no only in mode 1; mode 2 asks the map too and counts a +disagreement; mode 0 never reads the bits' answer; the switch is in the switches table. + + python -I -B tests/test_wrote_regions.py [--cxx clang++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +VK = ROOT / "gpu" / "vk" +RUN_ALL_ARGS = ["--cxx", "{cxx}"] + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +PROGRAM = r""" +#include +#include +#include +#include "wrote_regions.h" +static int fails = 0; +#define CHECK(x) do { if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) +static uint64_t state = 0x2545F4914F6CDD1Dull; +static uint64_t rnd() { state ^= state << 13, state ^= state >> 7, state ^= state << 17; return state; } +int main() { + using namespace wrote_regions; + const uint64_t MB = 1ull << 20; + CHECK(kShift == 21 && kRegions == (1ull << 19) && sizeof bits == 64 * 1024); + CHECK(!any(0, 64 * MB) && !any(5, 5)); + mark(5, 5); // (an empty range: nothing) + CHECK(!any(0, 2 * MB)); + mark(3 * MB - 1, 3 * MB + 1); // (across a region's edge: both regions) + CHECK(any(2 * MB, 2 * MB + 1) && any(3 * MB, 4 * MB) && !any(0, 2 * MB) && !any(4 * MB, 6 * MB)); + CHECK(any(0, 3 * MB) && any(1, 100 * MB)); + // against a list of the writes: a "no" is never wrong + std::vector> writes; + const uint64_t base = 0x20000000ull; + for (int i = 0; i < 20000; i++) { + const uint64_t lo = base + (rnd() % (64ull << 30) & ~0xfffull) * ((rnd() & 7) == 0), size = 1 + rnd() % (rnd() % 8 ? 256 * 1024 : 48 * MB); + const uint64_t at = lo + (rnd() % 4096) * 4096 * 512; + mark(at, at + size); + writes.push_back({at, at + size}); + } + int no = 0; + for (int q = 0; q < 60000; q++) { + const uint64_t lo = base + rnd() % (80ull << 30), hi = lo + 1 + rnd() % (8 * MB); + if (any(lo, hi)) continue; + no++; + for (const auto& w : writes) CHECK(!(w.first < hi && lo < w.second)); + if (fails) break; + } + CHECK(no > 100); // (the bits do answer no: the test has something to check) + for (const auto& w : writes) CHECK(any(w.first, w.second) && any(w.first, w.first + 1) && any(w.second - 1, w.second)); + // past 40 bits: the bits of the addresses below + mark((1ull << 40) + 700ull * (2 * MB), (1ull << 40) + 700ull * (2 * MB) + 1); + CHECK(any(700ull * (2 * MB), 700ull * (2 * MB) + 1)); + // a range longer than the bits: all of them + CHECK(span(0, 1ull << 41) == kRegions); + return fails ? 1 : 0; +} +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cxx", default="clang++") + args = parser.parse_args() + tex = (VK / "vk_texture.cpp").read_text(encoding="utf-8") + check('#include "wrote_regions.h"' in tex, "vk_texture.cpp does not use the bits") + wrote = block(tex, "void texture_gpu_wrote(uint64_t gpu, uint64_t size) {") + check(wrote.index("wrote_regions::mark(lo, hi);") < wrote.index("std::lock_guard lk(gpu_wrote_mu);") and + wrote.count("wrote_regions::mark(") == 1 and "RX_VK_WROTE_REGIONS" not in wrote, + "a write's regions are not marked before the map has it, or only under the switch") + into = block(tex, "bool gpu_wrote_into(Texture& tex) {") + at = [into.index(p) for p in ("const uint64_t lo = tex.gpu, hi = tex.gpu + std::max(tex.guest_size, 1);", + "const bool none = regions && !wrote_regions::any(lo, hi);", "if (regions == 1) return false;", + "std::lock_guard lk(gpu_wrote_mu);", "if (!newest) return false;", "if (none && st_wrong.fetch_add(")] + check(at == sorted(at), "the bits are not asked before the lock, or mode 2's check is not after the map's answer") + check('getenv("RX_VK_WROTE_REGIONS") ? std::clamp(atoi(getenv("RX_VK_WROTE_REGIONS")), 0, 2) : 1' in into, + "the switch is not RX_VK_WROTE_REGIONS, 1 unless set") + check("it->second.second <= completed_value && it->second.second > tex.gpu_wrote_seen" in into, "the map's own answer changed") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + check("`RX_VK_WROTE_REGIONS=0`, `=2`" in dev, "docs/development.md: RX_VK_WROTE_REGIONS is not in the switches table") + + with tempfile.TemporaryDirectory(prefix="wrote-regions-") as tmp: + c, exe = Path(tmp) / "regions.cpp", Path(tmp) / "regions.exe" + c.write_text(PROGRAM, encoding="utf-8") + subprocess.run([args.cxx, "-std=c++20", "-O2", "-Wall", "-Wextra", "-Werror", "-I" + str(VK), str(c), "-o", str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True, timeout=300) + check(r.returncode == 0, "the bits as made:\n" + r.stdout[-2000:]) + print("PASS: the regions' bits never say no to storage a GPU write reached, and are asked before the map's lock") + + +if __name__ == "__main__": + main() diff --git a/tests/test_x3d_placement.py b/tests/test_x3d_placement.py index 2b517ff..ad42537 100644 --- a/tests/test_x3d_placement.py +++ b/tests/test_x3d_placement.py @@ -15,8 +15,15 @@ And what follows from the die: the affinity a game thread gets (hos_x3d_thread_m with no die exactly what it was before), and the replay's core, which pick_reserved_core (cut out of hos_thread.c) takes from the die's cores: the die's last core, not the processor's. +(Triforce, 11-22, the performance audit's THREAD-49, after Source A's.) Whether a core of the die is kept for the replay +(hos_x3d_keep_replay_core, hos_x3d_reserved): with SMT off none is, so the game has all 8 of the die's cores; with SMT +on one is, as before; HOS_RESERVE_CORE=1 and a replay on a thread of its own keep one, =0 keeps none. Run from the +caches to the game threads' affinity on the made-up processors above and on this PC's kind (8 cores, 16 logical +processors, one L3), where nothing may change. + The source is read for what cannot run here: HOS_X3D=0 turns it off, the log says what was found, the reader passes the -process's affinity and the group count, and the reserved core is chosen among the die's cores. +process's affinity and the group count, the reserved core is chosen among the die's cores, and hos_reserved_cpus asks +the rule with the switch's value and whether the replay has its own thread. python -I -B tests/test_x3d_placement.py [--cc gcc] """ @@ -80,6 +87,21 @@ static uint64_t reserved(uint64_t die) { if (!die || (cores[i] & die)) m[n] = cores[i], c[n++] = classes[i]; return pick_reserved_core(m, c, n); } +/* ((Triforce, THREAD-49) the core hos_reserved_cpus keeps, from the caches on: the die, its last core, the rule. asked: + HOS_RESERVE_CORE, -1 not set; own: the replay on a thread of its own) */ +static uint64_t kept_die; +static uint64_t kept(uint64_t allowed, int asked, int own) { + int n = 0; + kept_die = pick(allowed, 1); + for (int i = 0; i < ncores; i++) + if (!kept_die || (cores[i] & kept_die)) n++; + return hos_x3d_reserved(reserved(kept_die), kept_die, n, asked, own); +} +/* (and the affinity a game thread then gets) */ +static uint64_t game(uint64_t allowed, int asked, int own) { + const uint64_t r = kept(allowed, asked, own); + return hos_x3d_thread_mask(allowed, kept_die, r); +} int main(void) { /* Ryzen 7 7800X3D: 8 cores with SMT, one L3 of 96 MB */ @@ -191,6 +213,66 @@ int main(void) { CHECK(hos_x3d_thread_mask(0xFFFF, 0xFFFF0000ull, 0xC000) == 0x3FFF, "a die the process is not on: as with none"); CHECK(hos_x3d_thread_mask(0xC0FF, 0xFF00, 0xC000) == 0xC000, "the die's processors all reserved: the die"); + /* ---- (Triforce, THREAD-49) whether a core of the die is kept for the replay ---- */ + CHECK(hos_x3d_keep_replay_core(0xFF, 8, -1, 0) == 0, "8 cores on 8 logical processors, not set: none kept"); + CHECK(hos_x3d_keep_replay_core(0xFFFF, 8, -1, 0) == 1, "8 cores on 16 logical processors, not set: one kept, as before"); + CHECK(hos_x3d_keep_replay_core(0xFF, 8, 1, 0) == 1 && hos_x3d_keep_replay_core(0xFFFF, 8, 1, 0) == 1, "asked for: kept"); + CHECK(hos_x3d_keep_replay_core(0xFF, 8, 0, 0) == 0 && hos_x3d_keep_replay_core(0xFFFF, 8, 0, 0) == 0, "asked not to: none"); + CHECK(hos_x3d_keep_replay_core(0xFFFF, 8, 0, 1) == 0, "asked not to, with the replay's own thread: none, as before"); + CHECK(hos_x3d_keep_replay_core(0xFF, 8, -1, 1) == 1, "the replay on a thread of its own: kept, the thread is pinned there"); + CHECK(hos_x3d_keep_replay_core(0, 8, -1, 0) == 1 && hos_x3d_keep_replay_core(0, 0, -1, 0) == 1, "no die: as before"); + CHECK(hos_x3d_keep_replay_core(0xFF, 0, -1, 0) == 1, "the die's cores not counted: as before"); + CHECK(hos_x3d_reserved(0x80, 0xFF, 8, -1, 0) == 0 && hos_x3d_reserved(0x80, 0xFF, 8, 1, 0) == 0x80, "the kept core, or none"); + + /* a two-die X3D with SMT off (16 logical processors): none kept, a game thread has the whole die */ + fresh(); + add_cores(16, 1, 0, MB(1), 0); + add_l3(0xFF, MB(96)), add_l3(0xFF00, MB(32)); + CHECK(kept(0xFFFF, -1, 0) == 0 && kept_die == 0xFF, "SMT off: no core of the die kept"); + CHECK(game(0xFFFF, -1, 0) == 0xFF, "a game thread there: all 8 of the die's cores"); + CHECK(kept(0xFFFF, 1, 0) == 0x80 && game(0xFFFF, 1, 0) == 0x7F, "HOS_RESERVE_CORE=1: the die's last core, as before"); + CHECK(kept(0xFFFF, -1, 1) == 0x80 && game(0xFFFF, -1, 1) == 0x7F, "RX_VK_ASYNC=1: the replay thread's core is kept"); + CHECK(kept(0xFFFF, 0, 0) == 0 && game(0xFFFF, 0, 0) == 0xFF, "HOS_RESERVE_CORE=0: none, as before"); + /* the large cache on the second die, SMT off */ + fresh(); + add_cores(16, 1, 0, MB(1), 0); + add_l3(0xFF, MB(32)), add_l3(0xFF00, MB(96)); + CHECK(kept(0xFFFF, -1, 0) == 0 && game(0xFFFF, -1, 0) == 0xFF00, "the second die, SMT off: none kept, the die whole"); + /* the same part with SMT on: one kept, exactly as before */ + fresh(); + add_cores(16, 2, 0, MB(1), 0); + add_l3(0xFFFF, MB(96)), add_l3(0xFFFF0000ull, MB(32)); + CHECK(kept(0xFFFFFFFFull, -1, 0) == 0xC000 && game(0xFFFFFFFFull, -1, 0) == 0x3FFF, "SMT on: the die's last core kept, as before"); + CHECK(kept(0xFFFFFFFFull, 1, 0) == 0xC000 && kept(0xFFFFFFFFull, -1, 1) == 0xC000, "and with the switch or the replay's own thread"); + /* SMT on, the process held to one logical processor of each core: 8 processors on the die's 8 cores, as SMT off */ + CHECK(kept(0xFFFF5555ull, -1, 0) == 0 && kept_die == 0x5555 && game(0xFFFF5555ull, -1, 0) == 0x5555, + "one processor a core allowed: none kept"); + /* this PC's kind: 8 cores, 16 logical processors, one L3: no die, and the core is kept whatever the replay's thread */ + fresh(); + add_cores(8, 2, 0, MB(1), 0); + add_l3(0xFFFF, MB(32)); + CHECK(kept(0xFFFF, -1, 0) == 0xC000 && kept_die == 0 && game(0xFFFF, -1, 0) == 0x3FFF, "8 cores, 16 processors, one L3: as before"); + CHECK(kept(0xFFFF, 1, 0) == 0xC000 && kept(0xFFFF, -1, 1) == 0xC000 && kept(0xFFFF, 0, 0) == 0, "and with the switch"); + /* the same with SMT off (8 logical processors, one L3): still no die, so still kept: the rule is the die's alone */ + fresh(); + add_cores(8, 1, 0, MB(1), 0); + add_l3(0xFF, MB(96)); + CHECK(kept(0xFF, -1, 0) == 0x80 && kept_die == 0 && game(0xFF, -1, 0) == 0x7F, "one die with SMT off: as before"); + /* two dies alike with SMT off, a six-core cache die, an Intel processor of two kinds of core: no die, as before */ + fresh(); + add_cores(16, 1, 0, MB(1), 0); + add_l3(0xFF, MB(32)), add_l3(0xFF00, MB(32)); + CHECK(kept(0xFFFF, -1, 0) == 0x8000 && game(0xFFFF, -1, 0) == 0x7FFF, "two dies alike, SMT off: the processor's last core"); + fresh(); + add_cores(12, 1, 0, MB(1), 0); + add_l3(0x3F, MB(96)), add_l3(0xFC0, MB(32)); + CHECK(kept(0xFFF, -1, 0) == 0x800 && kept_die == 0, "six cores under the large cache, SMT off: no die, the last core kept"); + fresh(); + add_cores(8, 2, 0, MB(2), 1); + add_cores(16, 1, 16, MB(4), 0); + add_l3(0xFFFFFFFFull, MB(36)); + CHECK(kept(0xFFFFFFFFull, -1, 0) == 0xC000 && game(0xFFFFFFFFull, -1, 0) == 0xFFFF3FFFull, "two kinds of core: as before"); + printf(fails ? "x3d placement: FAILED\n" : "x3d placement: all checks passed\n"); return fails != 0; } @@ -202,6 +284,7 @@ def check_source(thread): cpus = block(thread, "static uint64_t hos_x3d_cpus(") last = block(thread, "static uint64_t last_fastest_core(") pin = block(thread, "void hos_thread_pin(") + kept = block(thread, "uint64_t hos_reserved_cpus(void) {") for text, needle, why in ( (detect, "RelationCache", "the caches are not read"), (detect, "e->Cache.GroupMask.Group == group", "caches of another processor group are taken"), @@ -211,9 +294,23 @@ def check_source(thread): (cpus, "hos_log(\"cpu: logical processors", "what was found is not logged"), (last, "const uint64_t die = hos_x3d_cpus();", "the replay's core is not chosen with the die"), (last, "(!die || ((uint64_t)e->Processor.GroupMask[0].Mask & die))", "the replay's core is not among the die's cores"), - (pin, "hos_x3d_thread_mask((uint64_t)proc, die, reserved)", "a game thread's affinity is not the header's")): + (pin, "hos_x3d_thread_mask((uint64_t)proc, die, reserved)", "a game thread's affinity is not the header's"), + # ((Triforce, THREAD-49) the rule is asked with the switch's value and whether the replay has its own thread) + (kept, 'const char *e = getenv("HOS_RESERVE_CORE");', "HOS_RESERVE_CORE is not read"), + (kept, "const int asked = e ? atoi(e) != 0 : -1;", "the switch not set is not told from the switch set"), + (kept, "if (!asked) return mask;", "HOS_RESERVE_CORE=0 no longer keeps none on every processor"), + (kept, "const uint64_t die = hos_x3d_cpus();", "the rule is not asked about the die the game's threads are kept on"), + (kept, "if (cores && fast && !hos_x3d_reserved(fast, die, cores, asked, rx_async_replay())) {", + "hos_reserved_cpus does not ask the header whether a core of the die is kept"), + (kept, 'hos_log("cpu: no core kept for the replay on the cache die (HOS_RESERVE_CORE=1: one is)");', + "keeping no core on the die leaves no line in the log")): if needle not in text: raise SystemExit("host/hos_thread.c: " + why) + if kept.index("!hos_x3d_reserved(fast, die, cores, asked, rx_async_replay())") > kept.index("if (cores) return mask = fast;"): + raise SystemExit("host/hos_thread.c: hos_reserved_cpus keeps the die's core before it asks the rule") + dev = (ROOT / "docs" / "development.md").read_text(encoding="utf-8") + if "`HOS_RESERVE_CORE=0`, `=1` |" not in dev or "`cpu: no core kept for the replay on the cache die`" not in dev: + raise SystemExit("docs/development.md: HOS_RESERVE_CORE=1 and the cache die's rule are not in the switches table") # (the reader is Windows' alone: Linux and the web keep what they had) at = thread.index("static uint64_t x3d_detect(") if thread.rindex("#if defined(_WIN32)", 0, at) < thread.rindex("#endif", 0, at): diff --git a/tests/tex_count_test.cpp b/tests/tex_count_test.cpp new file mode 100644 index 0000000..322eada --- /dev/null +++ b/tests/tex_count_test.cpp @@ -0,0 +1,82 @@ +// The count of an image in the game's own BC or ASTC blocks (gpu/vk/tex_count.h); run by tests/test_tex_count_blocks.py. +// (Triforce, 11-22, the performance audit's TEX-86, after Source A's.) +#include "tex_count.h" + +#include + +static int checks, fails; +static void want(const char* what, uint64_t got, uint64_t bytes) { + checks++; + if (got != bytes) fails++, std::printf("FAIL %s: %llu bytes, want %llu\n", what, (unsigned long long)got, (unsigned long long)bytes); +} +#define CHECK(x) do { checks++; if (!(x)) { std::printf("FAIL:%d %s\n", __LINE__, #x); fails++; } } while (0) + +// (the count as it was, for what the rule replaces: `texel` bytes for every texel of the chain) +static uint64_t by_texels(uint32_t w, uint32_t h, uint32_t d, uint32_t levels, uint32_t layers, uint32_t texel) { + uint64_t px = 0; + for (uint32_t l = 0; l < levels; l++) + px += (uint64_t)((w >> l) ? (w >> l) : 1) * ((h >> l) ? (h >> l) : 1) * ((d >> l) ? (d >> l) : 1); + return px * layers * texel; +} + +int main() { + using tex_count::bytes; + using tex_count::level_blocks; + // the switch: unset, empty and 0 count by blocks + CHECK(!tex_count::by_texels(nullptr) && !tex_count::by_texels("") && !tex_count::by_texels("0")); + CHECK(tex_count::by_texels("1") && tex_count::by_texels("2")); + CHECK(!tex_count::by_texels("off")); + // a level's blocks: rounded up each way, a level of 1 x 1 is one block + CHECK(level_blocks(1365, 1365, 0, 8, 8) == 171 * 171 && level_blocks(1365, 1365, 2, 4, 4) == 86 * 86); + CHECK(level_blocks(1, 1, 0, 4, 4) == 1 && level_blocks(256, 256, 20, 4, 4) == 1 && level_blocks(5, 9, 0, 4, 4) == 2 * 3); + CHECK(level_blocks(100, 50, 0, 12, 10) == 9 * 5 && level_blocks(100, 50, 1, 12, 10) == 5 * 3); + // BC1 (8 bytes a block), 256 x 256 with its whole chain: 4096 + 1024 + 256 + 64 + 16 + 4 + 1 and two tails of one block + want("BC1 256x256, 9 levels", bytes(256, 256, 1, 0, 9, 1, 4, 4, 8), 5463ull * 8); + want("BC1 as it was counted", by_texels(256, 256, 1, 9, 1, 1), 87381); // (twice: a byte a texel) + // BC4 (8 bytes), the game's two 64 x 64 x 64 volumes: a level's depth is slices of flat blocks + want("BC4 64x64x64, 7 levels", bytes(64, 64, 64, 0, 7, 1, 4, 4, 8), (16384ull + 2048 + 256 + 32 + 4 + 2 + 1) * 8); + want("BC4 64x64x64, one level", bytes(64, 64, 64, 0, 1, 1, 4, 4, 8), 131072); + // BC7 (16 bytes): a byte a texel at whole blocks, a 1 x 1 tail a whole block + want("BC7 1024x512, one level", bytes(1024, 512, 1, 0, 1, 1, 4, 4, 16), 1024ull * 512); + want("BC7 1x1", bytes(1, 1, 1, 0, 1, 1, 4, 4, 16), 16); + want("BC7 5x5", bytes(5, 5, 1, 0, 1, 1, 4, 4, 16), 4 * 16); + want("BC7 4x4 with its tails", bytes(4, 4, 1, 0, 3, 1, 4, 4, 16), 3 * 16); + want("BC7 cube 64x64, 6 layers", bytes(64, 64, 1, 0, 1, 6, 4, 4, 16), 6ull * 64 * 64); + // ASTC, 16 bytes a block whatever the block + want("ASTC 4x4 512x512", bytes(512, 512, 1, 0, 1, 1, 4, 4, 16), 512ull * 512); // (a byte a texel; counted 16) + want("ASTC 8x8 512x512", bytes(512, 512, 1, 0, 1, 1, 8, 8, 16), 512ull * 512 / 4); // (counted 64 times this) + want("ASTC 12x12 100x100", bytes(100, 100, 1, 0, 1, 1, 12, 12, 16), 81 * 16); + want("ASTC 12x12 1x1 tail chain", bytes(12, 12, 1, 0, 4, 1, 12, 12, 16), 4 * 16); + want("ASTC 10x5 100x100", bytes(100, 100, 1, 0, 1, 1, 10, 5, 16), 10 * 20 * 16); + // the terrain array Source A measured: 1365 x 1365 x 90 layers, ASTC 8 x 8, 11 levels: 53.8 MiB, where 3,410 were counted + const uint64_t terrain = bytes(1365, 1365, 1, 0, 11, 90, 8, 8, 16); + want("the 1365x1365x90 ASTC 8x8 array", terrain, 39143ull * 16 * 90); + CHECK(terrain * 10 / (1ull << 20) == 537); // (53.7.. MiB: "53.8" rounded) + CHECK(by_texels(1365, 1365, 1, 11, 90, 16) >> 20 == 3410); + // the levels from `first` on: what the first size estimate asks when top levels are left out + want("from level 1", bytes(256, 256, 1, 1, 9, 1, 4, 4, 8), bytes(128, 128, 1, 0, 8, 1, 4, 4, 8)); + want("from level 3 of the array", bytes(1365, 1365, 1, 3, 11, 90, 8, 8, 16), (484ull + 121 + 36 + 9 + 4 + 3) * 16 * 90); + want("no level left", bytes(256, 256, 1, 9, 9, 1, 4, 4, 8), 0); + // never more than the old count for the formats it replaces it for (BC by a byte a texel only past whole blocks) + static const struct { uint32_t bw, bh, bb; } F[] = {{4, 4, 8}, {4, 4, 16}, {5, 4, 16}, {6, 6, 16}, {8, 8, 16}, {10, 10, 16}, {12, 12, 16}}; + for (const auto& f : F) + for (uint32_t w = 1; w <= 4096; w = w * 3 + 1) + for (uint32_t h = 1; h <= 4096; h = h * 5 + 2) { + uint32_t levels = 1; + while (((w > h ? w : h) >> levels) > 0) levels++; + const uint64_t n = bytes(w, h, 1, 0, levels, 3, f.bw, f.bh, f.bb); + checks++; + // (at least a block a level and layer; at most every level's texels rounded up to blocks) + if (n < (uint64_t)levels * 3 * f.bb) fails++, std::printf("FAIL %ux%u %ux%u: under a block a level\n", w, h, f.bw, f.bh); + uint64_t top = 0; + for (uint32_t l = 0; l < levels; l++) top += level_blocks(w, h, l, f.bw, f.bh); + checks++; + if (n != top * f.bb * 3) fails++, std::printf("FAIL %ux%u %ux%u: not the sum of its levels\n", w, h, f.bw, f.bh); + } + // a block size of 0 (a format the table does not know) is taken as 1, not divided by + want("a zero block", bytes(8, 8, 1, 0, 1, 1, 0, 0, 4), 8ull * 8 * 4); + want("zero layers", bytes(8, 8, 1, 0, 1, 0, 4, 4, 8), 4 * 8); + if (fails) std::printf("tex count: FAILED (%d of %d checks)\n", fails, checks); + else std::printf("tex count: all %d checks passed\n", checks); + return fails != 0; +} diff --git a/tests/texture_staging_publication_test.cpp b/tests/texture_staging_publication_test.cpp index 4fe92e3..aa51e44 100644 --- a/tests/texture_staging_publication_test.cpp +++ b/tests/texture_staging_publication_test.cpp @@ -25,6 +25,7 @@ bool fail_large = false, unmap_large = false, unmap_all = false; std::vector attempts; unsigned destroys = 0; uint64_t rec_value = 1, completed_value = 0; +std::atomic st_arena_makes{0}, st_arena_make_ns{0}; // ((Triforce, 11-22, TEX-28) staging() counts the arenas it makes) struct { VkPhysicalDeviceProperties props{}; } dev; int flushes = 0; bool create_buffer(uint64_t n, VkBufferUsageFlags, bool, VkBuffer& b, VmaAllocation& a, void** map) { diff --git a/tests/ub_update_batch_test.cpp b/tests/ub_update_batch_test.cpp new file mode 100644 index 0000000..a7feca9 --- /dev/null +++ b/tests/ub_update_batch_test.cpp @@ -0,0 +1,241 @@ +// tests/test_ub_update_batch.py puts vk_replay.cpp's own UbUpdate, gpu_write, ub_batch_mode, ub_batch_line, +// flush_ub_batched and flush_ub_updates in the place of ACTUAL and runs the flush on a stand-in that records the commands +// and then applies them to memory in order. No game, no GPU. (Triforce, 11-22, the performance audit's SYNC-21.) +// +// with RX_VK_UB_BATCH as the test sets it (unset: as before; 1: the batch; 2: as before, counted) +// +// Scenarios: aligned (every piece 4-byte aligned and in a pool: the old way's memory is the model's too), any (unaligned +// pieces, pieces outside the pools, a second buffer onto the first one's memory), full (the stream runs out), moved (the +// stream goes on in another buffer). +// It prints the hash of the bindings it asked for, of the memory it ended with, and its counts; with the batch it fails +// if memory is not the updates applied in stream order, if a copy's regions overlap, or if bytes written since the last +// barrier are written again. +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ub_batch.h" + +static int fails = 0; +#define CHECK(x) do { if (!(x)) { if (fails++ < 12) std::printf("FAIL:%d %s\n", __LINE__, #x); } } while (0) + +// ---- the stand-ins ---- +using VkBuffer = int; +using VkCommandBuffer = int; +#define VK_NULL_HANDLE 0 +struct VkBufferCopy { uint64_t srcOffset, dstOffset, size; }; +struct BufferRef { + VkBuffer buffer = VK_NULL_HANDLE; + uint64_t offset = 0, size = 0; + uint64_t address = 0; + explicit operator bool() const { return buffer != VK_NULL_HANDLE; } +}; +// (guest memory: three pools' ranges, bound as buffers 1, 2 and 3. Buffer 3 is a second buffer onto buffer 1's memory, as a +// pool's guest buffer and its mirror's are in the renderer when `alias`; else a pool of its own. Above them: outside the pools) +constexpr uint64_t kPool = 256 << 10, kBase = 0x10000000ull, kOutside = kBase + 3 * kPool, kOutsideBytes = 64 << 10; +static bool alias = false; +static std::vector mem[3], cpu_only(kOutsideBytes); // (mem[0], mem[1], mem[2]: the pools' memory) +static int mem_of(VkBuffer b) { return b == 3 && alias ? 0 : b - 1; } +static uint64_t bind_hash = 1469598103934665603ull; +static long binds = 0, cpu_writes = 0; +BufferRef gpu_buffer_bind(uint64_t gpu, uint64_t size, bool written) { + for (uint64_t v : {gpu, size, (uint64_t)written}) bind_hash = (bind_hash ^ v) * 1099511628211ull; + binds++; + if (gpu < kBase || gpu + size > kOutside) return {}; + const VkBuffer b = (VkBuffer)((gpu - kBase) / kPool) + 1; + return {b, (gpu - kBase) % kPool, size, 0}; +} +int guest_cpu_write(uint64_t gpu, const void* src, uint64_t n) { // (at record time: before any command runs) + cpu_writes++; + if (gpu >= kOutside) std::memcpy(cpu_only.data() + (gpu - kOutside), src, n); + else std::memcpy(mem[mem_of((VkBuffer)((gpu - kBase) / kPool) + 1)].data() + (gpu - kBase) % kPool, src, n); + return 1; +} +// (the recording's stream: buffers 100, 101, ...; `stream_room` bytes each; `stream_moves`: it goes on in the next one when full) +static uint64_t stream_room = 8 << 20, stream_used = 0; +static bool stream_moves = false; +static long moves = 0; +static std::vector> streams(1); +BufferRef stream_copy(const void* src, uint64_t n) { + uint64_t at = (stream_used + 255) & ~255ull; + if (at + n > stream_room) { + if (!stream_moves || n > stream_room) return {}; + streams.emplace_back(), stream_used = 0, at = 0, moves++; + } + std::vector& s = streams.back(); + if (s.size() < at + n) s.resize(at + n); + std::memcpy(s.data() + at, src, n); + stream_used = at + n; + return {(VkBuffer)(100 + streams.size() - 1), at, n, 0}; +} +struct Cmd { + int kind = 0; // 0 a barrier, 1 an update, 2 a copy + VkBuffer src = 0, dst = 0; + uint64_t offset = 0; + std::vector data; + std::vector regions; +}; +static std::vector cmds; +void full_barrier(VkCommandBuffer, const char* = "") { cmds.push_back(Cmd{}); } +void vkCmdUpdateBuffer(VkCommandBuffer, VkBuffer b, uint64_t offset, uint64_t n, const void* data) { + CHECK(n <= 65536 && !(n & 3) && !(offset & 3)); + Cmd c; + c.kind = 1, c.dst = b, c.offset = offset, c.data.assign((const uint8_t*)data, (const uint8_t*)data + n); + cmds.push_back(std::move(c)); +} +void vkCmdCopyBuffer(VkCommandBuffer, VkBuffer src, VkBuffer dst, uint32_t n, const VkBufferCopy* regions) { + CHECK(n > 0 && src >= 100 && dst >= 1 && dst <= 3); + Cmd c; + c.kind = 2, c.src = src, c.dst = dst, c.regions.assign(regions, regions + n); + cmds.push_back(std::move(c)); +} +struct GprofSpan { + int items = 1; + uint64_t bytes = 0; + GprofSpan(VkCommandBuffer, const char*) {} +}; +struct UbReadGuard { UbReadGuard() {} }; +namespace lab_ab { bool on(const char*) { return false; } } +static std::string last_line; +void logf(const char* fmt, ...) { + char line[1024]; + va_list ap; + va_start(ap, fmt); + vsnprintf(line, sizeof line, fmt, ap); + va_end(ap); + last_line = line; +} + +/* ACTUAL-1 */ +std::deque ub_updates; +size_t ub_first_of_recording = 0; +/* ACTUAL-2 */ + +// ---- the queue: the commands applied in order, and what they may not do ---- +static long unordered = 0, copies = 0, regions_n = 0, updates_n = 0, barriers = 0; +static void run_commands() { + std::vector> since[3]; // (per memory: [at, end) written since the last barrier) + const auto wrote = [&](int m, uint64_t at, uint64_t end) { + for (const auto& w : since[m]) + if (at < w.second && w.first < end) { unordered++; break; } + since[m].push_back({at, end}); + }; + for (const Cmd& c : cmds) { + if (c.kind == 0) { + barriers++; + for (auto& s : since) s.clear(); + } else if (c.kind == 1) { + updates_n++; + wrote(mem_of(c.dst), c.offset, c.offset + c.data.size()); + std::memcpy(mem[mem_of(c.dst)].data() + c.offset, c.data.data(), c.data.size()); + } else { + copies++, regions_n += (long)c.regions.size(); + const std::vector& s = streams[(size_t)(c.src - 100)]; + for (const VkBufferCopy& r : c.regions) { + CHECK(r.srcOffset + r.size <= s.size() && r.dstOffset + r.size <= kPool); + wrote(mem_of(c.dst), r.dstOffset, r.dstOffset + r.size); // (a region over another of its copy is counted here too) + std::memcpy(mem[mem_of(c.dst)].data() + r.dstOffset, s.data() + r.srcOffset, r.size); + } + } + } + cmds.clear(); +} +static uint64_t rnd_state = 0xA0761D6478BD642Full; +static uint64_t rnd() { + rnd_state ^= rnd_state << 13, rnd_state ^= rnd_state >> 7, rnd_state ^= rnd_state << 17; + return rnd_state; +} + +int main(int argc, char** argv) { + const std::string scenario = argc == 2 ? argv[1] : ""; + const bool aligned = scenario == "aligned", full = scenario == "full", moved = scenario == "moved"; + if (!(aligned || full || moved || scenario == "any")) return std::printf("a known scenario\n"), 1; + alias = scenario == "any"; + const char* e = std::getenv("RX_VK_UB_BATCH"); + const bool batched = e && std::atoi(e) == 1; + for (auto& m : mem) m.assign(kPool, 0xEE); + std::vector model[3] = {mem[0], mem[1], mem[2]}, model_cpu = cpu_only; + long flushes = 0, model_differs = 0, shape_wrong = 0; + for (int f = 0; f < 600; f++) { + // a recording's updates: a few behind the window (flushed by an earlier part) and a few past its limit + const size_t before = rnd() % 3, n = 1 + rnd() % 90, after = rnd() % 3; + ub_updates.clear(); + for (size_t i = 0; i < before + n + after; i++) { + UbUpdate u{}; + const uint64_t r = rnd(); + uint64_t size = r % 13 == 0 ? 60000 + rnd() % 80000 : 16 + rnd() % 600, at; + const int where = scenario == "any" ? (int)(rnd() % 8) : (int)(rnd() % 2); // (0, 1: pools 1, 2; 2: the second buffer onto 1; 7: outside) + if (where == 7) { + size = std::min(size, 4096); + at = kOutside + rnd() % (kOutsideBytes - size); + } else { + // (near the start of its pool, so that updates of one flush often share bytes) + const uint64_t pool = where == 2 ? 2 : (uint64_t)(where & 1); + at = kBase + pool * kPool + rnd() % (rnd() % 4 == 0 ? kPool - size : 6000); + } + if (scenario != "any") size = (size + 3) & ~3ull, at &= ~3ull; + if (at + size > kOutside && at < kOutside) at = kOutside - size; + u.dst = at, u.bytes.resize(size); + for (auto& b : u.bytes) b = (uint8_t)rnd(); + ub_updates.push_back(std::move(u)); + } + ub_first_of_recording = before; + const size_t lim = after ? before + n : (rnd() % 2 ? SIZE_MAX : before + n); + for (size_t i = before; i < before + n; i++) { // (the model: the window's updates, in stream order) + const UbUpdate& u = ub_updates[i]; + if (u.dst >= kOutside) std::memcpy(model_cpu.data() + (u.dst - kOutside), u.bytes.data(), u.bytes.size()); + else { + const int m = mem_of((VkBuffer)((u.dst - kBase) / kPool) + 1); + std::memcpy(model[m].data() + (u.dst - kBase) % kPool, u.bytes.data(), u.bytes.size()); + } + } + streams.assign(1, {}), stream_used = 0; + stream_room = full ? 4096 + rnd() % 30000 : moved ? 2048 + rnd() % 9000 : 8 << 20; + stream_moves = moved; + flush_ub_updates(1, lim); + flushes++; + // the flush begins and ends with a barrier, whatever is between + if (cmds.size() < 2 || cmds.front().kind != 0 || cmds.back().kind != 0) shape_wrong++; + if (!batched) + for (size_t i = 1; i + 1 < cmds.size(); i++) shape_wrong += cmds[i].kind != 1; // (as before: updates alone between the two) + run_commands(); + if (mem[0] != model[0] || mem[1] != model[1] || mem[2] != model[2] || cpu_only != model_cpu) { + model_differs++; + if (batched || aligned || full || moved) { // (the next flush starts from the model: one failure is one flush's) + for (int m = 0; m < 3; m++) mem[m] = model[m]; + cpu_only = model_cpu; + } else { // (as before with unaligned pieces: the CPU's writes at record time are out of order; go on from what it left) + for (int m = 0; m < 3; m++) model[m] = mem[m]; + model_cpu = cpu_only; + } + } + } + CHECK(shape_wrong == 0); + if (batched) { + CHECK(model_differs == 0); // (memory is the updates in stream order) + CHECK(unordered == 0); // (no copy's regions overlap; no byte written twice with no barrier between) + CHECK(copies > 0 && regions_n > copies); + if (scenario == "any" || aligned) CHECK(updates_n == 0); + if (full) CHECK(updates_n > 0); // (pieces written singly when the stream is full) + if (moved) CHECK(moves > 0); // (the stream went on in another buffer: a copy a source) + } else { + CHECK(copies == 0 && updates_n > 0 && barriers == 2 * flushes); + if (scenario != "any") CHECK(model_differs == 0); // (aligned pieces in pools: the old way's memory is the model's) + } + uint64_t h = 1469598103934665603ull; + for (int m = 0; m < 3; m++) + for (uint8_t b : mem[m]) h = (h ^ b) * 1099511628211ull; + for (uint8_t b : cpu_only) h = (h ^ b) * 1099511628211ull; + std::printf("binds %ld %016llx\nmemory %016llx\n", binds, (unsigned long long)bind_hash, (unsigned long long)h); + std::printf("counts: %ld flushes, %ld barriers, %ld updates, %ld copies of %ld regions, %ld writes on the CPU, %ld unordered, %ld flushes off the model\n", + flushes, barriers, updates_n, copies, regions_n, cpu_writes, unordered, model_differs); + std::printf("line: %s\n", last_line.c_str()); + return fails != 0; +} diff --git a/tests/upload_same_test.cpp b/tests/upload_same_test.cpp index 15817ff..202586c 100644 --- a/tests/upload_same_test.cpp +++ b/tests/upload_same_test.cpp @@ -23,6 +23,7 @@ #include #include "upload_same.h" +#include "new_wait.h" // ((Triforce, OS-04) same_copy's nothrow new answers at once: NoWait) static int failures; #define CHECK(c) \ diff --git a/tests/vbytes_test.c b/tests/vbytes_test.c new file mode 100644 index 0000000..5839fb6 --- /dev/null +++ b/tests/vbytes_test.c @@ -0,0 +1,129 @@ +/* tests/test_vbytes.py's driver ((Triforce, 11-21, the performance audit's LIFT-02)): each guest word as nx2/fastvec.py + rewrites it (a64_tbl1, A64_EXT16, A64_EXT8, a64_vld16, a64_vdup) against the same word as the kit's translator writes + it (A64_NO_VBYTES=1, A64_NO_VMOVES=1). Both start from one random Ctx and guest memory; every vector and general + register must match after. + + vbytes_test n random states for each guest word; index bytes biased to the edges + vbytes_test time ns for each kind of guest word, now and as the kit writes it + Exit 0: no mismatch. 1: a mismatch. 2: no guest memory where the loads read. */ +#include "a64.h" +#include +#include +#ifdef _WIN32 +#define WIN32_LEAN_AND_MEAN +#include +#else +#include +#include +#endif +#include "vbytes_lifted.h" + +enum { kGuestLo = 0x10000, kGuestLen = 1 << 20 }; /* (the guest memory the loads read, at A64_PTR(kGuestLo)) */ +static int map_at(void *p, size_t n) { +#ifdef _WIN32 + return VirtualAlloc(p, n, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE) == p; +#else + return mmap(p, n, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0) == p; +#endif +} + +static double now_ns(void) { +#ifdef _WIN32 + LARGE_INTEGER f, t; + QueryPerformanceFrequency(&f), QueryPerformanceCounter(&t); + return (double)t.QuadPart * 1e9 / (double)f.QuadPart; +#else + struct timespec t; + clock_gettime(CLOCK_MONOTONIC, &t); + return t.tv_sec * 1e9 + t.tv_nsec; +#endif +} +static uint64_t rs; +static uint64_t rnd(void) { rs ^= rs << 13; rs ^= rs >> 7; rs ^= rs << 17; return rs; } +static uint8_t edge_byte(void) { /* (TBL's index: around 16, and around the 0x80 the saturated add makes of it) */ + static const uint8_t e[] = {0, 1, 7, 8, 14, 15, 16, 17, 31, 32, 0x6f, 0x70, 0x7f, 0x80, 0x8f, 0x90, 0xf0, 0xfe, 0xff}; + switch (rnd() % 4) { + case 0: return e[rnd() % sizeof e]; + case 1: return (uint8_t)(rnd() % 16); + default: return (uint8_t)rnd(); + } +} +static void fill(Ctx *c) { + memset(c, 0, sizeof *c); + for (int r = 0; r < 32; r++) { + uint8_t b[16]; + for (int k = 0; k < 16; k++) b[k] = (rnd() & 1) ? edge_byte() : (uint8_t)rnd(); + memcpy(c->vreg[r], b, 16); + } + /* (the loads: guest addresses inside the mapped window, unaligned too; some of the memory new for each state) */ + for (int r = 0; r < 31; r++) c->x[r] = kGuestLo + 0x1000 + (rnd() % (kGuestLen - 0x2000)); + for (int k = 0; k < 64; k++) ((uint64_t *)A64_PTR(kGuestLo))[rnd() % (kGuestLen / 8)] = rnd(); +} + +int main(int argc, char **argv) { + if (!map_at(A64_PTR(kGuestLo), kGuestLen)) { + printf("no guest memory at %p\n", (void *)A64_PTR(kGuestLo)); + return 2; + } + for (uint64_t k = 0; k < kGuestLen / 8; k++) ((uint64_t *)A64_PTR(kGuestLo))[k] = k * 0x9e3779b97f4a7c15ULL; + if (argc > 1 && !strcmp(argv[1], "time")) { /* ns for each guest word, now and as the kit writes it: the best of 3 */ + static Ctx c; + rs = 7; + fill(&c); + uint64_t x0[32]; + memcpy(x0, c.x, sizeof x0); + const int iters = 400000, ncase = (int)(sizeof CASES / sizeof CASES[0]); + double sum[4][2] = {{0}}; + int count[4] = {0}; + for (int k = 0; k < ncase; k++) { + const int kind = CASES[k].name[0] == 'T' ? 0 : CASES[k].name[0] == 'E' ? 1 : CASES[k].name[0] == 'L' ? 2 : 3; + for (int w = 0; w < 2; w++) { + double best = 1e30; + void (*fn)(Ctx *) = w ? CASES[k].old : CASES[k].now; + for (int rep = 0; rep < 3; rep++) { + const double t0 = now_ns(); + for (int i = 0; i < iters; i += 1000) { /* (x put back each 1000: a post-indexed load moves its base) */ + memcpy(c.x, x0, sizeof x0); + for (int j = 0; j < 1000; j++) fn(&c); + } + const double ns = (now_ns() - t0) / iters; + if (ns < best) best = ns; + } + sum[kind][w] += best; + } + count[kind]++; + } + static const char *const kinds[4] = {"TBL", "EXT", "128-bit load", "DUP"}; + for (int k = 0; k < 4; k++) + printf(" %-13s %3d words: now %5.2f ns, as the kit writes it %5.2f ns (%.1fx)\n", kinds[k], count[k], sum[k][0] / count[k], + sum[k][1] / count[k], sum[k][1] / sum[k][0]); + return 0; + } + const long n = argc > 1 ? atol(argv[1]) : 200000; + rs = argc > 2 ? strtoull(argv[2], 0, 10) : 1; + if (!rs) rs = 1; + const int ncases = (int)(sizeof CASES / sizeof CASES[0]); + long bad = 0, checks = 0; + for (int k = 0; k < ncases; k++) + for (long i = 0; i < n; i++) { + static Ctx a, b; + fill(&a); + b = a; + CASES[k].now(&a); + CASES[k].old(&b); + checks++; + if (memcmp(a.vreg, b.vreg, sizeof a.vreg) || memcmp(a.x, b.x, sizeof a.x)) { + if (bad++ < 5) { + printf("MISMATCH %s:", CASES[k].name); + for (int r = 0; r < 32; r++) + if (memcmp(a.vreg[r], b.vreg[r], 16)) + printf(" v%d now %016llx%016llx kit %016llx%016llx", r, (unsigned long long)a.vreg[r][1], + (unsigned long long)a.vreg[r][0], (unsigned long long)b.vreg[r][1], + (unsigned long long)b.vreg[r][0]); + printf("\n"); + } + } + } + printf("%d guest words, %ld checks, %ld mismatches\n", ncases, checks, bad); + return bad ? 1 : 0; +} diff --git a/tests/vcore_spin_test.c b/tests/vcore_spin_test.c index eed12d9..da76104 100644 --- a/tests/vcore_spin_test.c +++ b/tests/vcore_spin_test.c @@ -63,6 +63,7 @@ typedef struct HosThread { volatile int vc_kick; int64_t vc_since, vc_grant_at; CONDITION_VARIABLE vc_cv; + int64_t cc_vcw, cc_vcw_ns; /* (HOS_CORE_COUNT: the waits for a virtual core, counted in hos_vcore_back) */ } HosThread; static int failures; @@ -133,6 +134,9 @@ static int hos_thread_core(HosThread *t) { /* (host/hos_thread.c's) */ if (t->ideal_core >= 0) return t->ideal_core % n; return t->core_mask ? (int)(__builtin_ctzll(t->core_mask) % (unsigned)n) : 0; } +/* ((Triforce, 11-22, OS-49) host/hos_thread.c's: the mask HOS_SIM_CPU holds the process to, none here; hos_sim_cpu.h's count) */ +static uint64_t sim_cpu_mask; +static int hos_sim_cpu_count(uint64_t mask) { return __builtin_popcountll(mask); } #include "vcore_block.inc" diff --git a/tests/vecfp_test.c b/tests/vecfp_test.c new file mode 100644 index 0000000..95f6652 --- /dev/null +++ b/tests/vecfp_test.c @@ -0,0 +1,170 @@ +/* tests/test_vecfp.py's driver ((Triforce, 11-22, the performance audit's LIFT-03)): each guest word as lifted with + A64_VECFP=1 (its C behind an SSE helper's test, nx2/vecfp.py), as the kit's translator writes it (A64_VECFP unset) and + with the exact body inline (translated with the fast path off). All three start from one random Ctx; the whole Ctx, + every vector and general register, the flags and FPSR, must match after. + + vecfp_test n random states for each guest word. Half the states have ordinary numbers in every lane (so + the helper does the op), half any operand in any lane: zeros of both signs, subnormals, + infinities, NaNs of both kinds with payloads, the extreme normals, any exponent, equal operands, + products at the underflow and overflow edges, cancelling multiply-adds; FPCR mostly 0 or FZ, + sometimes any other control bit; FPSR with IXC (the gate) set three times in four; NZCV through + all sixteen combinations. + It prints, for each word, how many states the helper did (counted by A64_VFPX_Q_PROBE). + Exit 0: no mismatch. 1: a mismatch. 3: this CPU cannot run the build. */ +#include +static long long vq_done; +#define A64_VFPX_Q_PROBE() (vq_done++) +#include "a64.h" +#include +#include +#include +#include "vecfp_lifted.h" + +static uint64_t rs = 1; +static uint64_t rnd(void) { rs ^= rs << 13; rs ^= rs >> 7; rs ^= rs << 17; return rs; } +static uint32_t rbelow(uint32_t n) { return (uint32_t)(((rnd() >> 32) * n) >> 32); } +static float f_of(uint32_t x) { float f; memcpy(&f, &x, 4); return f; } +static uint32_t f_bits(float f) { uint32_t x; memcpy(&x, &f, 4); return x; } + +static uint64_t rand_fpcr(void) { + static const uint64_t bits[] = {1u << 22, 1u << 23, 3u << 22, 1u << 25, 1u << 26, 1, 2, 4, 1u << 8, 1u << 9, 1u << 10, + 1u << 11, 1u << 12, 1u << 15, 1u << 19, UINT64_C(1) << 32}; + const uint32_t k = rbelow(100); + if (k < 45) return 0; + if (k < 80) return 1u << 24; /* FZ alone */ + uint64_t v = bits[rbelow(sizeof bits / sizeof bits[0])]; + if (rnd() & 1) v |= 1u << 24; + if (!(rnd() & 3)) v |= bits[rbelow(sizeof bits / sizeof bits[0])]; + return v; +} +static uint64_t rand_fpsr(void) { + uint64_t v = rnd() & 0x9f; + if (!(rnd() & 3)) v |= 1u << 27; + return (rnd() & 3) ? v | 16u : v & ~(uint64_t)16u; +} +static uint32_t frac32(void) { + switch (rbelow(8)) { + case 0: return 0; + case 1: return 0x7fffff; + case 2: return (uint32_t)rnd() & 0x7fffff & ~((1u << (1 + rbelow(22))) - 1); + case 3: return (1u << rbelow(23)) | ((uint32_t)rnd() & 1); + default: return (uint32_t)rnd() & 0x7fffff; + } +} +static uint32_t ord32(void) { return ((uint32_t)(rnd() & 1) << 31) | ((117 + rbelow(21)) << 23) | frac32(); } +static uint32_t any32(void) { + const uint32_t k = rbelow(100); + const uint32_t sign = (uint32_t)(rnd() & 1) << 31; + if (k < 8) return sign; /* zero */ + if (k < 15) return sign | (frac32() ? frac32() : 1); /* subnormal (or zero) */ + if (k < 20) return sign | 0x7f800000u; /* infinity */ + if (k < 24) return sign | 0x7fc00000u | ((uint32_t)rnd() & 0x3fffff); /* quiet NaN */ + if (k < 28) return sign | 0x7f800000u | (((uint32_t)rnd() & 0x3fffff) | 1); /* signalling NaN */ + if (k < 34) return sign | ((rnd() & 1) ? 0x00800000u | ((uint32_t)rnd() & 3) : 0x7f7fffffu - ((uint32_t)rnd() & 3)); + if (k < 55) return sign | ((1 + rbelow(254)) << 23) | frac32(); /* any exponent */ + return ord32(); +} +static uint64_t frac64(void) { + switch (rbelow(6)) { + case 0: return 0; + case 1: return (UINT64_C(1) << 52) - 1; + case 2: return (rnd() & ((UINT64_C(1) << 52) - 1)) & ~((UINT64_C(1) << (1 + rbelow(51))) - 1); /* (often a binary32 number) */ + default: return rnd() & ((UINT64_C(1) << 52) - 1); + } +} +static uint64_t ord64(void) { return ((rnd() & 1) << 63) | ((uint64_t)(1003 + rbelow(41)) << 52) | frac64(); } +static uint64_t any64(void) { + const uint32_t k = rbelow(100); + const uint64_t sign = (rnd() & 1) << 63; + static const uint32_t edge[] = {1, 2, 872, 873, 874, 895, 896, 897, 898, 1023, 1149, 1150, 1151, 1152, 2045, 2046}; + if (k < 8) return sign; + if (k < 15) return sign | (frac64() ? frac64() : 1); + if (k < 20) return sign | UINT64_C(0x7ff0000000000000); + if (k < 24) return sign | UINT64_C(0x7ff8000000000000) | (rnd() & ((UINT64_C(1) << 51) - 1)); + if (k < 28) return sign | UINT64_C(0x7ff0000000000000) | ((rnd() & ((UINT64_C(1) << 51) - 1)) | 1); + if (k < 45) return sign | ((uint64_t)edge[rbelow(sizeof edge / sizeof edge[0])] << 52) | frac64(); /* binary32's edges */ + if (k < 60) return sign | ((uint64_t)(1 + rbelow(2046)) << 52) | frac64(); + return ord64(); +} + +/* the operands of case k into the context: rn, rm and (the multiply-adds' addend) rd */ +static void operands(Ctx *c, int k, int mixed) { + const int rd = CASES[k].rd, rn = CASES[k].rn, rm = CASES[k].rm; + if (CASES[k].dbl) { /* (FCVT Sd, Dn and FCVTN: binary64 lanes) */ + for (int h = 0; h < 2; h++) c->vreg[rn][h] = mixed ? any64() : ord64(); + return; + } + uint32_t a[4], b[4], z[4]; + for (int l = 0; l < 4; l++) { + a[l] = mixed ? any32() : ord32(), b[l] = mixed ? any32() : ord32(), z[l] = mixed ? any32() : ord32(); + const uint32_t pick = mixed ? rbelow(16) : 15; + if (pick == 0) b[l] = a[l]; /* equal operands */ + else if (pick == 1) b[l] = a[l] ^ 0x80000000u; /* a and -a (min and max of zeros among them) */ + else if (pick == 2) z[l] = f_bits(-(f_of(a[l]) * f_of(b[l]))) + (uint32_t)((int)rbelow(5) - 2); /* a cancelling addend */ + else if (pick == 3) a[l] = ((1 + rbelow(40)) << 23) | frac32(), b[l] = ((87 + rbelow(40)) << 23) | frac32(); /* a tiny product or a huge quotient */ + else if (pick == 4) a[l] = ((200 + rbelow(54)) << 23) | frac32(), b[l] = ((150 + rbelow(54)) << 23) | frac32(); /* a huge product */ + } + memcpy(c->vreg[rd], z, 16); /* (rd first: where rd is rn or rm the addend is that operand, as in the guest) */ + memcpy(c->vreg[rm], b, 16); + memcpy(c->vreg[rn], a, 16); +} + +int main(int argc, char **argv) { + unsigned eax, ebx, ecx, edx; +#ifdef __SSE4_2__ + if (!__get_cpuid(1, &eax, &ebx, &ecx, &edx) || !(ecx >> 9 & 1) || !(ecx >> 20 & 1) || !(ecx >> 23 & 1)) { + printf("this CPU has no SSSE3/SSE4.2/POPCNT\n"); + return 3; + } +#endif +#ifdef __FMA__ + if (!__get_cpuid(1, &eax, &ebx, &ecx, &edx) || !(ecx >> 12 & 1)) { + printf("this CPU has no FMA\n"); + return 3; + } +#endif + (void)eax, (void)ebx, (void)ecx, (void)edx; + const long n = argc > 1 ? atol(argv[1]) : 20000; + rs = argc > 2 ? strtoull(argv[2], 0, 10) : 1; + if (!rs) rs = 1; + const int ncases = (int)(sizeof CASES / sizeof CASES[0]); + long long bad = 0, checks = 0; +#if defined(__FMA__) + const int fma = 1; +#else + const int fma = 0; +#endif + for (int k = 0; k < ncases; k++) { + long long done = 0; + for (long i = 0; i < n; i++) { + static Ctx base, a, b, e; + for (int v = 0; v < 32; v++) base.vreg[v][0] = rnd(), base.vreg[v][1] = rnd(); + for (int x = 0; x < 31; x++) base.x[x] = rnd(); + operands(&base, k, (int)(i & 1)); + base.n = (uint8_t)(i >> 1 & 1), base.z = (uint8_t)(i >> 2 & 1), base.c = (uint8_t)(i >> 3 & 1), base.v = (uint8_t)(i >> 4 & 1); + base.fpcr = rand_fpcr(), base.fpsr = rand_fpsr(); + a = base, b = base, e = base; + const long long before = vq_done; + CASES[k].now(&a); + done += vq_done - before; + CASES[k].old(&b); + CASES[k].exact(&e); + checks++; + const int da = memcmp(&a, &e, sizeof a) != 0, db = memcmp(&b, &e, sizeof b) != 0; + if ((da || db) && bad++ < 8) { + const int rd = CASES[k].rd, rn = CASES[k].rn, rm = CASES[k].rm; + printf("MISMATCH %s (%s%s against the exact body): fpcr %llx fpsr %llx -> now %llx kit %llx exact %llx\n" + " rn %016llx%016llx rm %016llx%016llx rd %016llx%016llx\n now %016llx%016llx kit %016llx%016llx exact %016llx%016llx\n", + CASES[k].name, da ? "now " : "", db ? "kit " : "", (unsigned long long)base.fpcr, (unsigned long long)base.fpsr, + (unsigned long long)a.fpsr, (unsigned long long)b.fpsr, (unsigned long long)e.fpsr, + (unsigned long long)base.vreg[rn][1], (unsigned long long)base.vreg[rn][0], (unsigned long long)base.vreg[rm][1], + (unsigned long long)base.vreg[rm][0], (unsigned long long)base.vreg[rd][1], (unsigned long long)base.vreg[rd][0], + (unsigned long long)a.vreg[rd][1], (unsigned long long)a.vreg[rd][0], (unsigned long long)b.vreg[rd][1], + (unsigned long long)b.vreg[rd][0], (unsigned long long)e.vreg[rd][1], (unsigned long long)e.vreg[rd][0]); + } + } + printf("case %d done %lld of %ld: %s\n", k, done, n, CASES[k].name); + } + printf("total fma=%d words=%d checks=%lld mismatches=%lld\n", fma, ncases, checks, bad); + return bad ? 1 : 0; +} diff --git a/tests/video_acquire_test.c b/tests/video_acquire_test.c new file mode 100644 index 0000000..2babc90 --- /dev/null +++ b/tests/video_acquire_test.c @@ -0,0 +1,141 @@ +/* tests/test_video_acquire_wait.py's fixture: host/nn_video.c's decoder record and its Submit, Stop/Flush and Acquire + calls (video_acquire_block.inc, cut out unchanged by the test) run over stand-ins: a clock that moves only when a + sleep moves it (each sleep 5 ms long, as a 2 ms sleep is on Windows), a decoder whose one picture comes out at a set + time, and a virtual core whose giving up and taking back are watched (never taken back under a lock). + + video_acquire_test : empty (nothing will come), stopped (a Stop, then nothing), resubmit (a Stop, a submission, + a picture 30 ms later), pending (a picture 60 ms later), noslot (no buffer registered). One RESULT line. */ +#include +#include +#include +#include + +typedef struct { uint64_t x[32]; } Ctx; +#define ARG(n) (c->x[n]) +#define RET(v) (c->x[0] = (uint64_t)(v)) +#define G64(a) (*(uint64_t *)(uintptr_t)(a)) +#define G32(a) (*(uint32_t *)(uintptr_t)(a)) +#define HOS_WRITTEN(p, n) ((void)(p), (void)(n)) +#define RESULT_NO_OUTPUT 0x2a1ca +#define POOL_MAGIC 0x4c4f4f50u +#define DEC_MAGIC 0x43454456u +#define MAX_SLOTS 16 +#define MAX_QUEUE 8 +#define SYNC_IDLE_TAG 0x5ec0000000000000ull +#define SYNC_IDLE_MASK 0x0000ffffffffffffull +#define AV_INPUT_BUFFER_PADDING_SIZE 64 +#define InterlockedCompareExchange64(p, a, b) ((void)(a), (void)(b), *(p)) + +typedef struct AVCodecContext AVCodecContext; +typedef struct AVFrame AVFrame; +struct SwsContext; +typedef struct { uint8_t *data; int64_t pts; } AVPacket; +typedef struct { int held; } SRWLOCK; + +static int64_t clock_ns, arrive_ns = -1; +static int core = 1, locks_held, decodes, decoded_with_core, back_under_lock, slept_under_lock, arrived; +static volatile int64_t submissions; + +static void AcquireSRWLockExclusive(SRWLOCK *l) { + if (l->held) { puts("FAIL: a lock taken twice"); exit(2); } + l->held = 1, locks_held++; +} +static void ReleaseSRWLockExclusive(SRWLOCK *l) { + if (!l->held) { puts("FAIL: a lock let go that was not held"); exit(2); } + l->held = 0, locks_held--; +} +static int64_t hos_now_ns(void) { return clock_ns; } +static void hos_sleep_ns(int64_t ns) { + (void)ns; + if (locks_held) slept_under_lock++; + clock_ns += 5000000; +} +static int hos_vcore_away(void) { + if (!core) return 0; + core = 0; + return 1; +} +static void hos_vcore_back(void) { + if (locks_held) back_under_lock++; + core = 1; +} +static void hos_log(const char *f, ...) { (void)f; } +static int video_log(void) { return 0; } +static void *hos_guest_range(uint64_t a, uint64_t n) { (void)a, (void)n; return NULL; } +static void *host_of(uint64_t obj, uint32_t magic) { (void)magic; return (void *)(uintptr_t)obj; } +static uint64_t submitted(void) { return 1; } +static AVPacket *av_packet_alloc(void) { return NULL; } +static int av_new_packet(AVPacket *p, int n) { (void)p, (void)n; return -1; } +static void av_packet_free(AVPacket **p) { *p = NULL; } +static void avcodec_flush_buffers(AVCodecContext *x) { (void)x; } +static void mf_av1_flush(void *m) { (void)m; } + +#include "../host/video_movie.h" /* (the Dec cut out keeps a movie's counts: VideoMovie) */ +#include "../host/video_catchup.h" /* (and its pictures' pace: VideoPace, the catch-up's rule) */ +#include "../host/video_input.h" /* (and the submissions' limit: its rule and result) */ +#include "video_acquire_types.inc" + +static void pending_push(Dec *d, AVPacket *pkt) { (void)d, (void)pkt; } +static void pending_free(Dec *d) { d->phead = d->ptail = NULL; } +static void picture_free(Picture *p) { + free(p->rgba), free(p->nv12); + memset(p, 0, sizeof *p); +} +static void pool_rgba(Pool *o) { (void)o; } +static void decode_pending(Dec *d) { /* (the decoder: its one picture out at arrive_ns) */ + decodes++; + if (core) decoded_with_core++; + if (arrive_ns >= 0 && clock_ns >= arrive_ns && d->qn < MAX_QUEUE) { + Picture *p = &d->queue[(d->qhead + d->qn) % MAX_QUEUE]; + memset(p, 0, sizeof *p); + p->w = p->h = 4; + p->nv12 = calloc(4 * 4 * 3 / 2, 1); + d->qn++; + arrive_ns = -1, arrived = 1; + } +} +static void decode_next(Dec *d) { decode_pending(d); } +static void movie_end(Dec *d) { memset(&d->movie, 0, sizeof d->movie); } /* (the movie's log line: not tested here) */ +/* (IO-34: the submissions' limit off here, as with HOS_VIDEO_INPUT_LIMIT=0; this fixture's waits are the acquire's) */ +static int input_limit(void) { return 0; } +static void input_room(Dec *d) { (void)d; } +static void hos_system_event_clear(uint64_t key) { (void)key; } + +#include "video_acquire_block.inc" + +int main(int argc, char **argv) { + const char *k = argc > 1 ? argv[1] : ""; + static Dec dec; + static Pool pool; + Ctx ctx, *c = &ctx; + uint32_t index = 0xffffffffu; + uint64_t sync = 0; + dec.ctx = (AVCodecContext *)(uintptr_t)1; + pool.w = pool.h = 4; + pool.nv12 = calloc(4 * 4 * 3 / 2, 1); + if (strcmp(k, "noslot")) dec.slots[0] = (uint64_t)(uintptr_t)&pool; + if (!strcmp(k, "stopped") || !strcmp(k, "resubmit")) { + memset(c, 0, sizeof *c); + ARG(0) = (uint64_t)(uintptr_t)&dec; + dec_flush(c, "Stop"); + } + if (!strcmp(k, "resubmit")) { + uint64_t id = 0; + memset(c, 0, sizeof *c); + ARG(0) = (uint64_t)(uintptr_t)&id, ARG(1) = (uint64_t)(uintptr_t)&dec; + dec_submit(c); + arrive_ns = clock_ns + 30000000; + } + if (!strcmp(k, "pending")) arrive_ns = clock_ns + 60000000; + if (locks_held || !core) { puts("FAIL: the set-up left a lock held or the core away"); return 2; } + const int64_t began = clock_ns; + memset(c, 0, sizeof *c); + ARG(0) = (uint64_t)(uintptr_t)&index, ARG(1) = (uint64_t)(uintptr_t)&sync, ARG(2) = (uint64_t)(uintptr_t)&dec; + dec_acquire(c); + const int delivered = arrived && !dec.qn && c->x[0] == 0; /* (the picture decoded and handed out) */ + printf("RESULT %s elapsed_ms=%lld ret=%#llx index=%d decodes=%d decoded_with_core=%d back_under_lock=%d " + "slept_under_lock=%d core=%d locks=%d delivered=%d\n", + k, (long long)((clock_ns - began) / 1000000), (unsigned long long)c->x[0], index == 0xffffffffu ? -1 : (int)index, + decodes, decoded_with_core, back_under_lock, slept_under_lock, core, locks_held, delivered); + return 0; +} diff --git a/tests/video_fixture.py b/tests/video_fixture.py new file mode 100644 index 0000000..aa2c2a7 --- /dev/null +++ b/tests/video_fixture.py @@ -0,0 +1,66 @@ +"""What the movie tests share (tests/test_video_*.py; not a test itself): host/nn_video.c compiled whole, as it is, over +the stand-ins of tests/video_standins.h, with a test's own program after it. + +The folder made for a build holds hos.h, libavcodec/avcodec.h and libswscale/swscale.h, each one line that includes the +stand-ins, and a copy of nn_video.c beside them: an include in quotes looks in the including file's folder first, so the +copy takes the stand-in hos.h and the real rule headers (video_wait.h and the others) from host/. + +The stand-ins take windows.h for the SRW locks and the interlocked counts nn_video.c uses: the fixture is built and run +on Windows only (WINDOWS_ONLY says why where a test skips it). What a test reads from the source, and the rule headers +it compiles alone, are checked on every system. +""" +import os +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +HOST = ROOT / "host" +TESTS = ROOT / "tests" +WINDOWS = os.name == "nt" +WINDOWS_ONLY = "SKIP: nn_video.c over its stand-ins is built on Windows only (they take windows.h's locks)" + + +def check(ok, why): + if not ok: + raise SystemExit("FAIL: " + why) + + +def block(text, marker): + """The text from marker to the brace that closes the first one opened after it""" + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def video_source(): + return (HOST / "nn_video.c").read_text(encoding="utf-8") + + +def build(tmp, cc, program, name="movie"): + """nn_video.c and `program` (C, written after it in one file) as an executable in tmp; its path""" + tmp = Path(tmp) + one_line = '#include "video_standins.h"\n' + (tmp / "hos.h").write_text(one_line, encoding="utf-8") + for folder, header in (("libavcodec", "avcodec.h"), ("libswscale", "swscale.h")): + (tmp / folder).mkdir(exist_ok=True) + (tmp / folder / header).write_text(one_line, encoding="utf-8") + (tmp / "nn_video.c").write_text(video_source(), encoding="utf-8") + src, exe = tmp / (name + ".c"), tmp / (name + ".exe") + src.write_text('#include "nn_video.c"\n#include "video_play.h"\n' + program, encoding="utf-8") + subprocess.run([cc, "-std=gnu11", "-O1", "-Wall", "-Wextra", "-Wno-unused-function", "-Wno-unused-parameter", + "-Wno-unused-variable", "-I", str(tmp), "-I", str(TESTS), "-I", str(HOST), str(src), "-o", str(exe)], + check=True) + return exe + + +def run(exe, *args, **env): + """The program's output; env: variables set for this run (None: unset), over a clean HOS_ / SI_ environment""" + e = {k: v for k, v in os.environ.items() if not k.startswith(("HOS_", "SI_"))} + e.update({k: str(v) for k, v in env.items() if v is not None}) + r = subprocess.run([str(exe), *args], capture_output=True, text=True, env=e) + check(r.returncode == 0, "%s %s failed (%d):\n%s%s" % (Path(exe).name, " ".join(args), r.returncode, r.stdout, r.stderr)) + return r.stdout diff --git a/tests/video_play.h b/tests/video_play.h new file mode 100644 index 0000000..b9e8d64 --- /dev/null +++ b/tests/video_play.h @@ -0,0 +1,103 @@ +/* The calls the game's movie library makes to nn::codec's AV1 decoder, for the movie tests: included after the copy of + host/nn_video.c that tests/video_standins.h lets them compile, so these call its entry points as the game does (the + arguments in x0 to x7, the result in x0) and can look at its Dec and Pool. One decoder (SI_DEC) with up to 8 picture + buffers, each an image pool of one image. */ +#pragma once + +#define SI_DEC 0x100ull /* the game's decoder object */ +#define SI_POOLS 0x200ull /* its image pools, 0x40 apart */ +#define SI_PREF 0x800ull /* the image preference a pool is made from */ +#define SI_OUT 0x900ull /* what the calls write: a submission's id, a buffer's index, its sync handle, its properties */ +#define SI_DATA 0x1000ull /* a submission's bytes */ +#define SI_FULL 0x1c266full /* the result of a submission refused while the decoder's input is full */ + +static Dec *si_dec(void) { return (Dec *)host_of(SI_DEC, DEC_MAGIC); } +static Pool *si_pool(int i) { return (Pool *)host_of(SI_POOLS + (uint64_t)i * 0x40, POOL_MAGIC); } + +/* the decoder for w x h pictures, with `buffers` picture buffers of that size in the pool format given (0 NV12) */ +static void si_open(int w, int h, int buffers, int pool_format) { + Ctx c; + for (int i = 0; i < buffers; i++) { + const Pref pref = {PREF_MAGIC, (uint16_t)w, (uint16_t)h, (uint32_t)pool_format, 0}; + memcpy(GP(SI_PREF), &pref, sizeof pref); + memset(&c, 0, sizeof c); + c.x[0] = SI_POOLS + (uint64_t)i * 0x40, c.x[3] = SI_PREF, c.x[4] = 1; + imp__ZN2nn5codec26InitializeDecoderImagePoolEPNS0_25VideoDecoderImagePoolTypeEPvmRKNS0_27VideoDecoderImagePreferenceEj(&c); + } + memset(&c, 0, sizeof c); + c.x[0] = SI_DEC, c.x[4] = (uint64_t)w, c.x[5] = (uint64_t)h; + imp__ZN2nn5codec36InitializeAv1DecoderByTransferMemoryEPNS0_14Av1DecoderTypeEPvmmttjNS0_23VideoDecoderInputFormatENS0_24VideoDecoderOutputFormatENS0_22VideoDecoderOutputModeE(&c); + for (int i = 0; i < buffers; i++) { + memset(&c, 0, sizeof c); + c.x[0] = SI_DEC, c.x[1] = (uint64_t)i, c.x[2] = SI_POOLS + (uint64_t)i * 0x40; + imp__ZN2nn5codec24RegisterAv1DecoderBufferEPNS0_14Av1DecoderTypeEjPNS0_25VideoDecoderImagePoolTypeEj(&c); + } +} +/* one submission of 16 bytes for the picture at pts (ns); last: the stream's last. The call's result */ +static uint64_t si_submit(int64_t pts, int last) { + Ctx c; + memset(&c, 0, sizeof c); + c.x[0] = SI_OUT, c.x[1] = SI_DEC, c.x[2] = SI_DATA, c.x[3] = 16, c.x[4] = (uint64_t)pts, c.x[5] = (uint64_t)last; + imp__ZN2nn5codec27SubmitAv1DecoderInputBufferEPmPNS0_14Av1DecoderTypeEPKvmNS_8TimeSpanEb(&c); + return c.x[0]; +} +/* The game's submit loop: pictures *next to total - 1, gap ns apart, until all are in or one is refused because the + decoder's input is full (it is submitted again by the next call). How many were taken; -1 on any other result */ +static int si_feed(int *next, int total, int64_t gap) { + int taken = 0; + while (*next < total) { + const uint64_t result = si_submit(*next * gap, *next == total - 1); + if (result == SI_FULL) break; + if (result) return -1; + (*next)++, taken++; + } + return taken; +} +/* a picture taken: its buffer's index, or -1 when the call failed */ +static int si_acquire(void) { + Ctx c; + memset(&c, 0, sizeof c); + c.x[0] = SI_OUT + 8, c.x[1] = SI_OUT + 16, c.x[2] = SI_DEC; + imp__ZN2nn5codec29AcquireAv1DecoderOutputBufferEPjPNS_2gs16NativeSyncHandleEPNS0_14Av1DecoderTypeE(&c); + return c.x[0] ? -1 : (int)G32(SI_OUT + 8); +} +static void si_release(int buffer) { + Ctx c; + memset(&c, 0, sizeof c); + c.x[0] = SI_DEC, c.x[1] = (uint64_t)buffer; + imp__ZN2nn5codec29ReleaseAv1DecoderOutputBufferEPNS0_14Av1DecoderTypeEjNS_2gs16NativeSyncHandleE(&c); +} +/* the pts of the picture in a buffer; *last: whether it is the stream's last */ +static int64_t si_pts(int buffer, int *last) { + Ctx c; + memset(&c, 0, sizeof c); + c.x[4] = SI_OUT + 24, c.x[5] = SI_OUT + 32, c.x[6] = SI_DEC, c.x[7] = (uint64_t)buffer; + imp__ZN2nn5codec35GetAv1DecoderOutputBufferPropertiesEPtS1_S1_S1_PNS_8TimeSpanEPbPNS0_14Av1DecoderTypeEj(&c); + if (last) *last = *(uint8_t *)GP(SI_OUT + 32); + return (int64_t)G64(SI_OUT + 24); +} +static void si_stop(void) { + Ctx c; + memset(&c, 0, sizeof c); + c.x[0] = SI_DEC; + imp__ZN2nn5codec14StopAv1DecoderEPNS0_14Av1DecoderTypeE(&c); +} +static void si_flush(void) { + Ctx c; + memset(&c, 0, sizeof c); + c.x[0] = SI_DEC; + imp__ZN2nn5codec15FlushAv1DecoderEPNS0_14Av1DecoderTypeE(&c); +} +static void si_finalize(void) { + Ctx c; + memset(&c, 0, sizeof c); + c.x[0] = SI_DEC; + imp__ZN2nn5codec18FinalizeAv1DecoderEPNS0_14Av1DecoderTypeE(&c); +} +static uint64_t si_ready_event(void) { /* GetAv1DecoderSubmissionReadyEvent: the event's guest address */ + Ctx c; + memset(&c, 0, sizeof c); + c.x[0] = SI_DEC; + imp__ZN2nn5codec33GetAv1DecoderSubmissionReadyEventEPNS0_14Av1DecoderTypeE(&c); + return c.x[0]; +} diff --git a/tests/video_standins.h b/tests/video_standins.h new file mode 100644 index 0000000..b5479a8 --- /dev/null +++ b/tests/video_standins.h @@ -0,0 +1,265 @@ +/* Stand-ins for compiling host/nn_video.c whole, as it is, with no game, no FFmpeg and no Media Foundation: the movie + tests (tests/test_video_*.py) put this file in a folder as hos.h, libavcodec/avcodec.h and libswscale/swscale.h, with + a copy of nn_video.c beside them, and include tests/video_play.h after it for the calls the game makes. + + What is here: the guest window and the HLE macros of hos.h; a clock that moves only when a decode or a sleep moves + it; a log kept in memory; system events that remember whether they are set; a decoder that gives one picture a + submission, of a format, a size and a cost the test sets (`si`), under the names of libavcodec's and of + host/mf_av1.c's; a libswscale that counts its conversions. On Windows the notice's message box, its thread and its + Store page are counted, never shown. */ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#ifdef _WIN32 +#define WIN32_LEAN_AND_MEAN +#include +#include +static int si_boxes, si_threads; +#undef MessageBoxA +#undef ShellExecuteA +#undef CreateThread +#define MessageBoxA(window, text, title, type) (si_boxes++, 0) +#define ShellExecuteA(window, verb, file, args, dir, show) ((void)0) +#define CreateThread(security, stack, fn, arg, flags, id) (si_threads++, (void)(fn), (HANDLE)NULL) +#endif + +/* ---- hos.h ---- */ +typedef struct { uint64_t x[32]; } Ctx; +#define SI_GUEST 0x40000ull +static uint8_t si_guest[SI_GUEST]; +#define ARG(n) (c->x[n]) +#define RET(v) (c->x[0] = (uint64_t)(v)) +#define GP(a) ((void *)(si_guest + (a))) +#define G64(a) (*(uint64_t *)GP(a)) +#define G32(a) (*(uint32_t *)GP(a)) +#define HOS_WRITTEN(p, n) ((void)0) +/* ((Triforce, 11-22) host_alloc.h's allocations (OS-04): away from Windows' commit limit they are the C library's */ +void *hos_xmalloc(size_t n) { return malloc(n); } +void *hos_xcalloc(size_t count, size_t size) { return calloc(count, size); } +void *hos_xrealloc(void *p, size_t n) { return realloc(p, n); } +#define HLE(sym) void imp_##sym(Ctx *c) +static inline void *hos_guest_range(uint64_t a, uint64_t n) { return a < SI_GUEST && n <= SI_GUEST - a ? GP(a) : NULL; } + +static char si_log[1 << 18]; /* every line logged, each ended by a newline */ +static size_t si_log_len; +static void hos_log(const char *fmt, ...) { + va_list ap; + va_start(ap, fmt); + if (si_log_len + 1024 < sizeof si_log) { + si_log_len += (size_t)vsnprintf(si_log + si_log_len, 1000, fmt, ap); + si_log[si_log_len++] = '\n', si_log[si_log_len] = 0; + } + va_end(ap); +} +static int si_logged(const char *text) { /* how many times the log has it */ + int n = 0; + for (const char *p = si_log; (p = strstr(p, text)) != NULL; p += strlen(text)) n++; + return n; +} +static void hos_fatal(Ctx *c, const char *fmt, ...) { + (void)c; + printf("FATAL: %s\n", fmt); + exit(3); +} +static int64_t si_now = 5000000000ll; /* the clock: a decode and a sleep move it, nothing else */ +static int64_t hos_now_ns(void) { return si_now; } +static void hos_sleep_ns(int64_t ns) { si_now += ns; } +static int hos_vcore_away(void) { return 0; } +static void hos_vcore_back(void) {} +static uint64_t si_sys = 0x30000; +static uint64_t hos_sys_alloc(uint64_t size, uint64_t align) { + (void)align; + const uint64_t at = si_sys; + si_sys += (size + 63) & ~63ull; + if (si_sys > SI_GUEST) hos_fatal(NULL, "the stand-in system region is full"); + return at; +} + +/* system events (host/nn_os_sync.c), by their guest address */ +static struct { uint64_t key; int set, inits, signals, clears; } si_events[64]; +static int si_nevents; +static int si_event(uint64_t key) { + int i = 0; + while (i < si_nevents && si_events[i].key != key) i++; + if (i == si_nevents) { + if (si_nevents == 64) hos_fatal(NULL, "more events than the stand-in keeps"); + si_events[si_nevents++].key = key; + } + return i; +} +static void hos_system_event_init(uint64_t key, int autoclear) { + (void)autoclear; + const int i = si_event(key); + si_events[i].set = 0, si_events[i].inits++; +} +static void hos_system_event_signal(uint64_t key) { + const int i = si_event(key); + si_events[i].set = 1, si_events[i].signals++; +} +static void hos_system_event_signal_held(uint64_t key) { hos_system_event_signal(key); } /* (under a host lock) */ +static void hos_system_event_clear(uint64_t key) { + const int i = si_event(key); + si_events[i].set = 0, si_events[i].clears++; +} +static int si_event_set(uint64_t key) { return si_events[si_event(key)].set; } +static int si_event_clears(uint64_t key) { return si_events[si_event(key)].clears; } + +/* ---- what the test sets, and what the stand-ins count ---- */ +enum AVCodecID { AV_CODEC_ID_NONE, AV_CODEC_ID_VP8, AV_CODEC_ID_VP9, AV_CODEC_ID_AV1 }; +enum AVPixelFormat { AV_PIX_FMT_NONE = -1, AV_PIX_FMT_YUV420P, AV_PIX_FMT_NV12, AV_PIX_FMT_YUVA420P, AV_PIX_FMT_RGBA, + AV_PIX_FMT_YUV420P10LE }; +static struct { + int have_dav1d, have_native; /* what the FFmpeg has: libdav1d by name, a decoder of its own for the codec */ + int open_fails; /* avcodec_open2 fails */ + int mf_ok; /* Media Foundation has an AV1 decoder */ + int format, w, h, pad; /* the pictures: an AVPixelFormat, their size, bytes of a row past its width */ + int64_t decode_ns; /* what decoding one submission takes, on the stand-in clock */ + int64_t first_decode_ns; /* ... and the first one after a start or a flush, when not 0 */ + char order[64]; /* what dec_init asked for, in order: d libdav1d by name, a libaom by name, n FFmpeg's own + decoder, o avcodec_open2, M Media Foundation quietly, T Media Foundation telling why */ + int sws_contexts, sws_scales; /* libswscale's conversions asked for */ + int64_t decodes, pictures; /* submissions decoded, pictures given */ + int started; +} si = {1, 1, 0, 1, AV_PIX_FMT_YUV420P, 64, 36, 32, 1000000, 0, "", 0, 0, 0, 0, 0}; + +static void si_ask(char what) { + const size_t n = strlen(si.order); + if (n + 1 < sizeof si.order) si.order[n] = what, si.order[n + 1] = 0; +} +static void si_decode_cost(void) { /* one submission's decode, on the clock */ + si_now += !si.started && si.first_decode_ns ? si.first_decode_ns : si.decode_ns; + si.started = 1; + si.decodes++; +} +/* a picture's planes, written again for each one: plane p's byte at (x, y) of picture n */ +static inline uint8_t si_byte(int p, int x, int y, int64_t n) { return (uint8_t)(x * 3 + y * 7 + p * 61 + (int)n * 5 + 1); } +static uint8_t *si_planes[4]; +static int si_stride[4]; +static void si_picture(int64_t n) { + for (int p = 0; p < 4; p++) { + const int chroma = p == 1 || p == 2; + const int semi = si.format == AV_PIX_FMT_NV12 && p == 1; /* (NV12's second plane: U and V in turns, a full row) */ + const int w = chroma && !semi ? (si.w + 1) / 2 : si.w, h = chroma ? (si.h + 1) / 2 : si.h; + si_stride[p] = w + si.pad; + free(si_planes[p]); + si_planes[p] = (uint8_t *)malloc((size_t)si_stride[p] * h); + memset(si_planes[p], 0xEE, (size_t)si_stride[p] * h); /* (a row's padding is never the picture's) */ + for (int y = 0; y < h; y++) + for (int x = 0; x < w; x++) + si_planes[p][(size_t)y * si_stride[p] + x] = semi ? si_byte(1 + (x & 1), x / 2, y, n) : si_byte(p, x, y, n); + } +} + +/* ---- libavcodec ---- */ +#define AV_INPUT_BUFFER_PADDING_SIZE 64 +#define AVERROR(e) (-(e)) +typedef struct AVCodec { const char *name; enum AVCodecID id; } AVCodec; +typedef struct AVPacket { uint8_t *data; int size; int64_t pts; } AVPacket; +typedef struct AVFrame { uint8_t *data[8]; int linesize[8]; int width, height, format; int64_t pts; } AVFrame; +typedef struct AVCodecContext { const AVCodec *codec; int thread_count; int64_t held[4]; int nheld; } AVCodecContext; +static const AVCodec si_dav1d = {"libdav1d", AV_CODEC_ID_AV1}, si_native = {"native", AV_CODEC_ID_NONE}; +static const AVCodec *avcodec_find_decoder_by_name(const char *name) { + const int dav1d = !strcmp(name, "libdav1d"); + si_ask(dav1d ? 'd' : 'a'); + return dav1d && si.have_dav1d ? &si_dav1d : NULL; +} +static const AVCodec *avcodec_find_decoder(enum AVCodecID id) { + (void)id; + si_ask('n'); + return si.have_native ? &si_native : NULL; +} +static AVCodecContext *avcodec_alloc_context3(const AVCodec *codec) { + AVCodecContext *ctx = (AVCodecContext *)calloc(1, sizeof *ctx); + ctx->codec = codec; + return ctx; +} +static int avcodec_open2(AVCodecContext *ctx, const AVCodec *codec, void *options) { + (void)ctx, (void)codec, (void)options; + si_ask('o'); + return si.open_fails ? -1 : 0; +} +static void avcodec_free_context(AVCodecContext **ctx) { + free(*ctx); + *ctx = NULL; +} +static void avcodec_flush_buffers(AVCodecContext *ctx) { ctx->nheld = 0, si.started = 0; } +static int avcodec_send_packet(AVCodecContext *ctx, const AVPacket *pkt) { + if (!pkt) return 0; + if (ctx->nheld == 4) return AVERROR(EAGAIN); + si_decode_cost(); + ctx->held[ctx->nheld++] = pkt->pts; + return 0; +} +static int avcodec_receive_frame(AVCodecContext *ctx, AVFrame *f) { + if (!ctx->nheld) return AVERROR(EAGAIN); + memset(f, 0, sizeof *f); + si_picture(si.pictures); + for (int p = 0; p < 4; p++) f->data[p] = si_planes[p], f->linesize[p] = si_stride[p]; + f->width = si.w, f->height = si.h, f->format = si.format, f->pts = ctx->held[0]; + memmove(ctx->held, ctx->held + 1, sizeof ctx->held - sizeof ctx->held[0]); + ctx->nheld--; + si.pictures++; + return 0; +} +static AVPacket *av_packet_alloc(void) { return (AVPacket *)calloc(1, sizeof(AVPacket)); } +static int av_new_packet(AVPacket *pkt, int size) { + pkt->data = (uint8_t *)calloc(1, (size_t)size + AV_INPUT_BUFFER_PADDING_SIZE); + pkt->size = size; + return pkt->data ? 0 : -1; +} +static void av_packet_free(AVPacket **pkt) { + if (*pkt) free((*pkt)->data); + free(*pkt); + *pkt = NULL; +} +static AVFrame *av_frame_alloc(void) { return (AVFrame *)calloc(1, sizeof(AVFrame)); } +static void av_frame_free(AVFrame **f) { + free(*f); + *f = NULL; +} +static void av_frame_unref(AVFrame *f) { (void)f; } + +/* ---- libswscale: a conversion is counted, and fills its picture with 0x5A ---- */ +#define SWS_BILINEAR 2 +struct SwsContext { int w, h, from; }; +static struct SwsContext *sws_getContext(int w, int h, enum AVPixelFormat from, int dw, int dh, enum AVPixelFormat to, int flags, + void *in_filter, void *out_filter, const double *param) { + (void)dw, (void)dh, (void)to, (void)flags, (void)in_filter, (void)out_filter, (void)param; + struct SwsContext *s = (struct SwsContext *)calloc(1, sizeof *s); + s->w = w, s->h = h, s->from = from; + si.sws_contexts++; + return s; +} +static int sws_scale(struct SwsContext *s, const uint8_t *const *src, const int *src_stride, int y, int h, uint8_t *const *dst, + const int *dst_stride) { + (void)s, (void)src, (void)src_stride, (void)y; + si.sws_scales++; + for (int r = 0; r < h; r++) memset(dst[0] + (size_t)r * dst_stride[0], 0x5A, (size_t)dst_stride[0]); + return h; +} +static void sws_freeContext(struct SwsContext *s) { free(s); } + +/* ---- host/mf_av1.c: NV12 pictures, one a submission ---- */ +typedef void (*si_mf_picture_fn)(void *user, const uint8_t *y, const uint8_t *uv, int w, int h, int stride, int64_t pts); +static void *mf_av1_open(int w, int h, int tell) { + (void)w, (void)h; + si_ask(tell ? 'T' : 'M'); + return si.mf_ok ? (void *)&si : NULL; +} +static void mf_av1_decode(void *m, const uint8_t *data, size_t n, int64_t pts, si_mf_picture_fn fn, void *user) { + (void)m, (void)n; + if (!data) return; /* (the end: nothing was held back) */ + const int format = si.format; + si.format = AV_PIX_FMT_NV12; + si_decode_cost(); + si_picture(si.pictures++); + fn(user, si_planes[0], si_planes[1], si.w, si.h, si_stride[0], pts); + si.format = format; +} +static void mf_av1_flush(void *m) { (void)m, si.started = 0; } +static void mf_av1_close(void *m) { (void)m; }