Всех с прошедшим, народ!
Делал бекап большого количества файлов на 3 Тб с одного харда на другой с помощь rsync. Системный диск отдельный и он ссд. Первые 2 часа копирования все было ок, система оставалась отзывчивой все работало прекрасно. Где то через 2 часа все повисло. У меня специально в углу экрана был сверху htop и там ни ОЗУ не была заполнена (около 24 Гб из 32 свободно), проц не был загружен (98% idle), при этом система зависла, экран стал черным (не отключился), двигается только курсор. На клавиатуру не реагирует совсем.
Я подключился с ноута по ssh и грепнул rsync. Он еще висел, причем в количестве 3 процессов. Я решил не тянуть кота и оставить это дело на ночь, чтобы все спокойно закончилось. Сегодня утром, то есть спустя где-то 10 часов после зависания, десктоп все еще в том же состоянии. По ssh я не увидел rsync процесса, поэтому я решил просто на всякий случай до того, как сделаю ресет компа, отмонтировать оба харда. Отмонтирование зависло. В процессах висит уже часа 2. При этом по mount, lsblk и df у меня оба диска НЕ примонтированны, но iostat показывает, что чтение/запись на диски есть и немного прыгает скорость (64-66 Мб/с). По lsof /mnt/hdd2 и lsof +f -- /mnt/hdd2, где у меня примонтирован диск для бекапа, ничего не выводит.
Я категорически не хочу терять данные на этих дисках. Каким мне сейчас образом до выполнения ресета убедиться, что бекап прошел нормально и фс не пострадала?
P.S. Причина зависания, судя по dmesg, в драйвере amdgpu.
[ 5251.263304] ------------[ cut here ]------------
[ 5251.263440] WARNING: CPU: 19 PID: 1894 at drivers/gpu/drm/amd/amdgpu/../display/dc/dcn20/dcn20_resource.c:3240 dcn20_validate_bandwidth_fp+0x8d/0xd0 [amdgpu]
[ 5251.263442] Modules linked in: fuse tun nft_counter nft_limit nft_ct nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 nf_tables nfnetlink efivarfs ipv6 nls_iso8859_1 nls_cp437 vfat fat snd_hda_codec_realtek snd_hda_codec_generic ledtrig_audio snd_hda_codec_hdmi snd_hda_intel snd_intel_dspcfg soundwire_intel amdgpu soundwire_generic_allocation btusb snd_soc_core btrtl iommu_v2 btbcm btintel gpu_sched snd_compress kvm_amd iwlmvm snd_pcm_dmaengine ttm ccp soundwire_cadence snd_usb_audio bluetooth snd_hda_codec snd_usbmidi_lib drm_kms_helper ecdh_generic mac80211 kvm ecc snd_hda_core snd_rawmidi snd_hwdep snd_seq_device drm irqbypass snd_pcm iwlwifi snd_timer crct10dif_pclmul crc32_pclmuljoydev evdev mc agpgart ghash_clmulni_intel igb fb_sys_fops snd syscopyarea soundcore sysfillrect dca rapl ac97_bus sysimgblt wmi_bmof k10temp i2c_piix4 cfg80211 i2c_algo_bit button acpi_cpufreq loop algif_skcipher af_alg ext4 mbcache jbd2 hid_multitouch hid_microsoft hid_lenovo hid_logitech_hidpp hid_logitech_dj
[ 5251.263495] hid_logitech hid_cherry hid_asus asus_wmi battery sparse_keymap rfkill wmi video hid_generic i2c_hid i2c_core usbhid hid uhci_hcd ohci_pci ehci_pci ohci_hcd ehci_hcd xhci_pci xhci_pci_renesas xhci_hcd usb_storage
[ 5251.263510] CPU: 19 PID: 1894 Comm: Xorg Not tainted 5.10.4 #1
[ 5251.263512] Hardware name: Micro-Star International Co., Ltd. MS-7B93/MPG X570 GAMING PRO CARBON WIFI (MS-7B93),BIOS 1.70 07/14/2020
[ 5251.263635] RIP: 0010:dcn20_validate_bandwidth_fp+0x8d/0xd0 [amdgpu]
[ 5251.263637] Code: 00 7b 35 22 85 14 1f 00 00 75 2f 31 d2 f2 0f 11 85 58 26 00 00 48 89 ee 4c 89 e7 e8 6d f6 ff ff 89 c2 22 95 14 1f 00 00 75 30 <0f> 0b 48 89 9d 58 26 00 00 5b 5d 41 5c c3 75 c9 48 89 9d 58 26 00
[ 5251.263639] RSP: 0018:ffffa6240150fb18 EFLAGS: 00010246
[ 5251.263641] RAX: 0000000000000001 RBX: 4079400000000000 RCX: 00000000000849b8
[ 5251.263642] RDX: 0000000000000000 RSI: 0e9b156e9c4f1265 RDI: 000000000002d1a0
[ 5251.263643] RBP: ffff8c8d878c0000 R08: ffff8c8e194ed000 R09: ffff8c8e90d80000
[ 5251.263644] R10: ffff8c8e194ed000 R11: 0000000100000001 R12: ffff8c8e90d80000
[ 5251.263645] R13: 0000000000000000 R14: ffff8c8e85176000 R15: ffff8c8d878c0000
[ 5251.263647] FS: 00007fc929e5b900(0000) GS:ffff8c958ecc0000(0000) knlGS:0000000000000000
[ 5251.263648] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 5251.263649] CR2: 00000000006225a8 CR3: 0000000113c60000 CR4: 0000000000350ee0
[ 5251.263650] Call Trace:
[ 5251.263771] dcn20_validate_bandwidth+0x24/0x40 [amdgpu]
[ 5251.263884] dc_validate_global_state+0x2f2/0x390 [amdgpu]
[ 5251.263888] ? _cond_resched+0x16/0x40
[ 5251.263891] ? ww_mutex_lock+0x13/0x70
[ 5251.264009] amdgpu_dm_atomic_check+0xb0d/0xc00 [amdgpu]
[ 5251.264032] drm_atomic_check_only+0x55a/0x7d0 [drm]
[ 5251.264052] drm_atomic_commit+0x13/0x50 [drm]
[ 5251.264072] drm_atomic_connector_commit_dpms+0xda/0x100 [drm]
[ 5251.264093] drm_mode_obj_set_property_ioctl+0x189/0x3b0 [drm]
[ 5251.264114] ? drm_connector_set_obj_prop+0x90/0x90 [drm]
[ 5251.264131] drm_connector_property_set_ioctl+0x37/0x60 [drm]
[ 5251.264149] drm_ioctl_kernel+0xaa/0xf0 [drm]
[ 5251.264161] ? ___sys_recvmsg+0x8e/0x100
[ 5251.264179] drm_ioctl+0x20f/0x3a0 [drm]
[ 5251.264225] ? drm_connector_set_obj_prop+0x90/0x90 [drm]
[ 5251.264338] amdgpu_drm_ioctl+0x49/0x80 [amdgpu]
[ 5251.264348] __x64_sys_ioctl+0x83/0xb0
[ 5251.264355] do_syscall_64+0x33/0x80
[ 5251.264359] entry_SYSCALL_64_after_hwframe+0x44/0xa9
[ 5251.264361] RIP: 0033:0x7fc92a5bd417
[ 5251.264363] Code: 00 00 90 48 8b 05 79 2a 0d 00 64 c7 00 26 00 00 00 48 c7 c0 ff ff ff ff c3 66 2e 0f 1f 84 00 00 00 00 00 b8 10 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d 49 2a 0d 00 f7 d8 64 89 01 48
[ 5251.264365] RSP: 002b:00007ffc0774da18 EFLAGS: 00000246 ORIG_RAX: 0000000000000010
[ 5251.264367] RAX: ffffffffffffffda RBX: 00007ffc0774da50 RCX: 00007fc92a5bd417
[ 5251.264368] RDX: 00007ffc0774da50 RSI: 00000000c01064ab RDI: 000000000000000e
[ 5251.264369] RBP: 00000000c01064ab R08: 0000000000000000 R09: 00007fc929dc4cf0
[ 5251.264370] R10: 00007fc929dc4d00 R11: 0000000000000246 R12: 0000000000c723d0
[ 5251.264371] R13: 000000000000000e R14: 0000000000000000 R15: 000000000064b3e0
[ 5251.264373] ---[ end trace b29ae2c09f14cc5f ]---
[21910.378685] amdgpu 0000:2f:00.0: amdgpu: failed to write reg 28b4 wait reg 28c6
[21911.344661] amdgpu 0000:2f:00.0: amdgpu: failed to write reg 1a6f4 wait reg 1a706
[21912.315659] amdgpu 0000:2f:00.0: amdgpu: failed to write reg 28b4 wait reg 28c6
[21913.283665] amdgpu 0000:2f:00.0: amdgpu: failed to write reg 1a6f4 wait reg 1a706
[21914.251659] amdgpu 0000:2f:00.0: amdgpu: failed to write reg 28b4 wait reg 28c6
[21914.511267] [drm:amdgpu_dm_atomic_commit_tail [amdgpu]] *ERROR* Waiting for fences timed out!
[21915.218663] amdgpu 0000:2f:00.0: amdgpu: failed to write reg 1a6f4 wait reg 1a706
[21916.188658] amdgpu 0000:2f:00.0: amdgpu: failed to write reg 28b4 wait reg 28c6
[21917.155657] amdgpu 0000:2f:00.0: amdgpu: failed to write reg 1a6f4 wait reg 1a706
[21918.123663] amdgpu 0000:2f:00.0: amdgpu: failed to write reg 28b4 wait reg 28c6
[21919.090663] amdgpu 0000:2f:00.0: amdgpu: failed to write reg 1a6f4 wait reg 1a706
[21919.631232] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring sdma0 timeout, signaled seq=11820, emitted seq=11822
[21919.631301] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process information: process pid 0 thread pid 0
[21919.631305] amdgpu 0000:2f:00.0: amdgpu: GPU reset begin!
[21919.641236] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring gfx_0.0.0 timeout, signaled seq=1558560, emitted seq=1558562
[21919.641314] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process information: process Xorg pid 1894 thread Xorg:cs0 pid 1895
[21919.641318] amdgpu 0000:2f:00.0: amdgpu: GPU reset begin!
[21919.641320] amdgpu 0000:2f:00.0: amdgpu: Bailing on TDR for s_job:1781f0, as another already in progress
[21920.065652] amdgpu 0000:2f:00.0: amdgpu: failed to write reg 28b4 wait reg 28c6
[21920.065850] BUG: unable to handle page fault for address: ffffa62405fa31c0
[21920.065854] #PF: supervisor write access in kernel mode
[21920.065855] #PF: error_code(0x0002) - not-present page
[21920.065857] PGD 100000067 P4D 100000067 PUD 100182067 PMD 19c171067 PTE 0
[21920.065864] Oops: 0002 [#1] SMP NOPTI
[21920.065867] CPU: 10 PID: 8992 Comm: kworker/10:0 Tainted: G W 5.10.4 #1
[21920.065869] Hardware name: Micro-Star International Co., Ltd. MS-7B93/MPG X570 GAMING PRO CARBON WIFI (MS-7B93),BIOS 1.70 07/14/2020
[21920.065876] Workqueue: events drm_sched_job_timedout [gpu_sched]
[21920.065995] RIP: 0010:amdgpu_device_lock_adev+0x2b/0x83 [amdgpu]
[21920.065998] Code: 1f 44 00 00 31 c0 ba 01 00 00 00 f0 0f b1 97 ac 70 01 00 45 31 c0 85 c0 75 64 53 48 89 fb 48 8d bf b8 70 01 00 e8 4e 07 80 db <f0> ff 83 a0 5c 00 00 48 8b 83 60 07 00 00 48 89 df 48 8b 40 28 e8
[21920.066000] RSP: 0018:ffffa624052a3d90 EFLAGS: 00010246
[21920.066003] RAX: ffff8c8e84952ac0 RBX: ffffa62405f9d520 RCX: 0000000000000001
[21920.066004] RDX: 0000000000000001 RSI: 0000000000000000 RDI: ffffa62405fb45d8
[21920.066006] RBP: 0000000000000000 R08: 0000000000000000 R09: 0000000000000000
[21920.066007] R10: 0000000000000333 R11: 0000000000000000 R12: ffffa62405f9d520
[21920.066009] R13: ffff8c8e85000000 R14: ffff8c8e850126a8 R15: 00000000000000e0
[21920.066011] FS: 0000000000000000(0000) GS:ffff8c958ea80000(0000) knlGS:0000000000000000
[21920.066013] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[21920.066014] CR2: ffffa62405fa31c0 CR3: 000000011a954000 CR4: 0000000000350ee0
[21920.066015] Call Trace:
[21920.066133] amdgpu_device_gpu_recover.cold+0x180/0x95d [amdgpu]
[21920.066244] amdgpu_job_timedout+0x11c/0x140 [amdgpu]
[21920.066279] drm_sched_job_timedout+0x60/0xd0 [gpu_sched]
[21920.066284] process_one_work+0x1d4/0x370
[21920.066320] worker_thread+0x4d/0x3d0
[21920.066327] ? rescuer_thread+0x3d0/0x3d0
[21920.066333] kthread+0x11b/0x140
[21920.066340] ? __kthread_bind_mask+0x60/0x60
[21920.066348] ret_from_fork+0x22/0x30
[21920.066355] Modules linked in: fuse tun nft_counter nft_limit nft_ct nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 nf_tables nfnetlink efivarfs ipv6 nls_iso8859_1 nls_cp437 vfat fat snd_hda_codec_realtek snd_hda_codec_generic ledtrig_audio snd_hda_codec_hdmi snd_hda_intel snd_intel_dspcfg soundwire_intel amdgpu soundwire_generic_allocation btusb snd_soc_core btrtl iommu_v2 btbcm btintel gpu_sched snd_compress kvm_amd iwlmvm snd_pcm_dmaengine ttm ccp soundwire_cadence snd_usb_audio bluetooth snd_hda_codec snd_usbmidi_lib drm_kms_helper ecdh_generic mac80211 kvm ecc snd_hda_core snd_rawmidi snd_hwdep snd_seq_device drm irqbypass snd_pcm iwlwifi snd_timer crct10dif_pclmul crc32_pclmuljoydev evdev mc agpgart ghash_clmulni_intel igb fb_sys_fops snd syscopyarea soundcore sysfillrect dca rapl ac97_bus sysimgblt wmi_bmof k10temp i2c_piix4 cfg80211 i2c_algo_bit button acpi_cpufreq loop algif_skcipher af_alg ext4 mbcache jbd2 hid_multitouch hid_microsoft hid_lenovo hid_logitech_hidpp hid_logitech_dj
[21920.066413] hid_logitech hid_cherry hid_asus asus_wmi battery sparse_keymap rfkill wmi video hid_generic i2c_hid i2c_core usbhid hid uhci_hcd ohci_pci ehci_pci ohci_hcd ehci_hcd xhci_pci xhci_pci_renesas xhci_hcd usb_storage
[21920.066430] CR2: ffffa62405fa31c0
[21920.066433] ---[ end trace b29ae2c09f14cc60 ]---
[21920.066561] RIP: 0010:amdgpu_device_lock_adev+0x2b/0x83 [amdgpu]
[21920.066597] Code: 1f 44 00 00 31 c0 ba 01 00 00 00 f0 0f b1 97 ac 70 01 00 45 31 c0 85 c0 75 64 53 48 89 fb 48 8d bf b8 70 01 00 e8 4e 07 80 db <f0> ff 83 a0 5c 00 00 48 8b 83 60 07 00 00 48 89 df 48 8b 40 28 e8
[21920.066605] RSP: 0018:ffffa624052a3d90 EFLAGS: 00010246
[21920.066614] RAX: ffff8c8e84952ac0 RBX: ffffa62405f9d520 RCX: 0000000000000001
[21920.066615] RDX: 0000000000000001 RSI: 0000000000000000 RDI: ffffa62405fb45d8
[21920.066617] RBP: 0000000000000000 R08: 0000000000000000 R09: 0000000000000000
[21920.066618] R10: 0000000000000333 R11: 0000000000000000 R12: ffffa62405f9d520
[21920.066620] R13: ffff8c8e85000000 R14: ffff8c8e850126a8 R15: 00000000000000e0
[21920.066622] FS: 0000000000000000(0000) GS:ffff8c958ea80000(0000) knlGS:0000000000000000
[21920.066624] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[21920.066625] CR2: ffffa62405fa31c0 CR3: 000000011a954000 CR4: 0000000000350ee0
[21920.317171] [drm:gmc_v10_0_flush_gpu_tlb [amdgpu]] *ERROR* Timeout waiting for VM flush ACK!
[21920.655288] [drm:amdgpu_dm_atomic_commit_tail [amdgpu]] *ERROR* Waiting for fences timed out!
[21925.775279] [drm:amdgpu_dm_atomic_commit_tail [amdgpu]] *ERROR* Waiting for fences timed out!
[21926.065270] [drm:gmc_v10_0_flush_gpu_tlb [amdgpu]] *ERROR* Timeout waiting for VM flush ACK!
[21926.328502] [drm:gmc_v10_0_flush_gpu_tlb [amdgpu]] *ERROR* Timeout waiting for VM flush ACK!
[21926.597331] [drm:gmc_v10_0_flush_gpu_tlb [amdgpu]] *ERROR* Timeout waiting for VM flush ACK!
[21926.849722] [drm:gmc_v10_0_flush_gpu_tlb [amdgpu]] *ERROR* Timeout waiting for VM flush ACK!
[21927.110762] [drm:gmc_v10_0_flush_gpu_tlb [amdgpu]] *ERROR* Timeout waiting for VM flush ACK!
[21927.386196] [drm:gmc_v10_0_flush_gpu_tlb [amdgpu]] *ERROR* Timeout waiting for VM flush ACK!
[21927.657367] [drm:gmc_v10_0_flush_gpu_tlb [amdgpu]] *ERROR* Timeout waiting for VM flush ACK!
[21927.932470] [drm:gmc_v10_0_flush_gpu_tlb [amdgpu]] *ERROR* Timeout waiting for VM flush ACK!
[21928.194553] [drm:gmc_v10_0_flush_gpu_tlb [amdgpu]] *ERROR* Timeout waiting for VM flush ACK!
[21946.441438] [drm:gmc_v10_0_flush_gpu_tlb [amdgpu]] *ERROR* Timeout waiting for VM flush ACK!
[22041.163193] [drm:gmc_v10_0_flush_gpu_tlb [amdgpu]] *ERROR* Timeout waiting for VM flush ACK!
[22100.850295] [drm:gmc_v10_0_flush_gpu_tlb [amdgpu]] *ERROR* Timeout waiting for VM flush ACK!
[22160.560611] [drm:gmc_v10_0_flush_gpu_tlb [amdgpu]] *ERROR* Timeout waiting for VM flush ACK!
[22199.665123] [drm:gmc_v10_0_flush_gpu_tlb [amdgpu]] *ERROR* Timeout waiting for VM flush ACK!
[22258.418095] [drm:gmc_v10_0_flush_gpu_tlb [amdgpu]] *ERROR* Timeout waiting for VM flush ACK!
[22602.645738] [drm:gmc_v10_0_flush_gpu_tlb [amdgpu]] *ERROR* Timeout waiting for VM flush ACK!
[22664.176534] [drm:gmc_v10_0_flush_gpu_tlb [amdgpu]] *ERROR* Timeout waiting for VM flush ACK!
