Package: zfsutils-linux
Followup-For: Bug #1105740

Hello,

The pd value in arcstats does look like it has nearly twice as many digits
in it than it's supposed to, so I'm inclined to agree that it's more likely
a kernel problem than userspace problem. FWIW my zpool is 32TB (3x12TB
disks) and is in the household NAS, which also shares its 32GB RAM with a handful of LXD containers.

As it happens there was a reboot yesterday. Values have returned to being plausible values (for now!):

root@grook:~# arc_summary |grep data\ target
MFU data target: 49.7 % 3.0 GiB
MFU metadata target: 8.8 % 541.1 MiB
MRU data target: 32.7 % 2.0 GiB
MRU metadata target: 8.8 % 541.1 MiB

I'll keep an eye on it. There have been both kernel and zfs updates in the meantime, so I expect there's a possibility that the problem fixed itself.


-- System Information:
Debian Release: 12.11
APT prefers unstable-debug
APT policy: (500, 'unstable-debug'), (500, 'testing-security-debug'), (500, 'testing-proposed-updates-debug'), (500, 'stable-updates'), (500, 'stable-security'), (500, 'stable-debug'), (500, 'oldstable-updates'), (500, 'oldstable-security'), (500, '
stable'), (500, 'oldstable'), (490, 'testing-security'), (490, 'testing-proposed-updates'), (490, 'testing-debug'), (490, 'testing'), (400, 'unstable')
Architecture: amd64 (x86_64)

Kernel: Linux 6.12.33+deb12-amd64 (SMP w/8 CPU threads; PREEMPT)
Kernel taint flags: TAINT_PROPRIETARY_MODULE, TAINT_OOT_MODULE, TAINT_UNSIGNED_MODULE
Locale: LANG=en_AU.UTF-8, LC_CTYPE=en_AU.UTF-8 (charmap=UTF-8), LANGUAGE=en_AU:en
Shell: /bin/sh linked to /usr/bin/dash
Init: systemd (via /run/systemd/system)
LSM: AppArmor: enabled

Versions of packages zfsutils-linux depends on:
ii init-system-helpers 1.65.2
ii libblkid1 2.38.1-5+deb12u3
ii libc6 2.36-9+deb12u10
ii libnvpair3linux 2.3.2-2~bpo12+1
ii libssl3 3.0.16-1~deb12u1
ii libudev1 252.38-1~deb12u1
ii libuuid1 2.38.1-5+deb12u3
ii libuutil3linux 2.3.2-2~bpo12+1
ii libzfs6linux 2.3.2-2~bpo12+1
ii libzpool6linux 2.3.2-2~bpo12+1
ii python3 3.11.2-1+b1

Versions of packages zfsutils-linux recommends:
ii zfs-dkms [zfs-modules] 2.3.2-2~bpo12+1
ii zfs-zed 2.3.2-2~bpo12+1

Versions of packages zfsutils-linux suggests:
ii nfs-kernel-server 1:2.6.2-4+deb12u1
pn samba-common-bin <none>
pn zfs-initramfs | zfs-dracut <none>

-- Configuration Files:
/etc/sudoers.d/zfs [Errno 13] Permission denied: '/etc/sudoers.d/zfs'

-- no debconf information

19 1 0x01 147 39984 4988816601 78162944623909
name type data
hits 4 15464593
iohits 4 698298
misses 4 1121846
demand_data_hits 4 4796322
demand_data_iohits 4 8651
demand_data_misses 4 252261
demand_metadata_hits 4 9901679
demand_metadata_iohits 4 2533
demand_metadata_misses 4 162221
prefetch_data_hits 4 125364
prefetch_data_iohits 4 852
prefetch_data_misses 4 657114
prefetch_metadata_hits 4 641228
prefetch_metadata_iohits 4 686262
prefetch_metadata_misses 4 50250
mru_hits 4 3731768
mru_ghost_hits 4 62342
mfu_hits 4 11732825
mfu_ghost_hits 4 71782
uncached_hits 4 0
deleted 4 534214
mutex_miss 4 7
access_skip 4 0
evict_skip 4 53
evict_not_enough 4 0
evict_l2_cached 4 0
evict_l2_eligible 4 80377389568
evict_l2_eligible_mfu 4 47247125504
evict_l2_eligible_mru 4 33130264064
evict_l2_ineligible 4 15804665856
evict_l2_skip 4 0
hash_elements 4 278610
hash_elements_max 4 278610
hash_collisions 4 78990
hash_chains 4 8800
hash_chain_max 4 4
meta 4 755768796
pd 4 1702888376
pm 4 2147483648
c 4 18061591104
c_min 4 1048701312
c_max 4 32484700160
size 4 7001644968
compressed_size 4 5281390080
uncompressed_size 4 7883245056
overhead_size 4 1135070208
hdr_size 4 68083296
data_size 4 5115185664
metadata_size 4 1301274624
dbuf_size 4 124183808
dnode_size 4 288059016
bonus_size 4 95839680
anon_size 4 0
anon_data 4 0
anon_metadata 4 0
anon_evictable_data 4 0
anon_evictable_metadata 4 0
mru_size 4 3785680384
mru_data 4 2994095616
mru_metadata 4 791584768
mru_evictable_data 4 2639451648
mru_evictable_metadata 4 406935552
mru_ghost_size 4 2661256192
mru_ghost_data 4 2661256192
mru_ghost_metadata 4 0
mru_ghost_evictable_data 4 2661256192
mru_ghost_evictable_metadata 4 0
mfu_size 4 2630779904
mfu_data 4 2121090048
mfu_metadata 4 509689856
mfu_evictable_data 4 1627361792
mfu_evictable_metadata 4 97701888
mfu_ghost_size 4 1190528000
mfu_ghost_data 4 1190528000
mfu_ghost_metadata 4 0
mfu_ghost_evictable_data 4 1190528000
mfu_ghost_evictable_metadata 4 0
uncached_size 4 0
uncached_data 4 0
uncached_metadata 4 0
uncached_evictable_data 4 0
uncached_evictable_metadata 4 0
l2_hits 4 0
l2_misses 4 0
l2_prefetch_asize 4 0
l2_mru_asize 4 0
l2_mfu_asize 4 0
l2_bufc_data_asize 4 0
l2_bufc_metadata_asize 4 0
l2_feeds 4 0
l2_rw_clash 4 0
l2_read_bytes 4 0
l2_write_bytes 4 0
l2_writes_sent 4 0
l2_writes_done 4 0
l2_writes_error 4 0
l2_writes_lock_retry 4 0
l2_evict_lock_retry 4 0
l2_evict_reading 4 0
l2_evict_l1cached 4 0
l2_free_on_write 4 0
l2_abort_lowmem 4 0
l2_cksum_bad 4 0
l2_io_error 4 0
l2_size 4 0
l2_asize 4 0
l2_hdr_size 4 0
l2_log_blk_writes 4 0
l2_log_blk_avg_asize 4 0
l2_log_blk_asize 4 0
l2_log_blk_count 4 0
l2_data_to_meta_ratio 4 0
l2_rebuild_success 4 0
l2_rebuild_unsupported 4 0
l2_rebuild_io_errors 4 0
l2_rebuild_dh_errors 4 0
l2_rebuild_cksum_lb_errors 4 0
l2_rebuild_lowmem 4 0
l2_rebuild_size 4 0
l2_rebuild_asize 4 0
l2_rebuild_bufs 4 0
l2_rebuild_bufs_precached 4 0
l2_rebuild_log_blks 4 0
memory_throttle_count 4 0
memory_direct_count 4 6581
memory_indirect_count 4 130159
memory_all_bytes 4 33558441984
memory_free_bytes 4 19348508672
memory_available_bytes 3 18123950720
arc_no_grow 4 0
arc_tempreserve 4 0
arc_loaned_bytes 4 0
arc_prune 4 0
arc_meta_used 4 1877440424
arc_dnode_limit 4 3248470016
async_upgrade_sync 4 9557
predictive_prefetch 4 2161053
demand_hit_predictive_prefetch 4 572689
demand_iohit_predictive_prefetch 4 18417
prescient_prefetch 4 17
demand_hit_prescient_prefetch 4 14
demand_iohit_prescient_prefetch 4 3
arc_need_free 4 0
arc_sys_free 4 1224557952
arc_raw_size 4 0
cached_only_in_progress 4 0
abd_chunk_waste_size 4 9018880

------------------------------------------------------------------------
ZFS Subsystem Report Thu Jul 31 10:16:07 2025
Linux 6.12.33+deb12-amd64 2.3.2-2~bpo12+1 Machine: grook (x86_64) 2.3.2-2~bpo12+1

ARC status:
Total memory size: 31.3 GiB
Min target size: 3.1 % 1000.1 MiB
Max target size: 96.8 % 30.3 GiB
Target size (adaptive): 21.5 % 16.8 GiB
Current size: 21.5 % 6.5 GiB
Free memory size: 18.0 GiB
Available memory size: 16.9 GiB

ARC structural breakdown (current size): 6.5 GiB
Compressed size: 75.4 % 4.9 GiB
Overhead size: 16.2 % 1.1 GiB
Bonus size: 1.4 % 91.4 MiB
Dnode size: 4.1 % 274.7 MiB
Dbuf size: 1.8 % 118.4 MiB
Header size: 1.0 % 64.9 MiB
L2 header size: 0.0 % 0 Bytes
ABD chunk waste size: 0.1 % 8.6 MiB

ARC types breakdown (compressed + overhead): 6.0 GiB
Data size: 79.7 % 4.8 GiB
Metadata size: 20.3 % 1.2 GiB

ARC states breakdown (compressed + overhead): 6.0 GiB
Anonymous data size: 0.0 % 0 Bytes
Anonymous metadata size: 0.0 % 0 Bytes
MFU data target: 49.7 % 3.0 GiB
MFU data size: 33.1 % 2.0 GiB
MFU evictable data size: 25.4 % 1.5 GiB
MFU ghost data size: 1.1 GiB
MFU metadata target: 8.8 % 537.8 MiB
MFU metadata size: 8.0 % 486.1 MiB
MFU evictable metadata size: 1.5 % 93.2 MiB
MFU ghost metadata size: 0 Bytes
MRU data target: 32.7 % 2.0 GiB
MRU data size: 46.6 % 2.8 GiB
MRU evictable data size: 41.1 % 2.5 GiB
MRU ghost data size: 2.5 GiB
MRU metadata target: 8.8 % 537.8 MiB
MRU metadata size: 12.4 % 754.9 MiB
MRU evictable metadata size: 6.3 % 388.1 MiB
MRU ghost metadata size: 0 Bytes
Uncached data size: 0.0 % 0 Bytes
Uncached metadata size: 0.0 % 0 Bytes

ARC hash breakdown:
Elements: 278.6k
Collisions: 79.0k
Chain max: 4
Chains: 8.8k

ARC misc:
Memory throttles: 0
Memory direct reclaims: 6581
Memory indirect reclaims: 130159
Deleted: 534.2k
Mutex misses: 7
Eviction skips: 53
Eviction skips due to L2 writes: 0
L2 cached evictions: 0 Bytes
L2 eligible evictions: 74.9 GiB
L2 eligible MFU evictions: 58.8 % 44.0 GiB
L2 eligible MRU evictions: 41.2 % 30.9 GiB
L2 ineligible evictions: 14.7 GiB

ARC total accesses: 17.3M
Total hits: 89.5 % 15.5M
Total I/O hits: 4.0 % 698.3k
Total misses: 6.5 % 1.1M

ARC demand data accesses: 29.3 % 5.1M
Demand data hits: 94.8 % 4.8M
Demand data I/O hits: 0.2 % 8.6k
Demand data misses: 5.0 % 252.2k

ARC demand metadata accesses: 58.2 % 10.1M
Demand metadata hits: 98.4 % 9.9M
Demand metadata I/O hits: < 0.1 % 2.5k
Demand metadata misses: 1.6 % 162.2k

ARC prefetch data accesses: 4.5 % 783.3k
Prefetch data hits: 16.0 % 125.4k
Prefetch data I/O hits: 0.1 % 852
Prefetch data misses: 83.9 % 657.1k

ARC prefetch metadata accesses: 8.0 % 1.4M
Prefetch metadata hits: 46.5 % 641.2k
Prefetch metadata I/O hits: 49.8 % 686.3k
Prefetch metadata misses: 3.6 % 50.2k

ARC predictive prefetches: 100.0 % 2.2M
Demand hits after predictive: 26.5 % 572.7k
Demand I/O hits after predictive: 0.9 % 18.4k
Never demanded after predictive: 72.6 % 1.6M

ARC prescient prefetches: < 0.1 % 17
Demand hits after prescient: 82.4 % 14
Demand I/O hits after prescient: 17.6 % 3
Never demanded after prescient: 0.0 % 0

ARC states hits of all accesses:
Most frequently used (MFU): 67.9 % 11.7M
Most recently used (MRU): 21.6 % 3.7M
Most frequently used (MFU) ghost: 0.4 % 71.8k
Most recently used (MRU) ghost: 0.4 % 62.3k
Uncached: 0.0 % 0

DMU predictive prefetcher calls: 5.0M
Stream hits: 62.3 % 3.1M
Hits ahead of stream: 28.0 % 1.4M
Hits behind stream: 4.3 % 212.9k
Stream misses: 5.4 % 269.9k
Streams limit reached: 77.7 % 209.7k
Stream strides: 15.4k
Prefetches issued 786.3k

L2ARC not detected, skipping section

Solaris Porting Layer (SPL):
spl_hostid 0
spl_hostid_path /etc/hostid
spl_kmem_alloc_max 1048576
spl_kmem_alloc_warn 65536
spl_kmem_cache_kmem_threads 4
spl_kmem_cache_magazine_size 0
spl_kmem_cache_max_size 32
spl_kmem_cache_obj_per_slab 8
spl_kmem_cache_slab_limit 16384
spl_panic_halt 0
spl_schedule_hrtimeout_slack_us 0
spl_taskq_kick 0
spl_taskq_thread_bind 0
spl_taskq_thread_dynamic 1
spl_taskq_thread_priority 1
spl_taskq_thread_sequential 4
spl_taskq_thread_timeout_ms 5000

Tunables:
brt_zap_default_bs 12
brt_zap_default_ibs 12
brt_zap_prefetch 1
dbuf_cache_hiwater_pct 10
dbuf_cache_lowater_pct 10
dbuf_cache_max_bytes 18446744073709551615
dbuf_cache_shift 5
dbuf_metadata_cache_max_bytes 18446744073709551615
dbuf_metadata_cache_shift 6
dbuf_mutex_cache_shift 0
ddt_zap_default_bs 15
ddt_zap_default_ibs 15
dmu_ddt_copies 0
dmu_object_alloc_chunk_shift 7
dmu_prefetch_max 134217728
icp_aes_impl cycle [fastest] generic x86_64 aesni
icp_gcm_avx_chunk_size 32736
icp_gcm_impl cycle [fastest] avx generic pclmulqdq
ignore_hole_birth 1
l2arc_exclude_special 0
l2arc_feed_again 1
l2arc_feed_min_ms 200
l2arc_feed_secs 1
l2arc_headroom 8
l2arc_headroom_boost 200
l2arc_meta_percent 33
l2arc_mfuonly 0
l2arc_noprefetch 1
l2arc_norw 0
l2arc_rebuild_blocks_min_l2size 1073741824
l2arc_rebuild_enabled 1
l2arc_trim_ahead 0
l2arc_write_boost 33554432
l2arc_write_max 33554432
metaslab_aliquot 1048576
metaslab_bias_enabled 1
metaslab_debug_load 0
metaslab_debug_unload 0
metaslab_df_max_search 16777216
metaslab_df_use_largest_segment 0
metaslab_force_ganging 16777217
metaslab_force_ganging_pct 3
metaslab_fragmentation_factor_enabled 1
metaslab_lba_weighting_enabled 1
metaslab_preload_enabled 1
metaslab_preload_limit 10
metaslab_preload_pct 50
metaslab_unload_delay 32
metaslab_unload_delay_ms 600000
raidz_expand_max_copy_bytes 167772160
raidz_expand_max_reflow_bytes 0
raidz_io_aggregate_rows 4
send_holes_without_birth_time 1
spa_asize_inflation 24
spa_config_path /etc/zfs/zpool.cache
spa_cpus_per_allocator 4
spa_load_print_vdev_tree 0
spa_load_verify_data 1
spa_load_verify_metadata 1
spa_load_verify_shift 4
spa_num_allocators 4
spa_slop_shift 5
spa_upgrade_errlog_limit 0
vdev_file_logical_ashift 9
vdev_file_physical_ashift 9
vdev_removal_max_span 32768
vdev_validate_skip 0
zap_iterate_prefetch 1
zap_micro_max_size 131072
zap_shrink_enabled 1
zfetch_hole_shift 2
zfetch_max_distance 67108864
zfetch_max_idistance 134217728
zfetch_max_reorder 16777216
zfetch_max_sec_reap 2
zfetch_max_streams 8
zfetch_min_distance 4194304
zfetch_min_sec_reap 1
zfs_abd_scatter_enabled 1
zfs_abd_scatter_max_order 9
zfs_abd_scatter_min_size 1536
zfs_active_allocator dynamic
zfs_admin_snapshot 0
zfs_allow_redacted_dataset_mount 0
zfs_arc_average_blocksize 8192
zfs_arc_dnode_limit 0
zfs_arc_dnode_limit_percent 10
zfs_arc_dnode_reduce_percent 10
zfs_arc_evict_batch_limit 10
zfs_arc_eviction_pct 200
zfs_arc_grow_retry 0
zfs_arc_lotsfree_percent 10
zfs_arc_max 0
zfs_arc_meta_balance 500
zfs_arc_min 0
zfs_arc_min_prefetch_ms 0
zfs_arc_min_prescient_prefetch_ms 0
zfs_arc_pc_percent 0
zfs_arc_prune_task_threads 1
zfs_arc_shrink_shift 0
zfs_arc_shrinker_limit 0
zfs_arc_shrinker_seeks 2
zfs_arc_sys_free 0
zfs_async_block_max_blocks 18446744073709551615
zfs_autoimport_disable 1
zfs_bclone_enabled 1
zfs_bclone_wait_dirty 0
zfs_blake3_impl cycle [fastest] generic sse2 sse41 avx2
zfs_btree_verify_intensity 0
zfs_checksum_events_per_second 20
zfs_commit_timeout_pct 10
zfs_compressed_arc_enabled 1
zfs_condense_indirect_commit_entry_delay_ms 0
zfs_condense_indirect_obsolete_pct 25
zfs_condense_indirect_vdevs_enable 1
zfs_condense_max_obsolete_bytes 1073741824
zfs_condense_min_mapping_bytes 131072
zfs_dbgmsg_enable 1
zfs_dbgmsg_maxsize 4194304
zfs_dbuf_state_index 0
zfs_ddt_data_is_special 1
zfs_deadman_checktime_ms 60000
zfs_deadman_enabled 1
zfs_deadman_events_per_second 1
zfs_deadman_failmode wait
zfs_deadman_synctime_ms 600000
zfs_deadman_ziotime_ms 300000
zfs_dedup_log_flush_entries_min 1000
zfs_dedup_log_flush_flow_rate_txgs 10
zfs_dedup_log_flush_min_time_ms 1000
zfs_dedup_log_flush_passes_max 8
zfs_dedup_log_mem_max 335584419
zfs_dedup_log_mem_max_percent 1
zfs_dedup_log_txg_max 8
zfs_dedup_prefetch 0
zfs_default_bs 9
zfs_default_ibs 17
zfs_delay_min_dirty_percent 60
zfs_delay_scale 500000
zfs_delete_blocks 20480
zfs_dio_enabled 1
zfs_dio_write_verify_events_per_second 20
zfs_dirty_data_max 3355844198
zfs_dirty_data_max_max 4294967296
zfs_dirty_data_max_max_percent 25
zfs_dirty_data_max_percent 10
zfs_dirty_data_sync_percent 20
zfs_disable_ivset_guid_check 0
zfs_dmu_offset_next_sync 1
zfs_embedded_slog_min_ms 64
zfs_expire_snapshot 300
zfs_fallocate_reserve_percent 110
zfs_flags 0
zfs_fletcher_4_impl [fastest] scalar superscalar superscalar4 sse2 ssse3 avx2
zfs_free_bpobj_enabled 1
zfs_free_leak_on_eio 0
zfs_free_min_time_ms 1000
zfs_history_output_max 1048576
zfs_immediate_write_sz 32768
zfs_initialize_chunk_size 1048576
zfs_initialize_value 16045690984833335022
zfs_keep_log_spacemaps_at_export 0
zfs_key_max_salt_uses 400000000
zfs_livelist_condense_new_alloc 0
zfs_livelist_condense_sync_cancel 0
zfs_livelist_condense_sync_pause 0
zfs_livelist_condense_zthr_cancel 0
zfs_livelist_condense_zthr_pause 0
zfs_livelist_max_entries 500000
zfs_livelist_min_percent_shared 75
zfs_lua_max_instrlimit 100000000
zfs_lua_max_memlimit 104857600
zfs_max_async_dedup_frees 100000
zfs_max_dataset_nesting 50
zfs_max_log_walking 5
zfs_max_logsm_summary_length 10
zfs_max_missing_tvds 0
zfs_max_nvlist_src_size 0
zfs_max_recordsize 16777216
zfs_metaslab_find_max_tries 100
zfs_metaslab_fragmentation_threshold 77
zfs_metaslab_max_size_cache_sec 3600
zfs_metaslab_mem_limit 25
zfs_metaslab_segment_weight_enabled 1
zfs_metaslab_switch_threshold 2
zfs_metaslab_try_hard_before_gang 0
zfs_mg_fragmentation_threshold 95
zfs_mg_noalloc_threshold 0
zfs_min_metaslabs_to_flush 1
zfs_multihost_fail_intervals 10
zfs_multihost_history 0
zfs_multihost_import_intervals 20
zfs_multihost_interval 1000

[continued in next message]

--- SoupGate-Win32 v1.05
* Origin: fsxNet Usenet Gateway (21:1/5)