zilstat not working. FreeNAS 11 RC1 through 4, on fresh UEFI build

Status
Not open for further replies.

skek

Dabbler
Joined
Mar 29, 2017
Messages
10
Hi,

Not sure if others have seen this. I note previous bugs from a few years back with same symptoms implying it was something to do with EFI grub. This is a new build, fresh install of FreeNAS 11 RC and subsequently upgraded on every RC to current RC4. Boot disk is an Intel 600P M2 PCIE installed in a SM X11SSH-LN4F onboard M2 slot. FreeNAS fresh install in UEFI mode.

Not sure if this is a bug, or a quirk, so any ideas welcome.

Running zilstat gives an error:
Code:
dtrace: invalid probe specifier

then dumps the dtrace script followed by
Code:
: in action list: index 1 is out of range for fbt::zil_lwb_write_start:entry args[ ]


Full output below:
Code:
root@vault:~ # zilstat
dtrace: invalid probe specifier
#pragma D option quiet
 inline int OPT_time = 0;
 inline int OPT_txg = 0;
 inline int OPT_pool = 0;
 inline int OPT_mega = 0;
 inline int INTERVAL = 1;
 inline int LINES = -1;
 inline int COUNTER = -1;
 inline int FILTER = 0;
 inline string POOL = "";
 dtrace:::BEGIN
 {
  /* starting values */
  MEGA = 1000000;
  counts = COUNTER;
  secs = INTERVAL;
  interval = INTERVAL;
  interval == 0 ? interval++ : 1;
  line = 0;
  last_event[""] = 0;
  nused=0;
  nused_max_per_sec=0;
  nused_per_sec=0;
  size=0;
  size_max_per_sec=0;
  size_per_sec=0;
  syncops=0;
  size_4k=0;
  size_4k_32k=0;
  size_32k=0;
  OPT_txg ? printf("waiting for txg commit...\n") : 1;
 }

 /*
  * collect info when zil_lwb_write_start fires
  */
fbt::zil_lwb_write_start:entry
/OPT_pool == 0 || POOL == args[0]->zl_dmu_pool->dp_spa->spa_name/
{
  nused += args[1]->lwb_nused;
  nused_per_sec += args[1]->lwb_nused;
  size += args[1]->lwb_sz;
  size_per_sec += args[1]->lwb_sz;
  syncops++;
  args[1]->lwb_sz <= 4096 ? size_4k++ : 1;
  args[1]->lwb_sz > 4096 && args[1]->lwb_sz < 32768 ? size_4k_32k++ : 1;
  args[1]->lwb_sz >= 32768 ? size_32k++ : 1;
}

/*
 * Timer
 */
profile:::tick-1sec
{
   OPT_txg ? secs++ : secs--;
  nused_per_sec > nused_max_per_sec ? nused_max_per_sec = nused_per_sec : 1;
  nused_per_sec = 0;
  size_per_sec > size_max_per_sec ? size_max_per_sec = size_per_sec : 1;
  size_per_sec = 0;
}

/*
 * Print header
 */
profile:::tick-1sec
/OPT_txg == 0 && line == 0/
{
   /* print optional headers */
   OPT_time  ? printf("%-20s ", "TIME")  : 1;

   /* print header */
  OPT_mega  ? printf("%10s %10s %10s %10s %10s %10s",
  "N-MB", "N-MB/s", "N-Max-Rate",
  "B-MB", "B-MB/s", "B-Max-Rate") :
  printf("%10s %10s %10s %10s %10s %10s",
  "N-Bytes", "N-Bytes/s", "N-Max-Rate",
  "B-Bytes", "B-Bytes/s", "B-Max-Rate");
  printf(" %6s %6s %6s %6s\n",
  "ops", "<=4kB", "4-32kB", ">=32kB");
   line = LINES;
}

 fbt::txg_quiesce:entry
 /OPT_txg == 1 && POOL == args[0]->dp_spa->spa_name && line == 0/
 {
   OPT_time  ? printf("%-20s ", "TIME")  : 1;

  OPT_mega  ? printf("%10s %10s %10s %10s %10s %10s %10s",
  "txg", "N-MB", "N-MB/s", "N-Max-Rate",
  "B-MB", "B-MB/s", "B-Max-Rate") :
  printf("%10s %10s %10s %10s %10s %10s %10s",
  "txg", "N-Bytes", "N-Bytes/s", "N-Max-Rate",
  "B-Bytes", "B-Bytes/s", "B-Max-Rate");
  printf(" %6s %6s %6s %6s\n",
  "ops", "<=4kB", "4-32kB", ">=32kB");
   line = LINES;
}

 /*
  * Print Output
  */
 profile:::tick-1sec
 /OPT_txg == 0 && secs == 0/
 {
   OPT_time  ? printf("%-20Y ", walltimestamp) : 1;
  OPT_mega  ?
  printf("%10d %10d %10d %10d %10d %10d",
  nused/MEGA, nused/(interval*MEGA), nused_max_per_sec/MEGA,
  size/MEGA, size/(interval*MEGA), size_max_per_sec/MEGA) :
  printf("%10d %10d %10d %10d %10d %10d",
  nused, nused/interval, nused_max_per_sec,
  size, size/interval, size_max_per_sec);
  printf(" %6d %6d %6d %6d\n",
  syncops, size_4k, size_4k_32k, size_32k);
  nused = 0;
  nused_per_sec = 0;
  nused_max_per_sec = 0;
  size=0;
  size_max_per_sec=0;
  size_per_sec=0;
  syncops=0;
  size_4k=0;
  size_4k_32k=0;
  size_32k=0;
  secs = INTERVAL;
   counts--;
   line--;
 }

fbt::txg_quiesce:entry
/OPT_txg == 1 && POOL == args[0]->dp_spa->spa_name/
{
  secs <= 0 ? secs=1 : 1;
   OPT_time ? printf("%-20Y ", walltimestamp) : 1;
  OPT_mega ?
  printf("%10d %10d %10d %10d %10d %10d %10d", args[1],
  nused/MEGA, nused/(secs*MEGA), nused_max_per_sec/MEGA,
  size/MEGA, size/(secs*MEGA), size_max_per_sec/MEGA) :
  printf("%10d %10d %10d %10d %10d %10d %10d", args[1],
  nused, nused/secs, nused_max_per_sec,
  size, size/secs, size_max_per_sec);
  printf(" %6d %6d %6d %6d\n",
  syncops, size_4k, size_4k_32k, size_32k);
  nused = 0;
  nused_per_sec = 0;
  nused_max_per_sec = 0;
  size=0;
  size_max_per_sec=0;
  size_per_sec=0;
  syncops=0;
  size_4k=0;
  size_4k_32k=0;
  size_32k=0;
  secs = 0;
   counts--;
   line--;
 }

 /*
  * End of program
  */
 profile:::tick-1sec
 /OPT_txg == 0 && counts == 0/
 {
   exit(0);
 }
 fbt::txg_quiesce:entry
 /OPT_txg == 1 && counts == 0/
 {
  exit(0);
 }
: in action list: index 1 is out of range for fbt::zil_lwb_write_start:entry args[ ]
 

joeschmuck

Old Man
Moderator
Joined
May 28, 2011
Messages
10,994
My test system running FN-11 RC4 is a BIOS install and zilstat returns a normal responce.

Someone here on the forums must have a UEFI install that they can verify your findings, so if you do, please chime in.

Regardless, I'd report it as a bug, zilstat shouldn't be tossing out an error message.
 

blaco

Explorer
Joined
Apr 28, 2015
Messages
53
I can confirm the bug on RC4 and UEFI on Supermicro X10 board.
Code:
/*																			 
  * End of program															 
  */																			
profile:::tick-1sec															
/OPT_txg == 0 && counts == 0/												 
{																			 
		exit(0);																
}																			 
fbt::txg_quiesce:entry														 
/OPT_txg == 1 && counts == 0/												 
{																			 
	exit(0);																	
}																			 
: in action list: index 1 is out of range for fbt::zil_lwb_write_start:entry arg
s[ ]					   
 

joeschmuck

Old Man
Moderator
Joined
May 28, 2011
Messages
10,994
Please create a bug report, this should not be happening in a RC.
 

ashfixit

Cadet
Joined
Jun 13, 2017
Messages
4
Would someone please run the following on one of these systems?
dtrace -ln 'fbt::zil*:entry'
 

blaco

Explorer
Joined
Apr 28, 2015
Messages
53
Sorry I'm only on mobile so I can only post it here not in the bug tracker.

Would someone please run the following on one of these systems?
dtrace -ln 'fbt::zil*:entry'
Code:
Last login: Tue Jun 13 18:00:19 2017 from 192.168.123.201
FreeBSD 11.0-STABLE (FreeNAS.amd64) #0 r313908+f4b711d1be8(freenas/11.0-stable): Mon Jun  5 23:11:22 UTC 2017

		FreeNAS (c) 2009-2017, The FreeNAS Development Team
		All rights reserved.
		FreeNAS is released under the modified BSD license.

		For more information, documentation, help or support, go here:
		http://freenas.org
Welcome to FreeNAS
root@sauron:~ # dtrace -ln 'fbt::zil*:entry'
   ID   PROVIDER			MODULE						  FUNCTION NAME
 2331		fbt			kernel				  zil_aitx_compare entry
 2333		fbt			kernel					 zil_alloc_lwb entry
 2335		fbt			kernel					zil_bp_compare entry
 2337		fbt			kernel			   zil_claim_log_block entry
 2339		fbt			kernel			  zil_claim_log_record entry
 2341		fbt			kernel				zil_free_log_block entry
 2343		fbt			kernel			   zil_free_log_record entry
 2345		fbt			kernel					 zil_incr_blks entry
 2347		fbt			kernel					zil_itxg_clean entry
 2349		fbt			kernel				zil_lwb_write_done entry
 2350		fbt			kernel				zil_lwb_write_init entry
 2352		fbt			kernel			   zil_lwb_write_start entry
 2354		fbt			kernel				 zil_read_log_data entry
 2356		fbt			kernel			 zil_replay_log_record entry
 2358		fbt			kernel				  zil_vdev_compare entry
27029		fbt			kernel					   zil_suspend entry
28441		fbt			kernel						 zil_close entry
30962		fbt			kernel				  zil_vdev_offline entry
32118		fbt			kernel			   zil_check_log_chain entry
33095		fbt			kernel						 zil_claim entry
33783		fbt			kernel						zil_resume entry
34045		fbt			kernel					zilog_is_dirty entry
34082		fbt			kernel						  zil_fini entry
34166		fbt			kernel						 zil_alloc entry
34312		fbt			kernel				 zil_async_to_sync entry
34421		fbt			kernel				   zil_bp_tree_add entry
35895		fbt			kernel					  zil_set_sync entry
36272		fbt			kernel					   zilog_dirty entry
36931		fbt			kernel						  zil_open entry
40318		fbt			kernel			 zilog_is_dirty_in_txg entry
40418		fbt			kernel					zil_itx_assign entry
42478		fbt			kernel					 zil_replaying entry
42482		fbt			kernel				  zil_destroy_sync entry
43118		fbt			kernel					zil_itx_create entry
44064		fbt			kernel						  zil_free entry
44114		fbt			kernel						zil_commit entry
47375		fbt			kernel					 zil_add_block entry
48183		fbt			kernel						 zil_clean entry
49329		fbt			kernel				   zil_itx_destroy entry
50460		fbt			kernel						zil_replay entry
53109		fbt			kernel						  zil_init entry
54206		fbt			kernel					   zil_destroy entry
54468		fbt			kernel				   zil_set_logbias entry
54989		fbt			kernel						 zil_parse entry
60383		fbt			kernel						  zil_sync entry
root@sauron:~ #
 

ashfixit

Cadet
Joined
Jun 13, 2017
Messages
4
Does anyone feel like giving me root access to a system that's sick with this? PM me.
 

blaco

Explorer
Joined
Apr 28, 2015
Messages
53
Sorry don't want to give root access to my production machine, but I tested it with VirtualBox and Freenas 11 Release (was a work of 5 minutes).
Create a Virtual Box Host (FreeBSD-64 bit - bridged networking - and turned on Efi under "System").
I installed it and tried the zilstat command and I got the same error.

If you want I can share the VM, but I think it will be faster to create it yourself.

EDIT: Thought about why using Virtualbox.... and so i did the same test with the Freenas VM feature.
Create a normal UEFI bhyve VM with the Freenas 11 GUI - install it over VNC - and run zilstat and you will get the same traceback :)
 
Last edited:

joeschmuck

Old Man
Moderator
Joined
May 28, 2011
Messages
10,994
Does anyone feel like giving me root access to a system that's sick with this? PM me.
Also, it would be unwise to give someone root access when they have no history on this forum. You have been a member for 1 day. It doesn't matter how good your intentions are, you just don't have the credibility here yet.

There are probably 5 people on this forum in which I would allow root access to my system and they have proven to be reliable and responsible. And I would never ask to log into someones system, to me that is a red flag.
 

ashfixit

Cadet
Joined
Jun 13, 2017
Messages
4
Too right; however in this case the ticket for fixing dtrace is assigned to me and I work in support at iX; wading around in customers' OSs' is not something I'm casual about.

Defcon --;

I have reproduced the behaviour under Virtualbox; I'm documenting the work within the ticket.
 

DavidPili

Cadet
Joined
Sep 10, 2015
Messages
1
This is reproducible in 9.10 as well, started building a production server around a uefi install and went back to bios over it.
 

tazinblack

Explorer
Joined
Apr 12, 2013
Messages
77
Same problem here with FreeNAS-11.1-U5 BIOS installation :confused:

Code:
dtrace -ln 'fbt::zil*:entry'
   ID   PROVIDER			MODULE						  FUNCTION NAME
 2449		fbt			kernel				  zil_aitx_compare entry
 2451		fbt			kernel					 zil_alloc_lwb entry
 2453		fbt			kernel					zil_bp_compare entry
 2455		fbt			kernel			   zil_claim_log_block entry
 2457		fbt			kernel			  zil_claim_log_record entry
 2459		fbt			kernel				zil_free_log_block entry
 2461		fbt			kernel			   zil_free_log_record entry
 2463		fbt			kernel					 zil_incr_blks entry
 2465		fbt			kernel					zil_itxg_clean entry
 2467		fbt			kernel				zil_lwb_write_done entry
 2468		fbt			kernel				zil_lwb_write_init entry
 2470		fbt			kernel			   zil_lwb_write_start entry
 2472		fbt			kernel				 zil_read_log_data entry
 2474		fbt			kernel			 zil_replay_log_record entry
 2476		fbt			kernel				  zil_vdev_compare entry
27281		fbt			kernel					   zil_suspend entry
28707		fbt			kernel						 zil_close entry
31255		fbt			kernel				  zil_vdev_offline entry
32411		fbt			kernel			   zil_check_log_chain entry
33402		fbt			kernel						 zil_claim entry
34099		fbt			kernel						zil_resume entry
34365		fbt			kernel					zilog_is_dirty entry
34404		fbt			kernel						  zil_fini entry
34491		fbt			kernel						 zil_alloc entry
34637		fbt			kernel				 zil_async_to_sync entry
34747		fbt			kernel				   zil_bp_tree_add entry
36230		fbt			kernel					  zil_set_sync entry
36609		fbt			kernel					   zilog_dirty entry
37269		fbt			kernel						  zil_open entry
40711		fbt			kernel			 zilog_is_dirty_in_txg entry
40811		fbt			kernel					zil_itx_assign entry
42882		fbt			kernel					 zil_replaying entry
42886		fbt			kernel				  zil_destroy_sync entry
43523		fbt			kernel					zil_itx_create entry
44492		fbt			kernel						  zil_free entry
44540		fbt			kernel						zil_commit entry
47840		fbt			kernel					 zil_add_block entry
48652		fbt			kernel						 zil_clean entry
49809		fbt			kernel				   zil_itx_destroy entry
50959		fbt			kernel						zil_replay entry
53653		fbt			kernel						  zil_init entry
54756		fbt			kernel					   zil_destroy entry
55011		fbt			kernel				   zil_set_logbias entry
55539		fbt			kernel						 zil_parse entry
60977		fbt			kernel						  zil_sync entry
 
Status
Not open for further replies.
Top