nvme: Inject artificial failure on reset

Add a sysctl to artificially fail the reset to test the failure to reset
hardware code path. While there are many ways that reset can fail, this
provides an adequate way that similates enough of the failures well
enough to shake out this failure path.

Sponsored by:		Netflix
Differential Revision:	https://reviews.freebsd.org/D28584

sys/dev/nvme/nvme_ctrlr.c

@@ -1152,7 +1152,7 @@ nvme_ctrlr_start_config_hook(void *arg)
 
 	TSENTER();
 
-	if (nvme_ctrlr_hw_reset(ctrlr) != 0) {
+	if (nvme_ctrlr_hw_reset(ctrlr) != 0 || ctrlr->fail_on_reset != 0) {
 		nvme_ctrlr_fail(ctrlr, true);
 		config_intrhook_disestablish(&ctrlr->config_hook);
 		return;

sys/dev/nvme/nvme_private.h

@@ -299,6 +299,7 @@ struct nvme_controller {
 
 	uint32_t			is_resetting;
 	uint32_t			notification_sent;
+	u_int				fail_on_reset;
 
 	bool				is_failed;
 	bool				is_failed_admin;

sys/dev/nvme/nvme_sysctl.c

@@ -426,6 +426,10 @@ nvme_sysctl_initialize_ctrlr(struct nvme_controller *ctrlr)
 	    CTLFLAG_RD, &ctrlr->cap_hi, 0,
 	    "Hi 32-bits of capacities for the drive");
 
+	SYSCTL_ADD_UINT(ctrlr_ctx, ctrlr_list, OID_AUTO, "fail_on_reset",
+	    CTLFLAG_RD, &ctrlr->fail_on_reset, 0,
+	    "Pretend the next reset fails and fail the controller");
+
 	que_tree = SYSCTL_ADD_NODE(ctrlr_ctx, ctrlr_list, OID_AUTO, "adminq",
 	    CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Admin Queue");