24 	struct nvme_ctrl *ctrl;  member
52 static inline int ctrl_max_dhchaps(struct nvme_ctrl *ctrl)  in ctrl_max_dhchaps()  argument
54 	return ctrl->opts->nr_io_queues + ctrl->opts->nr_write_queues +  in ctrl_max_dhchaps()
55 			ctrl->opts->nr_poll_queues + 1;  in ctrl_max_dhchaps()
58 static int nvme_auth_submit(struct nvme_ctrl *ctrl, int qid,  in nvme_auth_submit()  argument
63 	struct request_queue *q = ctrl->fabrics_q;  in nvme_auth_submit()
68 		q = ctrl->connect_q;  in nvme_auth_submit()
86 		dev_warn(ctrl->device,  in nvme_auth_submit()
89 		dev_err(ctrl->device,  in nvme_auth_submit()
94 static int nvme_auth_receive_validate(struct nvme_ctrl *ctrl, int qid,  in nvme_auth_receive_validate()  argument
98 	dev_dbg(ctrl->device, "%s: qid %d auth_type %d auth_id %x\n",  in nvme_auth_receive_validate()
107 		dev_warn(ctrl->device,  in nvme_auth_receive_validate()
113 		dev_warn(ctrl->device,  in nvme_auth_receive_validate()
121 static int nvme_auth_set_dhchap_negotiate_data(struct nvme_ctrl *ctrl,  in nvme_auth_set_dhchap_negotiate_data()  argument
135 	if (ctrl->opts->concat && chap->qid == 0) {  in nvme_auth_set_dhchap_negotiate_data()
136 		if (ctrl->opts->tls_key)  in nvme_auth_set_dhchap_negotiate_data()
159 static int nvme_auth_process_dhchap_challenge(struct nvme_ctrl *ctrl,  in nvme_auth_process_dhchap_challenge()  argument
175 		dev_warn(ctrl->device,  in nvme_auth_process_dhchap_challenge()
185 		dev_dbg(ctrl->device,  in nvme_auth_process_dhchap_challenge()
200 		dev_warn(ctrl->device,  in nvme_auth_process_dhchap_challenge()
209 		dev_warn(ctrl->device,  in nvme_auth_process_dhchap_challenge()
220 	dev_dbg(ctrl->device, "qid %d: selected hash %s\n",  in nvme_auth_process_dhchap_challenge()
226 		dev_warn(ctrl->device,  in nvme_auth_process_dhchap_challenge()
236 		dev_dbg(ctrl->device,  in nvme_auth_process_dhchap_challenge()
250 			dev_warn(ctrl->device,  in nvme_auth_process_dhchap_challenge()
261 			dev_warn(ctrl->device,  in nvme_auth_process_dhchap_challenge()
268 		dev_dbg(ctrl->device, "qid %d: selected DH group %s\n",  in nvme_auth_process_dhchap_challenge()
271 		dev_warn(ctrl->device,  in nvme_auth_process_dhchap_challenge()
291 		dev_dbg(ctrl->device, "ctrl public key %*ph\n",  in nvme_auth_process_dhchap_challenge()
298 static int nvme_auth_set_dhchap_reply_data(struct nvme_ctrl *ctrl,  in nvme_auth_set_dhchap_reply_data()  argument
321 	if (ctrl->ctrl_key)  in nvme_auth_set_dhchap_reply_data()
323 	if (ctrl->ctrl_key || ctrl->opts->concat) {  in nvme_auth_set_dhchap_reply_data()
328 		dev_dbg(ctrl->device, "%s: qid %d ctrl challenge %*ph\n",  in nvme_auth_set_dhchap_reply_data()
333 	if (ctrl->opts->concat)  in nvme_auth_set_dhchap_reply_data()
339 		dev_dbg(ctrl->device, "%s: qid %d host public key %*ph\n",  in nvme_auth_set_dhchap_reply_data()
349 static int nvme_auth_process_dhchap_success1(struct nvme_ctrl *ctrl,  in nvme_auth_process_dhchap_success1()  argument
361 		dev_warn(ctrl->device,  in nvme_auth_process_dhchap_success1()
370 		dev_info(ctrl->device,  in nvme_auth_process_dhchap_success1()
380 		dev_dbg(ctrl->device, "%s: qid %d ctrl response %*ph\n",  in nvme_auth_process_dhchap_success1()
382 		dev_dbg(ctrl->device, "%s: qid %d host response %*ph\n",  in nvme_auth_process_dhchap_success1()
385 		dev_warn(ctrl->device,  in nvme_auth_process_dhchap_success1()
394 		dev_info(ctrl->device,  in nvme_auth_process_dhchap_success1()
399 static int nvme_auth_set_dhchap_success2_data(struct nvme_ctrl *ctrl,  in nvme_auth_set_dhchap_success2_data()  argument
413 static int nvme_auth_set_dhchap_failure2_data(struct nvme_ctrl *ctrl,  in nvme_auth_set_dhchap_failure2_data()  argument
429 static int nvme_auth_dhchap_setup_host_response(struct nvme_ctrl *ctrl,  in nvme_auth_dhchap_setup_host_response()  argument
436 	dev_dbg(ctrl->device, "%s: qid %d host response seq %u transaction %d\n",  in nvme_auth_dhchap_setup_host_response()
440 		chap->transformed_key = nvme_auth_transform_key(ctrl->host_key,  in nvme_auth_dhchap_setup_host_response()
441 						ctrl->opts->host->nqn);  in nvme_auth_dhchap_setup_host_response()
448 		dev_dbg(ctrl->device, "%s: qid %d re-using host response\n",  in nvme_auth_dhchap_setup_host_response()
455 		dev_warn(ctrl->device, "qid %d: failed to set key, error %d\n",  in nvme_auth_dhchap_setup_host_response()
497 	ret = crypto_shash_update(shash, ctrl->opts->host->nqn,  in nvme_auth_dhchap_setup_host_response()
498 				  strlen(ctrl->opts->host->nqn));  in nvme_auth_dhchap_setup_host_response()
504 	ret = crypto_shash_update(shash, ctrl->opts->subsysnqn,  in nvme_auth_dhchap_setup_host_response()
505 			    strlen(ctrl->opts->subsysnqn));  in nvme_auth_dhchap_setup_host_response()
515 static int nvme_auth_dhchap_setup_ctrl_response(struct nvme_ctrl *ctrl,  in nvme_auth_dhchap_setup_ctrl_response()  argument
523 	transformed_key = nvme_auth_transform_key(ctrl->ctrl_key,  in nvme_auth_dhchap_setup_ctrl_response()
524 				ctrl->opts->subsysnqn);  in nvme_auth_dhchap_setup_ctrl_response()
533 		dev_warn(ctrl->device, "qid %d: failed to set key, error %d\n",  in nvme_auth_dhchap_setup_ctrl_response()
552 	dev_dbg(ctrl->device, "%s: qid %d ctrl response seq %u transaction %d\n",  in nvme_auth_dhchap_setup_ctrl_response()
554 	dev_dbg(ctrl->device, "%s: qid %d challenge %*ph\n",  in nvme_auth_dhchap_setup_ctrl_response()
556 	dev_dbg(ctrl->device, "%s: qid %d subsysnqn %s\n",  in nvme_auth_dhchap_setup_ctrl_response()
557 		__func__, chap->qid, ctrl->opts->subsysnqn);  in nvme_auth_dhchap_setup_ctrl_response()
558 	dev_dbg(ctrl->device, "%s: qid %d hostnqn %s\n",  in nvme_auth_dhchap_setup_ctrl_response()
559 		__func__, chap->qid, ctrl->opts->host->nqn);  in nvme_auth_dhchap_setup_ctrl_response()
582 	ret = crypto_shash_update(shash, ctrl->opts->subsysnqn,  in nvme_auth_dhchap_setup_ctrl_response()
583 				  strlen(ctrl->opts->subsysnqn));  in nvme_auth_dhchap_setup_ctrl_response()
589 	ret = crypto_shash_update(shash, ctrl->opts->host->nqn,  in nvme_auth_dhchap_setup_ctrl_response()
590 				  strlen(ctrl->opts->host->nqn));  in nvme_auth_dhchap_setup_ctrl_response()
601 static int nvme_auth_dhchap_exponential(struct nvme_ctrl *ctrl,  in nvme_auth_dhchap_exponential()  argument
607 		dev_dbg(ctrl->device,  in nvme_auth_dhchap_exponential()
628 		dev_dbg(ctrl->device,  in nvme_auth_dhchap_exponential()
647 		dev_dbg(ctrl->device,  in nvme_auth_dhchap_exponential()
652 	dev_dbg(ctrl->device, "shared secret %*ph\n",  in nvme_auth_dhchap_exponential()
691 void nvme_auth_revoke_tls_key(struct nvme_ctrl *ctrl)  in nvme_auth_revoke_tls_key()  argument
693 	dev_dbg(ctrl->device, "Wipe generated TLS PSK %08x\n",  in nvme_auth_revoke_tls_key()
694 		key_serial(ctrl->opts->tls_key));  in nvme_auth_revoke_tls_key()
695 	key_revoke(ctrl->opts->tls_key);  in nvme_auth_revoke_tls_key()
696 	key_put(ctrl->opts->tls_key);  in nvme_auth_revoke_tls_key()
697 	ctrl->opts->tls_key = NULL;  in nvme_auth_revoke_tls_key()
701 static int nvme_auth_secure_concat(struct nvme_ctrl *ctrl,  in nvme_auth_secure_concat()  argument
710 		dev_warn(ctrl->device,  in nvme_auth_secure_concat()
717 		dev_warn(ctrl->device,  in nvme_auth_secure_concat()
727 		dev_warn(ctrl->device,  in nvme_auth_secure_concat()
732 	dev_dbg(ctrl->device,  in nvme_auth_secure_concat()
736 					ctrl->opts->subsysnqn,  in nvme_auth_secure_concat()
737 					ctrl->opts->host->nqn, &digest);  in nvme_auth_secure_concat()
739 		dev_warn(ctrl->device,  in nvme_auth_secure_concat()
744 	dev_dbg(ctrl->device, "%s: generated digest %s\n",  in nvme_auth_secure_concat()
749 		dev_warn(ctrl->device,  in nvme_auth_secure_concat()
755 	tls_key = nvme_tls_psk_refresh(ctrl->opts->keyring,  in nvme_auth_secure_concat()
756 				       ctrl->opts->host->nqn,  in nvme_auth_secure_concat()
757 				       ctrl->opts->subsysnqn, chap->hash_id,  in nvme_auth_secure_concat()
761 		dev_warn(ctrl->device,  in nvme_auth_secure_concat()
767 	if (ctrl->opts->tls_key)  in nvme_auth_secure_concat()
768 		nvme_auth_revoke_tls_key(ctrl);  in nvme_auth_secure_concat()
769 	ctrl->opts->tls_key = tls_key;  in nvme_auth_secure_concat()
781 	struct nvme_ctrl *ctrl = chap->ctrl;  in nvme_queue_auth_work()  local
795 	chap->transaction = ctrl->transaction++;  in nvme_queue_auth_work()
798 	dev_dbg(ctrl->device, "%s: qid %d send negotiate\n",  in nvme_queue_auth_work()
800 	ret = nvme_auth_set_dhchap_negotiate_data(ctrl, chap);  in nvme_queue_auth_work()
806 	ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true);  in nvme_queue_auth_work()
813 	dev_dbg(ctrl->device, "%s: qid %d receive challenge\n",  in nvme_queue_auth_work()
817 	ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, CHAP_BUF_SIZE,  in nvme_queue_auth_work()
820 		dev_warn(ctrl->device,  in nvme_queue_auth_work()
826 	ret = nvme_auth_receive_validate(ctrl, chap->qid, chap->buf, chap->transaction,  in nvme_queue_auth_work()
834 	ret = nvme_auth_process_dhchap_challenge(ctrl, chap);  in nvme_queue_auth_work()
842 		dev_dbg(ctrl->device,  in nvme_queue_auth_work()
845 		ret = nvme_auth_dhchap_exponential(ctrl, chap);  in nvme_queue_auth_work()
852 	dev_dbg(ctrl->device, "%s: qid %d host response\n",  in nvme_queue_auth_work()
854 	mutex_lock(&ctrl->dhchap_auth_mutex);  in nvme_queue_auth_work()
855 	ret = nvme_auth_dhchap_setup_host_response(ctrl, chap);  in nvme_queue_auth_work()
856 	mutex_unlock(&ctrl->dhchap_auth_mutex);  in nvme_queue_auth_work()
863 	dev_dbg(ctrl->device, "%s: qid %d send reply\n",  in nvme_queue_auth_work()
865 	ret = nvme_auth_set_dhchap_reply_data(ctrl, chap);  in nvme_queue_auth_work()
872 	ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true);  in nvme_queue_auth_work()
879 	dev_dbg(ctrl->device, "%s: qid %d receive success1\n",  in nvme_queue_auth_work()
883 	ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, CHAP_BUF_SIZE,  in nvme_queue_auth_work()
886 		dev_warn(ctrl->device,  in nvme_queue_auth_work()
892 	ret = nvme_auth_receive_validate(ctrl, chap->qid,  in nvme_queue_auth_work()
901 	mutex_lock(&ctrl->dhchap_auth_mutex);  in nvme_queue_auth_work()
902 	if (ctrl->ctrl_key) {  in nvme_queue_auth_work()
903 		dev_dbg(ctrl->device,  in nvme_queue_auth_work()
906 		ret = nvme_auth_dhchap_setup_ctrl_response(ctrl, chap);  in nvme_queue_auth_work()
908 			mutex_unlock(&ctrl->dhchap_auth_mutex);  in nvme_queue_auth_work()
913 	mutex_unlock(&ctrl->dhchap_auth_mutex);  in nvme_queue_auth_work()
915 	ret = nvme_auth_process_dhchap_success1(ctrl, chap);  in nvme_queue_auth_work()
924 		dev_dbg(ctrl->device, "%s: qid %d send success2\n",  in nvme_queue_auth_work()
926 		tl = nvme_auth_set_dhchap_success2_data(ctrl, chap);  in nvme_queue_auth_work()
927 		ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true);  in nvme_queue_auth_work()
933 		if (ctrl->opts->concat &&  in nvme_queue_auth_work()
934 		    (ret = nvme_auth_secure_concat(ctrl, chap))) {  in nvme_queue_auth_work()
935 			dev_warn(ctrl->device,  in nvme_queue_auth_work()
946 	dev_dbg(ctrl->device, "%s: qid %d send failure2, status %x\n",  in nvme_queue_auth_work()
948 	tl = nvme_auth_set_dhchap_failure2_data(ctrl, chap);  in nvme_queue_auth_work()
949 	ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true);  in nvme_queue_auth_work()
958 int nvme_auth_negotiate(struct nvme_ctrl *ctrl, int qid)  in nvme_auth_negotiate()  argument
962 	if (!ctrl->host_key) {  in nvme_auth_negotiate()
963 		dev_warn(ctrl->device, "qid %d: no key\n", qid);  in nvme_auth_negotiate()
967 	if (ctrl->opts->dhchap_ctrl_secret && !ctrl->ctrl_key) {  in nvme_auth_negotiate()
968 		dev_warn(ctrl->device, "qid %d: invalid ctrl key\n", qid);  in nvme_auth_negotiate()
972 	chap = &ctrl->dhchap_ctxs[qid];  in nvme_auth_negotiate()
979 int nvme_auth_wait(struct nvme_ctrl *ctrl, int qid)  in nvme_auth_wait()  argument
984 	chap = &ctrl->dhchap_ctxs[qid];  in nvme_auth_wait()
995 	struct nvme_ctrl *ctrl =  in nvme_ctrl_auth_work()  local
1003 	if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE)  in nvme_ctrl_auth_work()
1007 	ret = nvme_auth_negotiate(ctrl, 0);  in nvme_ctrl_auth_work()
1009 		dev_warn(ctrl->device,  in nvme_ctrl_auth_work()
1013 	ret = nvme_auth_wait(ctrl, 0);  in nvme_ctrl_auth_work()
1015 		dev_warn(ctrl->device,  in nvme_ctrl_auth_work()
1022 	if (ctrl->opts->concat)  in nvme_ctrl_auth_work()
1025 	for (q = 1; q < ctrl->queue_count; q++) {  in nvme_ctrl_auth_work()
1026 		ret = nvme_auth_negotiate(ctrl, q);  in nvme_ctrl_auth_work()
1028 			dev_warn(ctrl->device,  in nvme_ctrl_auth_work()
1039 	for (q = 1; q < ctrl->queue_count; q++) {  in nvme_ctrl_auth_work()
1040 		ret = nvme_auth_wait(ctrl, q);  in nvme_ctrl_auth_work()
1042 			dev_warn(ctrl->device,  in nvme_ctrl_auth_work()
1047 int nvme_auth_init_ctrl(struct nvme_ctrl *ctrl)  in nvme_auth_init_ctrl()  argument
1052 	mutex_init(&ctrl->dhchap_auth_mutex);  in nvme_auth_init_ctrl()
1053 	INIT_WORK(&ctrl->dhchap_auth_work, nvme_ctrl_auth_work);  in nvme_auth_init_ctrl()
1054 	if (!ctrl->opts)  in nvme_auth_init_ctrl()
1056 	ret = nvme_auth_generate_key(ctrl->opts->dhchap_secret,  in nvme_auth_init_ctrl()
1057 			&ctrl->host_key);  in nvme_auth_init_ctrl()
1060 	ret = nvme_auth_generate_key(ctrl->opts->dhchap_ctrl_secret,  in nvme_auth_init_ctrl()
1061 			&ctrl->ctrl_key);  in nvme_auth_init_ctrl()
1065 	if (!ctrl->opts->dhchap_secret && !ctrl->opts->dhchap_ctrl_secret)  in nvme_auth_init_ctrl()
1068 	ctrl->dhchap_ctxs = kvcalloc(ctrl_max_dhchaps(ctrl),  in nvme_auth_init_ctrl()
1070 	if (!ctrl->dhchap_ctxs) {  in nvme_auth_init_ctrl()
1075 	for (i = 0; i < ctrl_max_dhchaps(ctrl); i++) {  in nvme_auth_init_ctrl()
1076 		chap = &ctrl->dhchap_ctxs[i];  in nvme_auth_init_ctrl()
1078 		chap->ctrl = ctrl;  in nvme_auth_init_ctrl()
1084 	nvme_auth_free_key(ctrl->ctrl_key);  in nvme_auth_init_ctrl()
1085 	ctrl->ctrl_key = NULL;  in nvme_auth_init_ctrl()
1087 	nvme_auth_free_key(ctrl->host_key);  in nvme_auth_init_ctrl()
1088 	ctrl->host_key = NULL;  in nvme_auth_init_ctrl()
1093 void nvme_auth_stop(struct nvme_ctrl *ctrl)  in nvme_auth_stop()  argument
1095 	cancel_work_sync(&ctrl->dhchap_auth_work);  in nvme_auth_stop()
1099 void nvme_auth_free(struct nvme_ctrl *ctrl)  in nvme_auth_free()  argument
1103 	if (ctrl->dhchap_ctxs) {  in nvme_auth_free()
1104 		for (i = 0; i < ctrl_max_dhchaps(ctrl); i++)  in nvme_auth_free()
1105 			nvme_auth_free_dhchap(&ctrl->dhchap_ctxs[i]);  in nvme_auth_free()
1106 		kfree(ctrl->dhchap_ctxs);  in nvme_auth_free()
1108 	if (ctrl->host_key) {  in nvme_auth_free()
1109 		nvme_auth_free_key(ctrl->host_key);  in nvme_auth_free()
1110 		ctrl->host_key = NULL;  in nvme_auth_free()
1112 	if (ctrl->ctrl_key) {  in nvme_auth_free()
1113 		nvme_auth_free_key(ctrl->ctrl_key);  in nvme_auth_free()
1114 		ctrl->ctrl_key = NULL;  in nvme_auth_free()