83 static inline struct fw_priv *to_fw_priv(struct kref *ref)  in to_fw_priv()
85 	return container_of(ref, struct fw_priv, ref);  in to_fw_priv()
98 void fw_state_init(struct fw_priv *fw_priv)  in fw_state_init()  argument
100 	struct fw_state *fw_st = &fw_priv->fw_st;  in fw_state_init()
106 static inline int fw_state_wait(struct fw_priv *fw_priv)  in fw_state_wait()  argument
108 	return __fw_state_wait_common(fw_priv, MAX_SCHEDULE_TIMEOUT);  in fw_state_wait()
111 static void fw_cache_piggyback_on_request(struct fw_priv *fw_priv);
113 static struct fw_priv *__allocate_fw_priv(const char *fw_name,  in __allocate_fw_priv()
120 	struct fw_priv *fw_priv;  in __allocate_fw_priv()  local
130 	fw_priv = kzalloc(sizeof(*fw_priv), GFP_ATOMIC);  in __allocate_fw_priv()
131 	if (!fw_priv)  in __allocate_fw_priv()
134 	fw_priv->fw_name = kstrdup_const(fw_name, GFP_ATOMIC);  in __allocate_fw_priv()
135 	if (!fw_priv->fw_name) {  in __allocate_fw_priv()
136 		kfree(fw_priv);  in __allocate_fw_priv()
140 	kref_init(&fw_priv->ref);  in __allocate_fw_priv()
141 	fw_priv->fwc = fwc;  in __allocate_fw_priv()
142 	fw_priv->data = dbuf;  in __allocate_fw_priv()
143 	fw_priv->allocated_size = size;  in __allocate_fw_priv()
144 	fw_priv->offset = offset;  in __allocate_fw_priv()
145 	fw_priv->opt_flags = opt_flags;  in __allocate_fw_priv()
146 	fw_state_init(fw_priv);  in __allocate_fw_priv()
148 	INIT_LIST_HEAD(&fw_priv->pending_list);  in __allocate_fw_priv()
151 	pr_debug("%s: fw-%s fw_priv=%p\n", __func__, fw_name, fw_priv);  in __allocate_fw_priv()
153 	return fw_priv;  in __allocate_fw_priv()
156 static struct fw_priv *__lookup_fw_priv(const char *fw_name)  in __lookup_fw_priv()
158 	struct fw_priv *tmp;  in __lookup_fw_priv()
169 			 struct fw_priv **fw_priv, void *dbuf, size_t size,  in alloc_lookup_fw_priv()  argument
172 	struct fw_priv *tmp;  in alloc_lookup_fw_priv()
184 			*fw_priv = tmp;  in alloc_lookup_fw_priv()
198 	*fw_priv = tmp;  in alloc_lookup_fw_priv()
206 	struct fw_priv *fw_priv = to_fw_priv(ref);  in __free_fw_priv()  local
207 	struct firmware_cache *fwc = fw_priv->fwc;  in __free_fw_priv()
210 		 __func__, fw_priv->fw_name, fw_priv, fw_priv->data,  in __free_fw_priv()
211 		 (unsigned int)fw_priv->size);  in __free_fw_priv()
213 	list_del(&fw_priv->list);  in __free_fw_priv()
216 	if (fw_is_paged_buf(fw_priv))  in __free_fw_priv()
217 		fw_free_paged_buf(fw_priv);  in __free_fw_priv()
218 	else if (!fw_priv->allocated_size)  in __free_fw_priv()
219 		vfree(fw_priv->data);  in __free_fw_priv()
221 	kfree_const(fw_priv->fw_name);  in __free_fw_priv()
222 	kfree(fw_priv);  in __free_fw_priv()
225 void free_fw_priv(struct fw_priv *fw_priv)  in free_fw_priv()  argument
227 	struct firmware_cache *fwc = fw_priv->fwc;  in free_fw_priv()
229 	if (!kref_put(&fw_priv->ref, __free_fw_priv))  in free_fw_priv()
234 bool fw_is_paged_buf(struct fw_priv *fw_priv)  in fw_is_paged_buf()  argument
236 	return fw_priv->is_paged_buf;  in fw_is_paged_buf()
239 void fw_free_paged_buf(struct fw_priv *fw_priv)  in fw_free_paged_buf()  argument
243 	if (!fw_priv->pages)  in fw_free_paged_buf()
246 	vunmap(fw_priv->data);  in fw_free_paged_buf()
248 	for (i = 0; i < fw_priv->nr_pages; i++)  in fw_free_paged_buf()
249 		__free_page(fw_priv->pages[i]);  in fw_free_paged_buf()
250 	kvfree(fw_priv->pages);  in fw_free_paged_buf()
251 	fw_priv->pages = NULL;  in fw_free_paged_buf()
252 	fw_priv->page_array_size = 0;  in fw_free_paged_buf()
253 	fw_priv->nr_pages = 0;  in fw_free_paged_buf()
254 	fw_priv->data = NULL;  in fw_free_paged_buf()
255 	fw_priv->size = 0;  in fw_free_paged_buf()
258 int fw_grow_paged_buf(struct fw_priv *fw_priv, int pages_needed)  in fw_grow_paged_buf()  argument
261 	if (fw_priv->page_array_size < pages_needed) {  in fw_grow_paged_buf()
263 					 fw_priv->page_array_size * 2);  in fw_grow_paged_buf()
270 		memcpy(new_pages, fw_priv->pages,  in fw_grow_paged_buf()
271 		       fw_priv->page_array_size * sizeof(void *));  in fw_grow_paged_buf()
272 		memset(&new_pages[fw_priv->page_array_size], 0, sizeof(void *) *  in fw_grow_paged_buf()
273 		       (new_array_size - fw_priv->page_array_size));  in fw_grow_paged_buf()
274 		kvfree(fw_priv->pages);  in fw_grow_paged_buf()
275 		fw_priv->pages = new_pages;  in fw_grow_paged_buf()
276 		fw_priv->page_array_size = new_array_size;  in fw_grow_paged_buf()
279 	while (fw_priv->nr_pages < pages_needed) {  in fw_grow_paged_buf()
280 		fw_priv->pages[fw_priv->nr_pages] =  in fw_grow_paged_buf()
283 		if (!fw_priv->pages[fw_priv->nr_pages])  in fw_grow_paged_buf()
285 		fw_priv->nr_pages++;  in fw_grow_paged_buf()
291 int fw_map_paged_buf(struct fw_priv *fw_priv)  in fw_map_paged_buf()  argument
294 	if (!fw_priv->pages)  in fw_map_paged_buf()
297 	vunmap(fw_priv->data);  in fw_map_paged_buf()
298 	fw_priv->data = vmap(fw_priv->pages, fw_priv->nr_pages, 0,  in fw_map_paged_buf()
300 	if (!fw_priv->data)  in fw_map_paged_buf()
311 static int fw_decompress_zstd(struct device *dev, struct fw_priv *fw_priv,  in fw_decompress_zstd()  argument
319 	if (fw_priv->allocated_size) {  in fw_decompress_zstd()
320 		out_size = fw_priv->allocated_size;  in fw_decompress_zstd()
321 		out_buf = fw_priv->data;  in fw_decompress_zstd()
358 	if (!fw_priv->allocated_size)  in fw_decompress_zstd()
359 		fw_priv->data = out_buf;  in fw_decompress_zstd()
360 	fw_priv->size = len;  in fw_decompress_zstd()
365 	if (err && !fw_priv->allocated_size)  in fw_decompress_zstd()
386 static int fw_decompress_xz_single(struct device *dev, struct fw_priv *fw_priv,  in fw_decompress_xz_single()  argument
400 	xz_buf.out_size = fw_priv->allocated_size;  in fw_decompress_xz_single()
401 	xz_buf.out = fw_priv->data;  in fw_decompress_xz_single()
407 	fw_priv->size = xz_buf.out_pos;  in fw_decompress_xz_single()
412 static int fw_decompress_xz_pages(struct device *dev, struct fw_priv *fw_priv,  in fw_decompress_xz_pages()  argument
429 	fw_priv->is_paged_buf = true;  in fw_decompress_xz_pages()
430 	fw_priv->size = 0;  in fw_decompress_xz_pages()
432 		if (fw_grow_paged_buf(fw_priv, fw_priv->nr_pages + 1)) {  in fw_decompress_xz_pages()
438 		page = fw_priv->pages[fw_priv->nr_pages - 1];  in fw_decompress_xz_pages()
444 		fw_priv->size += xz_buf.out_pos;  in fw_decompress_xz_pages()
452 		err = fw_map_paged_buf(fw_priv);  in fw_decompress_xz_pages()
459 static int fw_decompress_xz(struct device *dev, struct fw_priv *fw_priv,  in fw_decompress_xz()  argument
463 	if (fw_priv->data)  in fw_decompress_xz()
464 		return fw_decompress_xz_single(dev, fw_priv, in_size, in_buffer);  in fw_decompress_xz()
466 		return fw_decompress_xz_pages(dev, fw_priv, in_size, in_buffer);  in fw_decompress_xz()
489 fw_get_filesystem_firmware(struct device *device, struct fw_priv *fw_priv,  in fw_get_filesystem_firmware()  argument
492 					     struct fw_priv *fw_priv,  in fw_get_filesystem_firmware()
504 	if (!decompress && fw_priv->data) {  in fw_get_filesystem_firmware()
505 		buffer = fw_priv->data;  in fw_get_filesystem_firmware()
506 		msize = fw_priv->allocated_size;  in fw_get_filesystem_firmware()
532 			       fw_priv->fw_name, suffix);  in fw_get_filesystem_firmware()
538 		fw_priv->size = 0;  in fw_get_filesystem_firmware()
545 		if ((fw_priv->opt_flags & FW_OPT_PARTIAL) && buffer)  in fw_get_filesystem_firmware()
549 		rc = kernel_read_file_from_path_initns(path, fw_priv->offset,  in fw_get_filesystem_firmware()
554 			if (!(fw_priv->opt_flags & FW_OPT_NO_WARN)) {  in fw_get_filesystem_firmware()
572 				fw_priv->fw_name);  in fw_get_filesystem_firmware()
573 			rc = decompress(device, fw_priv, size, buffer);  in fw_get_filesystem_firmware()
578 				fw_free_paged_buf(fw_priv);  in fw_get_filesystem_firmware()
583 				fw_priv->fw_name);  in fw_get_filesystem_firmware()
584 			if (!fw_priv->data)  in fw_get_filesystem_firmware()
585 				fw_priv->data = buffer;  in fw_get_filesystem_firmware()
586 			fw_priv->size = size;  in fw_get_filesystem_firmware()
588 		fw_state_done(fw_priv);  in fw_get_filesystem_firmware()
608 static void fw_set_page_data(struct fw_priv *fw_priv, struct firmware *fw)  in fw_set_page_data()  argument
610 	fw->priv = fw_priv;  in fw_set_page_data()
611 	fw->size = fw_priv->size;  in fw_set_page_data()
612 	fw->data = fw_priv->data;  in fw_set_page_data()
615 		 __func__, fw_priv->fw_name, fw_priv, fw_priv->data,  in fw_set_page_data()
616 		 (unsigned int)fw_priv->size);  in fw_set_page_data()
697 	struct fw_priv *fw_priv = fw->priv;  in assign_fw()  local
701 	if (!fw_priv->size || fw_state_is_aborted(fw_priv)) {  in assign_fw()
714 	if (device && (fw_priv->opt_flags & FW_OPT_UEVENT) &&  in assign_fw()
715 	    !(fw_priv->opt_flags & FW_OPT_NOCACHE)) {  in assign_fw()
716 		ret = fw_add_devm_name(device, fw_priv->fw_name);  in assign_fw()
727 	if (!(fw_priv->opt_flags & FW_OPT_NOCACHE) &&  in assign_fw()
728 	    fw_priv->fwc->state == FW_LOADER_START_CACHE)  in assign_fw()
729 		fw_cache_piggyback_on_request(fw_priv);  in assign_fw()
732 	fw_set_page_data(fw_priv, fw);  in assign_fw()
747 	struct fw_priv *fw_priv;  in _request_firmware_prepare()  local
762 	ret = alloc_lookup_fw_priv(name, &fw_cache, &fw_priv, dbuf, size,  in _request_firmware_prepare()
769 	firmware->priv = fw_priv;  in _request_firmware_prepare()
772 		ret = fw_state_wait(fw_priv);  in _request_firmware_prepare()
774 			fw_set_page_data(fw_priv, firmware);  in _request_firmware_prepare()
795 	struct fw_priv *fw_priv;  in fw_abort_batch_reqs()  local
801 	fw_priv = fw->priv;  in fw_abort_batch_reqs()
803 	if (!fw_state_is_aborted(fw_priv))  in fw_abort_batch_reqs()
804 		fw_state_aborted(fw_priv);  in fw_abort_batch_reqs()
1339 static struct fw_priv *lookup_fw_priv(const char *fw_name)  in lookup_fw_priv()
1341 	struct fw_priv *tmp;  in lookup_fw_priv()
1364 	struct fw_priv *fw_priv;  in uncache_firmware()  local
1372 	fw_priv = lookup_fw_priv(fw_name);  in uncache_firmware()
1373 	if (fw_priv) {  in uncache_firmware()
1374 		free_fw_priv(fw_priv);  in uncache_firmware()
1411 static void fw_cache_piggyback_on_request(struct fw_priv *fw_priv)  in fw_cache_piggyback_on_request()  argument
1413 	const char *name = fw_priv->fw_name;  in fw_cache_piggyback_on_request()
1414 	struct firmware_cache *fwc = fw_priv->fwc;  in fw_cache_piggyback_on_request()
1424 		kref_get(&fw_priv->ref);  in fw_cache_piggyback_on_request()
1659 static void fw_cache_piggyback_on_request(struct fw_priv *fw_priv)  in fw_cache_piggyback_on_request()  argument