162 	int (*invalidate_tlb)(struct apple_dart_stream_map *stream_map);
294 #define for_each_stream_map(i, base, stream_map)                               \  argument
295 	for (i = 0, stream_map = &(base)->stream_maps[0];                      \
296 	     i < MAX_DARTS_PER_DEVICE && stream_map->dart;                     \
297 	     stream_map = &(base)->stream_maps[++i])
308 apple_dart_hw_enable_translation(struct apple_dart_stream_map *stream_map)  in apple_dart_hw_enable_translation()  argument
310 	struct apple_dart *dart = stream_map->dart;  in apple_dart_hw_enable_translation()
313 	for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)  in apple_dart_hw_enable_translation()
317 static void apple_dart_hw_disable_dma(struct apple_dart_stream_map *stream_map)  in apple_dart_hw_disable_dma()  argument
319 	struct apple_dart *dart = stream_map->dart;  in apple_dart_hw_disable_dma()
322 	for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)  in apple_dart_hw_disable_dma()
327 apple_dart_hw_enable_bypass(struct apple_dart_stream_map *stream_map)  in apple_dart_hw_enable_bypass()  argument
329 	struct apple_dart *dart = stream_map->dart;  in apple_dart_hw_enable_bypass()
332 	WARN_ON(!stream_map->dart->supports_bypass);  in apple_dart_hw_enable_bypass()
333 	for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)  in apple_dart_hw_enable_bypass()
338 static void apple_dart_hw_set_ttbr(struct apple_dart_stream_map *stream_map,  in apple_dart_hw_set_ttbr()  argument
341 	struct apple_dart *dart = stream_map->dart;  in apple_dart_hw_set_ttbr()
345 	for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)  in apple_dart_hw_set_ttbr()
351 static void apple_dart_hw_clear_ttbr(struct apple_dart_stream_map *stream_map,  in apple_dart_hw_clear_ttbr()  argument
354 	struct apple_dart *dart = stream_map->dart;  in apple_dart_hw_clear_ttbr()
357 	for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)  in apple_dart_hw_clear_ttbr()
362 apple_dart_hw_clear_all_ttbrs(struct apple_dart_stream_map *stream_map)  in apple_dart_hw_clear_all_ttbrs()  argument
366 	for (i = 0; i < stream_map->dart->hw->ttbr_count; ++i)  in apple_dart_hw_clear_all_ttbrs()
367 		apple_dart_hw_clear_ttbr(stream_map, i);  in apple_dart_hw_clear_all_ttbrs()
371 apple_dart_t8020_hw_stream_command(struct apple_dart_stream_map *stream_map,  in apple_dart_t8020_hw_stream_command()  argument
378 	spin_lock_irqsave(&stream_map->dart->lock, flags);  in apple_dart_t8020_hw_stream_command()
380 	for (i = 0; i < BITS_TO_U32(stream_map->dart->num_streams); i++)  in apple_dart_t8020_hw_stream_command()
381 		writel(stream_map->sidmap[i],  in apple_dart_t8020_hw_stream_command()
382 		       stream_map->dart->regs + DART_T8020_STREAM_SELECT + 4 * i);  in apple_dart_t8020_hw_stream_command()
383 	writel(command, stream_map->dart->regs + DART_T8020_STREAM_COMMAND);  in apple_dart_t8020_hw_stream_command()
386 		stream_map->dart->regs + DART_T8020_STREAM_COMMAND, command_reg,  in apple_dart_t8020_hw_stream_command()
390 	spin_unlock_irqrestore(&stream_map->dart->lock, flags);  in apple_dart_t8020_hw_stream_command()
393 		dev_err(stream_map->dart->dev,  in apple_dart_t8020_hw_stream_command()
395 			command, stream_map->sidmap[0]);  in apple_dart_t8020_hw_stream_command()
403 apple_dart_t8110_hw_tlb_command(struct apple_dart_stream_map *stream_map,  in apple_dart_t8110_hw_tlb_command()  argument
406 	struct apple_dart *dart = stream_map->dart;  in apple_dart_t8110_hw_tlb_command()
413 	for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) {  in apple_dart_t8110_hw_tlb_command()
431 		dev_err(stream_map->dart->dev,  in apple_dart_t8110_hw_tlb_command()
441 apple_dart_t8020_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map)  in apple_dart_t8020_hw_invalidate_tlb()  argument
444 		stream_map, DART_T8020_STREAM_COMMAND_INVALIDATE);  in apple_dart_t8020_hw_invalidate_tlb()
448 apple_dart_t8110_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map)  in apple_dart_t8110_hw_invalidate_tlb()  argument
451 		stream_map, DART_T8110_TLB_CMD_OP_FLUSH_SID);  in apple_dart_t8110_hw_invalidate_tlb()
457 	struct apple_dart_stream_map stream_map;  in apple_dart_hw_reset()  local
467 	stream_map.dart = dart;  in apple_dart_hw_reset()
468 	bitmap_zero(stream_map.sidmap, DART_MAX_STREAMS);  in apple_dart_hw_reset()
469 	bitmap_set(stream_map.sidmap, 0, dart->num_streams);  in apple_dart_hw_reset()
470 	apple_dart_hw_disable_dma(&stream_map);  in apple_dart_hw_reset()
471 	apple_dart_hw_clear_all_ttbrs(&stream_map);  in apple_dart_hw_reset()
483 	return dart->hw->invalidate_tlb(&stream_map);  in apple_dart_hw_reset()
490 	struct apple_dart_stream_map stream_map;  in apple_dart_domain_flush_tlb()  local
493 		stream_map.dart = domain_stream_map->dart;  in apple_dart_domain_flush_tlb()
495 		for (j = 0; j < BITS_TO_LONGS(stream_map.dart->num_streams); j++)  in apple_dart_domain_flush_tlb()
496 			stream_map.sidmap[j] = atomic_long_read(&domain_stream_map->sidmap[j]);  in apple_dart_domain_flush_tlb()
498 		stream_map.dart->hw->invalidate_tlb(&stream_map);  in apple_dart_domain_flush_tlb()
560 			     struct apple_dart_stream_map *stream_map)  in apple_dart_setup_translation()  argument
567 		apple_dart_hw_set_ttbr(stream_map, i,  in apple_dart_setup_translation()
569 	for (; i < stream_map->dart->hw->ttbr_count; ++i)  in apple_dart_setup_translation()
570 		apple_dart_hw_clear_ttbr(stream_map, i);  in apple_dart_setup_translation()
572 	apple_dart_hw_enable_translation(stream_map);  in apple_dart_setup_translation()
573 	stream_map->dart->hw->invalidate_tlb(stream_map);  in apple_dart_setup_translation()
666 	struct apple_dart_stream_map *stream_map;  in apple_dart_attach_dev_paging()  local
678 	for_each_stream_map(i, cfg, stream_map)  in apple_dart_attach_dev_paging()
679 		apple_dart_setup_translation(dart_domain, stream_map);  in apple_dart_attach_dev_paging()
687 	struct apple_dart_stream_map *stream_map;  in apple_dart_attach_dev_identity()  local
693 	for_each_stream_map(i, cfg, stream_map)  in apple_dart_attach_dev_identity()
694 		apple_dart_hw_enable_bypass(stream_map);  in apple_dart_attach_dev_identity()
711 	struct apple_dart_stream_map *stream_map;  in apple_dart_attach_dev_blocked()  local
714 	for_each_stream_map(i, cfg, stream_map)  in apple_dart_attach_dev_blocked()
715 		apple_dart_hw_disable_dma(stream_map);  in apple_dart_attach_dev_blocked()
731 	struct apple_dart_stream_map *stream_map;  in apple_dart_probe_device()  local
737 	for_each_stream_map(i, cfg, stream_map)  in apple_dart_probe_device()
739 			dev, stream_map->dart->dev,  in apple_dart_probe_device()
837 	struct apple_dart_stream_map *stream_map;  in apple_dart_release_group()  local
842 	for_each_stream_map(i, group_master_cfg, stream_map)  in apple_dart_release_group()
843 		for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams)  in apple_dart_release_group()
844 			stream_map->dart->sid2group[sid] = NULL;  in apple_dart_release_group()
877 	struct apple_dart_stream_map *stream_map;  in apple_dart_device_group()  local
884 	for_each_stream_map(i, cfg, stream_map) {  in apple_dart_device_group()
885 		for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams) {  in apple_dart_device_group()
887 				stream_map->dart->sid2group[sid];  in apple_dart_device_group()
937 	for_each_stream_map(i, cfg, stream_map)  in apple_dart_device_group()
938 		for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams)  in apple_dart_device_group()
939 			stream_map->dart->sid2group[sid] = group;  in apple_dart_device_group()