1f25ce65dSSergey Makeev // The MIT License (MIT)
2f25ce65dSSergey Makeev //
3f25ce65dSSergey Makeev // 	Copyright (c) 2015 Sergey Makeev, Vadim Slyusarev
4f25ce65dSSergey Makeev //
5f25ce65dSSergey Makeev // 	Permission is hereby granted, free of charge, to any person obtaining a copy
6f25ce65dSSergey Makeev // 	of this software and associated documentation files (the "Software"), to deal
7f25ce65dSSergey Makeev // 	in the Software without restriction, including without limitation the rights
8f25ce65dSSergey Makeev // 	to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9f25ce65dSSergey Makeev // 	copies of the Software, and to permit persons to whom the Software is
10f25ce65dSSergey Makeev // 	furnished to do so, subject to the following conditions:
11f25ce65dSSergey Makeev //
12f25ce65dSSergey Makeev //  The above copyright notice and this permission notice shall be included in
13f25ce65dSSergey Makeev // 	all copies or substantial portions of the Software.
14f25ce65dSSergey Makeev //
15f25ce65dSSergey Makeev // 	THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16f25ce65dSSergey Makeev // 	IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17f25ce65dSSergey Makeev // 	FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18f25ce65dSSergey Makeev // 	AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19f25ce65dSSergey Makeev // 	LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20f25ce65dSSergey Makeev // 	OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21f25ce65dSSergey Makeev // 	THE SOFTWARE.
22f25ce65dSSergey Makeev 
2358964d0bSSergey Makeev #include <MTScheduler.h>
24c73d1551SSergey Makeev #include <string.h> // for memset
2558964d0bSSergey Makeev 
2658964d0bSSergey Makeev namespace MT
2758964d0bSSergey Makeev {
2858964d0bSSergey Makeev 
290727c1f9SSergey Makeev #ifdef MT_INSTRUMENTED_BUILD
30d7cf17b1Ss.makeev_local 	TaskScheduler::TaskScheduler(uint32 workerThreadsCount, WorkerThreadParams* workerParameters, IProfilerEventListener* listener)
310727c1f9SSergey Makeev #else
32d7cf17b1Ss.makeev_local 	TaskScheduler::TaskScheduler(uint32 workerThreadsCount, WorkerThreadParams* workerParameters)
330727c1f9SSergey Makeev #endif
3458964d0bSSergey Makeev 		: roundRobinThreadIndex(0)
3543c41d5cSSergey Makeev 		, startedThreadsCount(0)
3658964d0bSSergey Makeev 	{
3751901c6bSSergey Makeev 
38ff5c86b0SSergey Makeev #ifdef MT_INSTRUMENTED_BUILD
390727c1f9SSergey Makeev 		profilerEventListener = listener;
40ff5c86b0SSergey Makeev #endif
41ff5c86b0SSergey Makeev 
4234a394c3SSergey Makeev 		if (workerThreadsCount != 0)
4334a394c3SSergey Makeev 		{
44a23fd933SSergey Makeev 			threadsCount.StoreRelaxed( MT::Clamp(workerThreadsCount, (uint32)1, (uint32)MT_MAX_THREAD_COUNT) );
4534a394c3SSergey Makeev 		} else
4634a394c3SSergey Makeev 		{
4734a394c3SSergey Makeev 			//query number of processor
48d7cf17b1Ss.makeev_local 			threadsCount.StoreRelaxed( (uint32)MT::Clamp(Thread::GetNumberOfHardwareThreads() - 1, 1, (int)MT_MAX_THREAD_COUNT) );
4934a394c3SSergey Makeev 		}
5034a394c3SSergey Makeev 
51f4db079dSs.makeev 		// create fiber pool (fibers with standard stack size)
52f4db079dSs.makeev 		for (uint32 i = 0; i < MT_MAX_STANDART_FIBERS_COUNT; i++)
5358964d0bSSergey Makeev 		{
54f4db079dSs.makeev 			FiberContext& context = standartFiberContexts[i];
55f4db079dSs.makeev 			context.fiber.Create(MT_STANDART_FIBER_STACK_SIZE, FiberMain, &context);
5660ac17fbSs.makeev_local 			bool res = standartFibersAvailable.TryPush( &context );
5760ac17fbSs.makeev_local 			MT_USED_IN_ASSERT(res);
5860ac17fbSs.makeev_local 			MT_ASSERT(res == true, "Can't add fiber to storage");
5958964d0bSSergey Makeev 		}
6058964d0bSSergey Makeev 
61f4db079dSs.makeev 		// create fiber pool (fibers with extended stack size)
62f4db079dSs.makeev 		for (uint32 i = 0; i < MT_MAX_EXTENDED_FIBERS_COUNT; i++)
63f4db079dSs.makeev 		{
64f4db079dSs.makeev 			FiberContext& context = extendedFiberContexts[i];
65f4db079dSs.makeev 			context.fiber.Create(MT_EXTENDED_FIBER_STACK_SIZE, FiberMain, &context);
6660ac17fbSs.makeev_local 			bool res = extendedFibersAvailable.TryPush( &context );
6760ac17fbSs.makeev_local 			MT_USED_IN_ASSERT(res);
6860ac17fbSs.makeev_local 			MT_ASSERT(res == true, "Can't add fiber to storage");
69f4db079dSs.makeev 		}
70f4db079dSs.makeev 
71f4db079dSs.makeev 
722e846c40SSergey Makeev 		for (int16 i = 0; i < TaskGroup::MT_MAX_GROUPS_COUNT; i++)
73806ef292SSergey Makeev 		{
74754cb997SSergey Makeev 			if (i != TaskGroup::DEFAULT)
75806ef292SSergey Makeev 			{
7660ac17fbSs.makeev_local 				bool res = availableGroups.TryPush( TaskGroup(i) );
7760ac17fbSs.makeev_local 				MT_USED_IN_ASSERT(res);
7860ac17fbSs.makeev_local 				MT_ASSERT(res == true, "Can't add group to storage");
79806ef292SSergey Makeev 			}
80806ef292SSergey Makeev 		}
81806ef292SSergey Makeev 
829d94a278Ss.makeev_local #if MT_GROUP_DEBUG
832e846c40SSergey Makeev 		groupStats[TaskGroup::DEFAULT].SetDebugIsFree(false);
849d94a278Ss.makeev_local #endif
85806ef292SSergey Makeev 
8658964d0bSSergey Makeev 		// create worker thread pool
87a23fd933SSergey Makeev 		int32 totalThreadsCount = GetWorkersCount();
88a23fd933SSergey Makeev 		for (int32 i = 0; i < totalThreadsCount; i++)
8958964d0bSSergey Makeev 		{
902b4df162SSergey Makeev 			threadContext[i].SetThreadIndex(i);
9158964d0bSSergey Makeev 			threadContext[i].taskScheduler = this;
92d7cf17b1Ss.makeev_local 
93d7cf17b1Ss.makeev_local 			uint32 threadCore = i;
94d7cf17b1Ss.makeev_local 			ThreadPriority::Type priority = ThreadPriority::DEFAULT;
95d7cf17b1Ss.makeev_local 			if (workerParameters != nullptr)
96d7cf17b1Ss.makeev_local 			{
97d7cf17b1Ss.makeev_local 				const WorkerThreadParams& params = workerParameters[i];
98d7cf17b1Ss.makeev_local 
99d7cf17b1Ss.makeev_local 				threadCore = params.core;
100d7cf17b1Ss.makeev_local 				priority = params.priority;
101d7cf17b1Ss.makeev_local 			}
102d7cf17b1Ss.makeev_local 
103d7cf17b1Ss.makeev_local 			threadContext[i].thread.Start( MT_SCHEDULER_STACK_SIZE, WorkerThreadMain, &threadContext[i], threadCore, priority);
10458964d0bSSergey Makeev 		}
10558964d0bSSergey Makeev 	}
10658964d0bSSergey Makeev 
107d7cf17b1Ss.makeev_local 
10858964d0bSSergey Makeev 	TaskScheduler::~TaskScheduler()
10958964d0bSSergey Makeev 	{
110a23fd933SSergey Makeev 		int32 totalThreadsCount = GetWorkersCount();
111a23fd933SSergey Makeev 		for (int32 i = 0; i < totalThreadsCount; i++)
11258964d0bSSergey Makeev 		{
11381ec7369SSergey Makeev 			threadContext[i].state.Store(internal::ThreadState::EXIT);
11458964d0bSSergey Makeev 			threadContext[i].hasNewTasksEvent.Signal();
11558964d0bSSergey Makeev 		}
11658964d0bSSergey Makeev 
117a23fd933SSergey Makeev 		for (int32 i = 0; i < totalThreadsCount; i++)
11858964d0bSSergey Makeev 		{
119c7362320Ss.makeev_local 			threadContext[i].thread.Join();
12058964d0bSSergey Makeev 		}
12158964d0bSSergey Makeev 	}
12258964d0bSSergey Makeev 
1234965e494Ss.makeev_local 	FiberContext* TaskScheduler::RequestFiberContext(internal::GroupedTask& task)
124f4db079dSs.makeev 	{
1254965e494Ss.makeev_local 		FiberContext *fiberContext = task.awaitingFiber;
1264965e494Ss.makeev_local 		if (fiberContext)
1274965e494Ss.makeev_local 		{
1284965e494Ss.makeev_local 			task.awaitingFiber = nullptr;
1294965e494Ss.makeev_local 			return fiberContext;
1304965e494Ss.makeev_local 		}
13160ac17fbSs.makeev_local 
1324965e494Ss.makeev_local 		MT::StackRequirements::Type stackRequirements = task.desc.stackRequirements;
1334965e494Ss.makeev_local 
1344965e494Ss.makeev_local 		fiberContext = nullptr;
13560ac17fbSs.makeev_local 		bool res = false;
13660ac17fbSs.makeev_local 		MT_USED_IN_ASSERT(res);
137f4db079dSs.makeev 		switch(stackRequirements)
138f4db079dSs.makeev 		{
139f4db079dSs.makeev 		case MT::StackRequirements::STANDARD:
14060ac17fbSs.makeev_local 			res = standartFibersAvailable.TryPop(fiberContext);
14160ac17fbSs.makeev_local 			MT_ASSERT(res, "Can't get more standard fibers!");
142f4db079dSs.makeev 			break;
143f4db079dSs.makeev 		case MT::StackRequirements::EXTENDED:
14460ac17fbSs.makeev_local 			res = extendedFibersAvailable.TryPop(fiberContext);
14560ac17fbSs.makeev_local 			MT_ASSERT(res, "Can't get more extended fibers!");
146f4db079dSs.makeev 			break;
147f4db079dSs.makeev 		default:
148f4db079dSs.makeev 			MT_REPORT_ASSERT("Unknown stack requrements");
149f4db079dSs.makeev 		}
150f4db079dSs.makeev 
15160ac17fbSs.makeev_local 		MT_ASSERT(fiberContext != nullptr, "Can't get more fibers. Too many tasks in flight simultaneously?");
15258964d0bSSergey Makeev 
15358964d0bSSergey Makeev 		fiberContext->currentTask = task.desc;
15458964d0bSSergey Makeev 		fiberContext->currentGroup = task.group;
15558964d0bSSergey Makeev 		fiberContext->parentFiber = task.parentFiber;
156f4db079dSs.makeev 		fiberContext->stackRequirements = stackRequirements;
15758964d0bSSergey Makeev 		return fiberContext;
15858964d0bSSergey Makeev 	}
15958964d0bSSergey Makeev 
16060ac17fbSs.makeev_local 	void TaskScheduler::ReleaseFiberContext(FiberContext*&& fiberContext)
16158964d0bSSergey Makeev 	{
162721f8c0bSs.makeev_local 		MT_ASSERT(fiberContext, "Can't release nullptr Fiber. fiberContext is nullptr");
163f4db079dSs.makeev 
164f4db079dSs.makeev 		MT::StackRequirements::Type stackRequirements = fiberContext->stackRequirements;
16558964d0bSSergey Makeev 		fiberContext->Reset();
166f4db079dSs.makeev 
1674965e494Ss.makeev_local 		MT_ASSERT(fiberContext != nullptr, "Fiber context can't be nullptr");
1684965e494Ss.makeev_local 
1694965e494Ss.makeev_local 		bool res = false;
1704965e494Ss.makeev_local 		MT_USED_IN_ASSERT(res);
1714965e494Ss.makeev_local 		switch(stackRequirements)
1724965e494Ss.makeev_local 		{
1734965e494Ss.makeev_local 		case MT::StackRequirements::STANDARD:
1744965e494Ss.makeev_local 			res = standartFibersAvailable.TryPush(std::move(fiberContext));
1754965e494Ss.makeev_local 			break;
1764965e494Ss.makeev_local 		case MT::StackRequirements::EXTENDED:
1774965e494Ss.makeev_local 			res = extendedFibersAvailable.TryPush(std::move(fiberContext));
1784965e494Ss.makeev_local 			break;
1794965e494Ss.makeev_local 		default:
1804965e494Ss.makeev_local 			MT_REPORT_ASSERT("Unknown stack requrements");
1814965e494Ss.makeev_local 		}
1824965e494Ss.makeev_local 
18360ac17fbSs.makeev_local 		MT_USED_IN_ASSERT(res);
18460ac17fbSs.makeev_local 		MT_ASSERT(res != false, "Can't return fiber to storage");
18558964d0bSSergey Makeev 	}
18658964d0bSSergey Makeev 
18758964d0bSSergey Makeev 	FiberContext* TaskScheduler::ExecuteTask(internal::ThreadContext& threadContext, FiberContext* fiberContext)
18858964d0bSSergey Makeev 	{
18934a394c3SSergey Makeev 		MT_ASSERT(threadContext.thread.IsCurrentThread(), "Thread context sanity check failed");
19058964d0bSSergey Makeev 
19134a394c3SSergey Makeev 		MT_ASSERT(fiberContext, "Invalid fiber context");
19234a394c3SSergey Makeev 		MT_ASSERT(fiberContext->currentTask.IsValid(), "Invalid task");
19358964d0bSSergey Makeev 
19458964d0bSSergey Makeev 		// Set actual thread context to fiber
19558964d0bSSergey Makeev 		fiberContext->SetThreadContext(&threadContext);
19658964d0bSSergey Makeev 
19758964d0bSSergey Makeev 		// Update task status
19858964d0bSSergey Makeev 		fiberContext->SetStatus(FiberTaskStatus::RUNNED);
19958964d0bSSergey Makeev 
20034a394c3SSergey Makeev 		MT_ASSERT(fiberContext->GetThreadContext()->thread.IsCurrentThread(), "Thread context sanity check failed");
20158964d0bSSergey Makeev 
202f4db079dSs.makeev 		const void* poolUserData = fiberContext->currentTask.userData;
20311d67161SSergey Makeev 		TPoolTaskDestroy poolDestroyFunc = fiberContext->currentTask.poolDestroyFunc;
20411d67161SSergey Makeev 
20560ac17fbSs.makeev_local #ifdef MT_INSTRUMENTED_BUILD
206*6e90b535Ss.makeev_local 		//threadContext.NotifyTaskExecuteStateChanged( MT_SYSTEM_TASK_COLOR, MT_SYSTEM_TASK_NAME, TaskExecuteState::SUSPEND);
207*6e90b535Ss.makeev_local 		threadContext.NotifyTaskExecuteStateChanged( MT_SYSTEM_TASK_COLOR, MT_SYSTEM_TASK_NAME, TaskExecuteState::STOP);
20860ac17fbSs.makeev_local #endif
20960ac17fbSs.makeev_local 
21058964d0bSSergey Makeev 		// Run current task code
21158964d0bSSergey Makeev 		Fiber::SwitchTo(threadContext.schedulerFiber, fiberContext->fiber);
21258964d0bSSergey Makeev 
21360ac17fbSs.makeev_local #ifdef MT_INSTRUMENTED_BUILD
214*6e90b535Ss.makeev_local 		//threadContext.NotifyTaskExecuteStateChanged( MT_SYSTEM_TASK_COLOR, MT_SYSTEM_TASK_NAME, TaskExecuteState::RESUME);
215*6e90b535Ss.makeev_local 		threadContext.NotifyTaskExecuteStateChanged( MT_SYSTEM_TASK_COLOR, MT_SYSTEM_TASK_NAME, TaskExecuteState::START);
21660ac17fbSs.makeev_local #endif
21760ac17fbSs.makeev_local 
21858964d0bSSergey Makeev 		// If task was done
21958964d0bSSergey Makeev 		FiberTaskStatus::Type taskStatus = fiberContext->GetStatus();
22058964d0bSSergey Makeev 		if (taskStatus == FiberTaskStatus::FINISHED)
22158964d0bSSergey Makeev 		{
22211d67161SSergey Makeev 			//destroy task (call dtor) for "fire and forget" type of task from TaskPool
22311d67161SSergey Makeev 			if (poolDestroyFunc != nullptr)
22411d67161SSergey Makeev 			{
22511d67161SSergey Makeev 				poolDestroyFunc(poolUserData);
22611d67161SSergey Makeev 			}
22711d67161SSergey Makeev 
228806ef292SSergey Makeev 			TaskGroup taskGroup = fiberContext->currentGroup;
22931bebe6fSSergey Makeev 
230806ef292SSergey Makeev 			TaskScheduler::TaskGroupDescription  & groupDesc = threadContext.taskScheduler->GetGroupDesc(taskGroup);
23158964d0bSSergey Makeev 
23258964d0bSSergey Makeev 			// Update group status
233806ef292SSergey Makeev 			int groupTaskCount = groupDesc.Dec();
23434a394c3SSergey Makeev 			MT_ASSERT(groupTaskCount >= 0, "Sanity check failed!");
23558964d0bSSergey Makeev 			if (groupTaskCount == 0)
23658964d0bSSergey Makeev 			{
237806ef292SSergey Makeev 				// Signal pending threads that group work is finished. Group can be destroyed after this call.
238806ef292SSergey Makeev 				groupDesc.Signal();
239806ef292SSergey Makeev 
240754cb997SSergey Makeev 				fiberContext->currentGroup = TaskGroup::INVALID;
24158964d0bSSergey Makeev 			}
24258964d0bSSergey Makeev 
24358964d0bSSergey Makeev 			// Update total task count
244806ef292SSergey Makeev 			int allGroupTaskCount = threadContext.taskScheduler->allGroups.Dec();
245806ef292SSergey Makeev 			MT_ASSERT(allGroupTaskCount >= 0, "Sanity check failed!");
246806ef292SSergey Makeev 			if (allGroupTaskCount == 0)
24758964d0bSSergey Makeev 			{
24858964d0bSSergey Makeev 				// Notify all tasks in all group finished
2492a2464ccSSergey Makeev 				threadContext.taskScheduler->allGroups.Signal();
25058964d0bSSergey Makeev 			}
25158964d0bSSergey Makeev 
25258964d0bSSergey Makeev 			FiberContext* parentFiberContext = fiberContext->parentFiber;
25358964d0bSSergey Makeev 			if (parentFiberContext != nullptr)
25458964d0bSSergey Makeev 			{
25581ec7369SSergey Makeev 				int childrenFibersCount = parentFiberContext->childrenFibersCount.DecFetch();
25634a394c3SSergey Makeev 				MT_ASSERT(childrenFibersCount >= 0, "Sanity check failed!");
25758964d0bSSergey Makeev 
25858964d0bSSergey Makeev 				if (childrenFibersCount == 0)
25958964d0bSSergey Makeev 				{
26058964d0bSSergey Makeev 					// This is a last subtask. Restore parent task
26134a394c3SSergey Makeev 					MT_ASSERT(threadContext.thread.IsCurrentThread(), "Thread context sanity check failed");
26234a394c3SSergey Makeev 					MT_ASSERT(parentFiberContext->GetThreadContext() == nullptr, "Inactive parent should not have a valid thread context");
26358964d0bSSergey Makeev 
26458964d0bSSergey Makeev 					// WARNING!! Thread context can changed here! Set actual current thread context.
26558964d0bSSergey Makeev 					parentFiberContext->SetThreadContext(&threadContext);
26658964d0bSSergey Makeev 
26734a394c3SSergey Makeev 					MT_ASSERT(parentFiberContext->GetThreadContext()->thread.IsCurrentThread(), "Thread context sanity check failed");
26858964d0bSSergey Makeev 
26958964d0bSSergey Makeev 					// All subtasks is done.
27058964d0bSSergey Makeev 					// Exiting and return parent fiber to scheduler
27158964d0bSSergey Makeev 					return parentFiberContext;
27258964d0bSSergey Makeev 				} else
27358964d0bSSergey Makeev 				{
27458964d0bSSergey Makeev 					// Other subtasks still exist
27558964d0bSSergey Makeev 					// Exiting
27658964d0bSSergey Makeev 					return nullptr;
27758964d0bSSergey Makeev 				}
27858964d0bSSergey Makeev 			} else
27958964d0bSSergey Makeev 			{
28058964d0bSSergey Makeev 				// Task is finished and no parent task
28158964d0bSSergey Makeev 				// Exiting
28258964d0bSSergey Makeev 				return nullptr;
28358964d0bSSergey Makeev 			}
28458964d0bSSergey Makeev 		}
28558964d0bSSergey Makeev 
28634a394c3SSergey Makeev 		MT_ASSERT(taskStatus != FiberTaskStatus::RUNNED, "Incorrect task status")
28758964d0bSSergey Makeev 		return nullptr;
28858964d0bSSergey Makeev 	}
28958964d0bSSergey Makeev 
29058964d0bSSergey Makeev 
29158964d0bSSergey Makeev 	void TaskScheduler::FiberMain(void* userData)
29258964d0bSSergey Makeev 	{
29358964d0bSSergey Makeev 		FiberContext& fiberContext = *(FiberContext*)(userData);
29458964d0bSSergey Makeev 		for(;;)
29558964d0bSSergey Makeev 		{
29634a394c3SSergey Makeev 			MT_ASSERT(fiberContext.currentTask.IsValid(), "Invalid task in fiber context");
29734a394c3SSergey Makeev 			MT_ASSERT(fiberContext.GetThreadContext(), "Invalid thread context");
29834a394c3SSergey Makeev 			MT_ASSERT(fiberContext.GetThreadContext()->thread.IsCurrentThread(), "Thread context sanity check failed");
29958964d0bSSergey Makeev 
30060ac17fbSs.makeev_local #ifdef MT_INSTRUMENTED_BUILD
30160ac17fbSs.makeev_local 			fiberContext.fiber.SetName( MT_SYSTEM_TASK_FIBER_NAME );
302*6e90b535Ss.makeev_local 			fiberContext.GetThreadContext()->NotifyTaskExecuteStateChanged( fiberContext.currentTask.debugColor, fiberContext.currentTask.debugID, TaskExecuteState::START );
30360ac17fbSs.makeev_local #endif
30458964d0bSSergey Makeev 
30560ac17fbSs.makeev_local 			fiberContext.currentTask.taskFunc( fiberContext, fiberContext.currentTask.userData );
30658964d0bSSergey Makeev 			fiberContext.SetStatus(FiberTaskStatus::FINISHED);
30758964d0bSSergey Makeev 
3088112dedfSSergey Makeev #ifdef MT_INSTRUMENTED_BUILD
30960ac17fbSs.makeev_local 			fiberContext.fiber.SetName( MT_SYSTEM_TASK_FIBER_NAME );
310*6e90b535Ss.makeev_local 			fiberContext.GetThreadContext()->NotifyTaskExecuteStateChanged( fiberContext.currentTask.debugColor, fiberContext.currentTask.debugID, TaskExecuteState::STOP );
3118112dedfSSergey Makeev #endif
3128112dedfSSergey Makeev 
31358964d0bSSergey Makeev 			Fiber::SwitchTo(fiberContext.fiber, fiberContext.GetThreadContext()->schedulerFiber);
31458964d0bSSergey Makeev 		}
31558964d0bSSergey Makeev 
31658964d0bSSergey Makeev 	}
31758964d0bSSergey Makeev 
31858964d0bSSergey Makeev 
319a1626843SSergey Makeev 	bool TaskScheduler::TryStealTask(internal::ThreadContext& threadContext, internal::GroupedTask & task, uint32 workersCount)
3202b4df162SSergey Makeev 	{
3212b4df162SSergey Makeev 		if (workersCount <= 1)
3222b4df162SSergey Makeev 		{
3232b4df162SSergey Makeev 			return false;
3242b4df162SSergey Makeev 		}
3252b4df162SSergey Makeev 
326a1626843SSergey Makeev 		uint32 victimIndex = threadContext.random.Get();
327a1626843SSergey Makeev 
328a1626843SSergey Makeev 		for (uint32 attempt = 0; attempt < workersCount; attempt++)
3292b4df162SSergey Makeev 		{
330a1626843SSergey Makeev 			uint32 index = victimIndex % workersCount;
331a1626843SSergey Makeev 			if (index == threadContext.workerIndex)
332a1626843SSergey Makeev 			{
333a1626843SSergey Makeev 				victimIndex++;
334a1626843SSergey Makeev 				index = victimIndex % workersCount;
3352b4df162SSergey Makeev 			}
3362b4df162SSergey Makeev 
337a1626843SSergey Makeev 			internal::ThreadContext& victimContext = threadContext.taskScheduler->threadContext[index];
338d7cf17b1Ss.makeev_local 			if (victimContext.queue.TryPopNewest(task))
339a1626843SSergey Makeev 			{
340a1626843SSergey Makeev 				return true;
341a1626843SSergey Makeev 			}
342a1626843SSergey Makeev 
343a1626843SSergey Makeev 			victimIndex++;
344a1626843SSergey Makeev 		}
345a1626843SSergey Makeev 		return false;
3462b4df162SSergey Makeev 	}
3472b4df162SSergey Makeev 
34802d170cfSs.makeev_local 
349c88507a8Ss.makeev_local 	void TaskScheduler::WorkerThreadMain( void* userData )
35058964d0bSSergey Makeev 	{
35158964d0bSSergey Makeev 		internal::ThreadContext& context = *(internal::ThreadContext*)(userData);
35234a394c3SSergey Makeev 		MT_ASSERT(context.taskScheduler, "Task scheduler must be not null!");
353c88507a8Ss.makeev_local 
354d7cf17b1Ss.makeev_local #ifdef MT_INSTRUMENTED_BUILD
355c88507a8Ss.makeev_local 		const char* threadNames[] = {"worker0","worker1","worker2","worker3","worker4","worker5","worker6","worker7","worker8","worker9","worker10","worker11","worker12"};
356c88507a8Ss.makeev_local 		if (context.workerIndex < MT_ARRAY_SIZE(threadNames))
357c88507a8Ss.makeev_local 		{
358d7cf17b1Ss.makeev_local 			Thread::SetThreadName(threadNames[context.workerIndex]);
359c88507a8Ss.makeev_local 		} else
360c88507a8Ss.makeev_local 		{
361d7cf17b1Ss.makeev_local 			Thread::SetThreadName("worker_thread");
362c88507a8Ss.makeev_local 		}
363d7cf17b1Ss.makeev_local #endif
364c88507a8Ss.makeev_local 
365ae5bbefbSs.makeev_local 		context.schedulerFiber.CreateFromCurrentThreadAndRun(SchedulerFiberMain, userData);
36602d170cfSs.makeev_local 	}
36702d170cfSs.makeev_local 
36802d170cfSs.makeev_local 
36971198a5eSs.makeev_local 	void TaskScheduler::SchedulerFiberMain( void* userData )
37002d170cfSs.makeev_local 	{
37102d170cfSs.makeev_local 		internal::ThreadContext& context = *(internal::ThreadContext*)(userData);
37202d170cfSs.makeev_local 		MT_ASSERT(context.taskScheduler, "Task scheduler must be not null!");
3730727c1f9SSergey Makeev 
3740727c1f9SSergey Makeev #ifdef MT_INSTRUMENTED_BUILD
3750727c1f9SSergey Makeev 		context.NotifyThreadCreate(context.workerIndex);
3760727c1f9SSergey Makeev #endif
3770727c1f9SSergey Makeev 
378a23fd933SSergey Makeev 		uint32 workersCount = context.taskScheduler->GetWorkersCount();
379a23fd933SSergey Makeev 		int32 totalThreadsCount = context.taskScheduler->threadsCount.LoadRelaxed();
380a1626843SSergey Makeev 
38181ec7369SSergey Makeev 		context.taskScheduler->startedThreadsCount.IncFetch();
38243c41d5cSSergey Makeev 
383a23fd933SSergey Makeev 		//Simple spinlock until all threads is started and initialized
3846bbc0262SSergey Makeev 		for(;;)
38543c41d5cSSergey Makeev 		{
386a23fd933SSergey Makeev 			int32 initializedThreadsCount = context.taskScheduler->startedThreadsCount.Load();
387a23fd933SSergey Makeev 			if (initializedThreadsCount == totalThreadsCount)
38843c41d5cSSergey Makeev 			{
38943c41d5cSSergey Makeev 				break;
39043c41d5cSSergey Makeev 			}
391d7cf17b1Ss.makeev_local 
392d7cf17b1Ss.makeev_local 			// sleep some time until all other thread initialized
39343c41d5cSSergey Makeev 			Thread::Sleep(1);
39443c41d5cSSergey Makeev 		}
39543c41d5cSSergey Makeev 
396f4db079dSs.makeev 		HardwareFullMemoryBarrier();
397f4db079dSs.makeev 
3980727c1f9SSergey Makeev #ifdef MT_INSTRUMENTED_BUILD
3990727c1f9SSergey Makeev 		context.NotifyThreadStart(context.workerIndex);
400*6e90b535Ss.makeev_local 		context.NotifyTaskExecuteStateChanged( MT_SYSTEM_TASK_COLOR, MT_SYSTEM_TASK_NAME, TaskExecuteState::START);
4010727c1f9SSergey Makeev #endif
4020727c1f9SSergey Makeev 
40381ec7369SSergey Makeev 		while(context.state.Load() != internal::ThreadState::EXIT)
40458964d0bSSergey Makeev 		{
40558964d0bSSergey Makeev 			internal::GroupedTask task;
406d7cf17b1Ss.makeev_local 			if (context.queue.TryPopOldest(task) || TryStealTask(context, task, workersCount) )
40758964d0bSSergey Makeev 			{
40860ac17fbSs.makeev_local #ifdef MT_INSTRUMENTED_BUILD
40960ac17fbSs.makeev_local 				bool isNewTask = (task.awaitingFiber == nullptr);
41060ac17fbSs.makeev_local #endif
41160ac17fbSs.makeev_local 
41258964d0bSSergey Makeev 				// There is a new task
41358964d0bSSergey Makeev 				FiberContext* fiberContext = context.taskScheduler->RequestFiberContext(task);
41434a394c3SSergey Makeev 				MT_ASSERT(fiberContext, "Can't get execution context from pool");
41534a394c3SSergey Makeev 				MT_ASSERT(fiberContext->currentTask.IsValid(), "Sanity check failed");
416f4db079dSs.makeev 				MT_ASSERT(fiberContext->stackRequirements == task.desc.stackRequirements, "Sanity check failed");
41758964d0bSSergey Makeev 
41858964d0bSSergey Makeev 				while(fiberContext)
41958964d0bSSergey Makeev 				{
4208112dedfSSergey Makeev #ifdef MT_INSTRUMENTED_BUILD
42160ac17fbSs.makeev_local 					if (isNewTask)
42260ac17fbSs.makeev_local 					{
42360ac17fbSs.makeev_local 						//TODO:
42460ac17fbSs.makeev_local 						isNewTask = false;
42560ac17fbSs.makeev_local 					}
4268112dedfSSergey Makeev #endif
42758964d0bSSergey Makeev 					// prevent invalid fiber resume from child tasks, before ExecuteTask is done
42881ec7369SSergey Makeev 					fiberContext->childrenFibersCount.IncFetch();
42958964d0bSSergey Makeev 
43058964d0bSSergey Makeev 					FiberContext* parentFiber = ExecuteTask(context, fiberContext);
43158964d0bSSergey Makeev 
43258964d0bSSergey Makeev 					FiberTaskStatus::Type taskStatus = fiberContext->GetStatus();
43358964d0bSSergey Makeev 
43458964d0bSSergey Makeev 					//release guard
43581ec7369SSergey Makeev 					int childrenFibersCount = fiberContext->childrenFibersCount.DecFetch();
43658964d0bSSergey Makeev 
43758964d0bSSergey Makeev 					// Can drop fiber context - task is finished
43858964d0bSSergey Makeev 					if (taskStatus == FiberTaskStatus::FINISHED)
43958964d0bSSergey Makeev 					{
44034a394c3SSergey Makeev 						MT_ASSERT( childrenFibersCount == 0, "Sanity check failed");
44160ac17fbSs.makeev_local 						context.taskScheduler->ReleaseFiberContext(std::move(fiberContext));
44258964d0bSSergey Makeev 
44358964d0bSSergey Makeev 						// If parent fiber is exist transfer flow control to parent fiber, if parent fiber is null, exit
44458964d0bSSergey Makeev 						fiberContext = parentFiber;
44558964d0bSSergey Makeev 					} else
44658964d0bSSergey Makeev 					{
44734a394c3SSergey Makeev 						MT_ASSERT( childrenFibersCount >= 0, "Sanity check failed");
44858964d0bSSergey Makeev 
44958964d0bSSergey Makeev 						// No subtasks here and status is not finished, this mean all subtasks already finished before parent return from ExecuteTask
45058964d0bSSergey Makeev 						if (childrenFibersCount == 0)
45158964d0bSSergey Makeev 						{
45234a394c3SSergey Makeev 							MT_ASSERT(parentFiber == nullptr, "Sanity check failed");
45358964d0bSSergey Makeev 						} else
45458964d0bSSergey Makeev 						{
45558964d0bSSergey Makeev 							// If subtasks still exist, drop current task execution. task will be resumed when last subtask finished
45658964d0bSSergey Makeev 							break;
45758964d0bSSergey Makeev 						}
45858964d0bSSergey Makeev 
45958964d0bSSergey Makeev 						// If task is in await state drop execution. task will be resumed when RestoreAwaitingTasks called
46058964d0bSSergey Makeev 						if (taskStatus == FiberTaskStatus::AWAITING_GROUP)
46158964d0bSSergey Makeev 						{
46258964d0bSSergey Makeev 							break;
46358964d0bSSergey Makeev 						}
46458964d0bSSergey Makeev 					}
46558964d0bSSergey Makeev 				} //while(fiberContext)
46658964d0bSSergey Makeev 
46758964d0bSSergey Makeev 			} else
46858964d0bSSergey Makeev 			{
469a1626843SSergey Makeev #ifdef MT_INSTRUMENTED_BUILD
4704a90b4a6SSergey Makeev 				context.NotifyThreadIdleBegin(context.workerIndex);
471a1626843SSergey Makeev #endif
472a1626843SSergey Makeev 
47360ac17fbSs.makeev_local 				// Queue if empty and stealing attempt failed
47460ac17fbSs.makeev_local 				// Wait for new events
47558964d0bSSergey Makeev 				context.hasNewTasksEvent.Wait(2000);
476a1626843SSergey Makeev 
477a1626843SSergey Makeev #ifdef MT_INSTRUMENTED_BUILD
4784a90b4a6SSergey Makeev 				context.NotifyThreadIdleEnd(context.workerIndex);
479a1626843SSergey Makeev #endif
480a1626843SSergey Makeev 
48158964d0bSSergey Makeev 			}
48258964d0bSSergey Makeev 
48358964d0bSSergey Makeev 		} // main thread loop
4840727c1f9SSergey Makeev 
4850727c1f9SSergey Makeev #ifdef MT_INSTRUMENTED_BUILD
486*6e90b535Ss.makeev_local 		context.NotifyTaskExecuteStateChanged( MT_SYSTEM_TASK_COLOR, MT_SYSTEM_TASK_NAME, TaskExecuteState::STOP);
4870727c1f9SSergey Makeev 		context.NotifyThreadStop(context.workerIndex);
4880727c1f9SSergey Makeev #endif
4890727c1f9SSergey Makeev 
49058964d0bSSergey Makeev 	}
49158964d0bSSergey Makeev 
49256aa031bSSergey Makeev 	void TaskScheduler::RunTasksImpl(ArrayView<internal::TaskBucket>& buckets, FiberContext * parentFiber, bool restoredFromAwaitState)
49358964d0bSSergey Makeev 	{
494806ef292SSergey Makeev 		// This storage is necessary to calculate how many tasks we add to different groups
495754cb997SSergey Makeev 		int newTaskCountInGroup[TaskGroup::MT_MAX_GROUPS_COUNT];
496806ef292SSergey Makeev 
497806ef292SSergey Makeev 		// Default value is 0
498806ef292SSergey Makeev 		memset(&newTaskCountInGroup[0], 0, sizeof(newTaskCountInGroup));
499806ef292SSergey Makeev 
50058964d0bSSergey Makeev 		// Set parent fiber pointer
50158964d0bSSergey Makeev 		// Calculate the number of tasks per group
50258964d0bSSergey Makeev 		// Calculate total number of tasks
50358964d0bSSergey Makeev 		size_t count = 0;
504a101e543SSergey Makeev 		for (size_t i = 0; i < buckets.Size(); ++i)
50558964d0bSSergey Makeev 		{
50658964d0bSSergey Makeev 			internal::TaskBucket& bucket = buckets[i];
50758964d0bSSergey Makeev 			for (size_t taskIndex = 0; taskIndex < bucket.count; taskIndex++)
50858964d0bSSergey Makeev 			{
50958964d0bSSergey Makeev 				internal::GroupedTask & task = bucket.tasks[taskIndex];
51058964d0bSSergey Makeev 
51158964d0bSSergey Makeev 				task.parentFiber = parentFiber;
512754cb997SSergey Makeev 
513754cb997SSergey Makeev 				int idx = task.group.GetValidIndex();
514754cb997SSergey Makeev 				MT_ASSERT(idx >= 0 && idx < TaskGroup::MT_MAX_GROUPS_COUNT, "Invalid index");
515754cb997SSergey Makeev 				newTaskCountInGroup[idx]++;
51631bebe6fSSergey Makeev 			}
51731bebe6fSSergey Makeev 
51858964d0bSSergey Makeev 			count += bucket.count;
51958964d0bSSergey Makeev 		}
52058964d0bSSergey Makeev 
52158964d0bSSergey Makeev 		// Increments child fibers count on parent fiber
52258964d0bSSergey Makeev 		if (parentFiber)
52358964d0bSSergey Makeev 		{
52481ec7369SSergey Makeev 			parentFiber->childrenFibersCount.AddFetch((int)count);
52558964d0bSSergey Makeev 		}
52658964d0bSSergey Makeev 
52758964d0bSSergey Makeev 		if (restoredFromAwaitState == false)
52858964d0bSSergey Makeev 		{
529806ef292SSergey Makeev 			// Increase the number of active tasks in the group using data from temporary storage
530754cb997SSergey Makeev 			for (size_t i = 0; i < TaskGroup::MT_MAX_GROUPS_COUNT; i++)
531806ef292SSergey Makeev 			{
532806ef292SSergey Makeev 				int groupNewTaskCount = newTaskCountInGroup[i];
533806ef292SSergey Makeev 				if (groupNewTaskCount > 0)
534806ef292SSergey Makeev 				{
535806ef292SSergey Makeev 					groupStats[i].Reset();
536806ef292SSergey Makeev 					groupStats[i].Add((uint32)groupNewTaskCount);
537806ef292SSergey Makeev 				}
538806ef292SSergey Makeev 			}
539806ef292SSergey Makeev 
54058964d0bSSergey Makeev 			// Increments all task in progress counter
5412a2464ccSSergey Makeev 			allGroups.Reset();
5422a2464ccSSergey Makeev 			allGroups.Add((uint32)count);
54358964d0bSSergey Makeev 		} else
54458964d0bSSergey Makeev 		{
54558964d0bSSergey Makeev 			// If task's restored from await state, counters already in correct state
54658964d0bSSergey Makeev 		}
54758964d0bSSergey Makeev 
54858964d0bSSergey Makeev 		// Add to thread queue
549a101e543SSergey Makeev 		for (size_t i = 0; i < buckets.Size(); ++i)
55058964d0bSSergey Makeev 		{
551a23fd933SSergey Makeev 			int bucketIndex = roundRobinThreadIndex.IncFetch() % threadsCount.LoadRelaxed();
55258964d0bSSergey Makeev 			internal::ThreadContext & context = threadContext[bucketIndex];
55358964d0bSSergey Makeev 
55458964d0bSSergey Makeev 			internal::TaskBucket& bucket = buckets[i];
55558964d0bSSergey Makeev 
556d7cf17b1Ss.makeev_local 			for(;;)
557d7cf17b1Ss.makeev_local 			{
558d7cf17b1Ss.makeev_local 				bool res = context.queue.Add(bucket.tasks, bucket.count);
559d7cf17b1Ss.makeev_local 				if (res == true)
560d7cf17b1Ss.makeev_local 				{
561d7cf17b1Ss.makeev_local 					break;
562d7cf17b1Ss.makeev_local 				}
563d7cf17b1Ss.makeev_local 
564d7cf17b1Ss.makeev_local 				//Can't add new tasks onto the queue. Look like the job system is overloaded. Wait some time and try again.
565d7cf17b1Ss.makeev_local 				//TODO: implement waiting until workers done using events.
566d7cf17b1Ss.makeev_local 				Thread::Sleep(10);
567d7cf17b1Ss.makeev_local 			}
568d7cf17b1Ss.makeev_local 
56958964d0bSSergey Makeev 			context.hasNewTasksEvent.Signal();
57058964d0bSSergey Makeev 		}
57158964d0bSSergey Makeev 	}
57258964d0bSSergey Makeev 
573f4db079dSs.makeev 	void TaskScheduler::RunAsync(TaskGroup group, const TaskHandle* taskHandleArray, uint32 taskHandleCount)
57458d12dadSSergey Makeev 	{
57558d12dadSSergey Makeev 		MT_ASSERT(!IsWorkerThread(), "Can't use RunAsync inside Task. Use FiberContext.RunAsync() instead.");
57658d12dadSSergey Makeev 
57758d12dadSSergey Makeev 		ArrayView<internal::GroupedTask> buffer(MT_ALLOCATE_ON_STACK(sizeof(internal::GroupedTask) * taskHandleCount), taskHandleCount);
57858d12dadSSergey Makeev 
579a23fd933SSergey Makeev 		uint32 bucketCount = MT::Min((uint32)GetWorkersCount(), taskHandleCount);
58058d12dadSSergey Makeev 		ArrayView<internal::TaskBucket> buckets(MT_ALLOCATE_ON_STACK(sizeof(internal::TaskBucket) * bucketCount), bucketCount);
58158d12dadSSergey Makeev 
58258d12dadSSergey Makeev 		internal::DistibuteDescriptions(group, taskHandleArray, buffer, buckets);
58358d12dadSSergey Makeev 		RunTasksImpl(buckets, nullptr, false);
58458d12dadSSergey Makeev 	}
58558d12dadSSergey Makeev 
586806ef292SSergey Makeev 	bool TaskScheduler::WaitGroup(TaskGroup group, uint32 milliseconds)
58758964d0bSSergey Makeev 	{
58834a394c3SSergey Makeev 		MT_VERIFY(IsWorkerThread() == false, "Can't use WaitGroup inside Task. Use FiberContext.WaitGroupAndYield() instead.", return false);
58958964d0bSSergey Makeev 
590806ef292SSergey Makeev 		TaskScheduler::TaskGroupDescription & groupDesc = GetGroupDesc(group);
5912f083884Ss.makeev_local 
592806ef292SSergey Makeev 		return groupDesc.Wait(milliseconds);
59358964d0bSSergey Makeev 	}
59458964d0bSSergey Makeev 
59558964d0bSSergey Makeev 	bool TaskScheduler::WaitAll(uint32 milliseconds)
59658964d0bSSergey Makeev 	{
59734a394c3SSergey Makeev 		MT_VERIFY(IsWorkerThread() == false, "Can't use WaitAll inside Task.", return false);
59858964d0bSSergey Makeev 
5992a2464ccSSergey Makeev 		return allGroups.Wait(milliseconds);
60058964d0bSSergey Makeev 	}
60158964d0bSSergey Makeev 
60258964d0bSSergey Makeev 	bool TaskScheduler::IsEmpty()
60358964d0bSSergey Makeev 	{
60458964d0bSSergey Makeev 		for (uint32 i = 0; i < MT_MAX_THREAD_COUNT; i++)
60558964d0bSSergey Makeev 		{
60658964d0bSSergey Makeev 			if (!threadContext[i].queue.IsEmpty())
60758964d0bSSergey Makeev 			{
60858964d0bSSergey Makeev 				return false;
60958964d0bSSergey Makeev 			}
61058964d0bSSergey Makeev 		}
61158964d0bSSergey Makeev 		return true;
61258964d0bSSergey Makeev 	}
61358964d0bSSergey Makeev 
614a23fd933SSergey Makeev 	int32 TaskScheduler::GetWorkersCount() const
61558964d0bSSergey Makeev 	{
616a23fd933SSergey Makeev 		return threadsCount.LoadRelaxed();
61758964d0bSSergey Makeev 	}
61858964d0bSSergey Makeev 
61958964d0bSSergey Makeev 	bool TaskScheduler::IsWorkerThread() const
62058964d0bSSergey Makeev 	{
62158964d0bSSergey Makeev 		for (uint32 i = 0; i < MT_MAX_THREAD_COUNT; i++)
62258964d0bSSergey Makeev 		{
62358964d0bSSergey Makeev 			if (threadContext[i].thread.IsCurrentThread())
62458964d0bSSergey Makeev 			{
62558964d0bSSergey Makeev 				return true;
62658964d0bSSergey Makeev 			}
62758964d0bSSergey Makeev 		}
62858964d0bSSergey Makeev 		return false;
62958964d0bSSergey Makeev 	}
63058964d0bSSergey Makeev 
631806ef292SSergey Makeev 	TaskGroup TaskScheduler::CreateGroup()
632806ef292SSergey Makeev 	{
633806ef292SSergey Makeev 		MT_ASSERT(IsWorkerThread() == false, "Can't use CreateGroup inside Task.");
634806ef292SSergey Makeev 
635754cb997SSergey Makeev 		TaskGroup group;
63660ac17fbSs.makeev_local 		if (!availableGroups.TryPop(group))
637806ef292SSergey Makeev 		{
6386bbc0262SSergey Makeev 			MT_REPORT_ASSERT("Group pool is empty");
639806ef292SSergey Makeev 		}
640806ef292SSergey Makeev 
641754cb997SSergey Makeev 		int idx = group.GetValidIndex();
6424965e494Ss.makeev_local 		MT_USED_IN_ASSERT(idx);
6432e846c40SSergey Makeev 		MT_ASSERT(groupStats[idx].GetDebugIsFree() == true, "Bad logic!");
6449d94a278Ss.makeev_local #if MT_GROUP_DEBUG
6452e846c40SSergey Makeev 		groupStats[idx].SetDebugIsFree(false);
6469d94a278Ss.makeev_local #endif
647806ef292SSergey Makeev 
648806ef292SSergey Makeev 		return group;
649806ef292SSergey Makeev 	}
650806ef292SSergey Makeev 
651806ef292SSergey Makeev 	void TaskScheduler::ReleaseGroup(TaskGroup group)
652806ef292SSergey Makeev 	{
653806ef292SSergey Makeev 		MT_ASSERT(IsWorkerThread() == false, "Can't use ReleaseGroup inside Task.");
654754cb997SSergey Makeev 		MT_ASSERT(group.IsValid(), "Invalid group ID");
655806ef292SSergey Makeev 
656754cb997SSergey Makeev 		int idx = group.GetValidIndex();
6574965e494Ss.makeev_local 		MT_USED_IN_ASSERT(idx);
6582e846c40SSergey Makeev 		MT_ASSERT(groupStats[idx].GetDebugIsFree() == false, "Group already released");
6599d94a278Ss.makeev_local #if MT_GROUP_DEBUG
6602e846c40SSergey Makeev 		groupStats[idx].SetDebugIsFree(true);
6619d94a278Ss.makeev_local #endif
662806ef292SSergey Makeev 
66360ac17fbSs.makeev_local 		bool res = availableGroups.TryPush(std::move(group));
66460ac17fbSs.makeev_local 		MT_USED_IN_ASSERT(res);
66560ac17fbSs.makeev_local 		MT_ASSERT(res, "Can't return group to pool");
666806ef292SSergey Makeev 	}
667806ef292SSergey Makeev 
668806ef292SSergey Makeev 	TaskScheduler::TaskGroupDescription & TaskScheduler::GetGroupDesc(TaskGroup group)
669806ef292SSergey Makeev 	{
670754cb997SSergey Makeev 		MT_ASSERT(group.IsValid(), "Invalid group ID");
671806ef292SSergey Makeev 
672754cb997SSergey Makeev 		int idx = group.GetValidIndex();
673754cb997SSergey Makeev 		TaskScheduler::TaskGroupDescription & groupDesc = groupStats[idx];
674806ef292SSergey Makeev 
6752e846c40SSergey Makeev 		MT_ASSERT(groupDesc.GetDebugIsFree() == false, "Invalid group");
676806ef292SSergey Makeev 		return groupDesc;
677806ef292SSergey Makeev 	}
67858964d0bSSergey Makeev }
67902d170cfSs.makeev_local 
68002d170cfSs.makeev_local 
681