Lines Matching refs:cpuctx
179 static void perf_ctx_lock(struct perf_cpu_context *cpuctx, in perf_ctx_lock() argument
182 __perf_ctx_lock(&cpuctx->ctx); in perf_ctx_lock()
202 static void perf_ctx_unlock(struct perf_cpu_context *cpuctx, in perf_ctx_unlock() argument
207 __perf_ctx_unlock(&cpuctx->ctx); in perf_ctx_unlock()
258 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); in event_function() local
259 struct perf_event_context *task_ctx = cpuctx->task_ctx; in event_function()
264 perf_ctx_lock(cpuctx, task_ctx); in event_function()
289 WARN_ON_ONCE(&cpuctx->ctx != ctx); in event_function()
292 efs->func(event, cpuctx, ctx, efs->data); in event_function()
294 perf_ctx_unlock(cpuctx, task_ctx); in event_function()
303 struct perf_cpu_context *cpuctx; in event_function_call() local
332 cpuctx = this_cpu_ptr(&perf_cpu_context); in event_function_call()
333 perf_ctx_lock(cpuctx, ctx); in event_function_call()
342 perf_ctx_unlock(cpuctx, ctx); in event_function_call()
348 perf_ctx_unlock(cpuctx, ctx); in event_function_call()
359 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); in event_function_local() local
372 perf_ctx_lock(cpuctx, task_ctx); in event_function_local()
388 if (WARN_ON_ONCE(cpuctx->task_ctx != ctx)) in event_function_local()
392 WARN_ON_ONCE(&cpuctx->ctx != ctx); in event_function_local()
395 func(event, cpuctx, ctx, data); in event_function_local()
397 perf_ctx_unlock(cpuctx, task_ctx); in event_function_local()
800 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); in perf_cgroup_match() local
807 if (!cpuctx->cgrp) in perf_cgroup_match()
816 return cgroup_is_descendant(cpuctx->cgrp->css.cgroup, in perf_cgroup_match()
861 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx, bool final) in update_cgrp_time_from_cpuctx() argument
863 struct perf_cgroup *cgrp = cpuctx->cgrp; in update_cgrp_time_from_cpuctx()
901 perf_cgroup_set_timestamp(struct perf_cpu_context *cpuctx) in perf_cgroup_set_timestamp() argument
903 struct perf_event_context *ctx = &cpuctx->ctx; in perf_cgroup_set_timestamp()
904 struct perf_cgroup *cgrp = cpuctx->cgrp; in perf_cgroup_set_timestamp()
931 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); in perf_cgroup_switch() local
938 if (READ_ONCE(cpuctx->cgrp) == NULL) in perf_cgroup_switch()
941 WARN_ON_ONCE(cpuctx->ctx.nr_cgroups == 0); in perf_cgroup_switch()
944 if (READ_ONCE(cpuctx->cgrp) == cgrp) in perf_cgroup_switch()
947 perf_ctx_lock(cpuctx, cpuctx->task_ctx); in perf_cgroup_switch()
948 perf_ctx_disable(&cpuctx->ctx, true); in perf_cgroup_switch()
950 ctx_sched_out(&cpuctx->ctx, NULL, EVENT_ALL|EVENT_CGROUP); in perf_cgroup_switch()
956 cpuctx->cgrp = cgrp; in perf_cgroup_switch()
962 ctx_sched_in(&cpuctx->ctx, NULL, EVENT_ALL|EVENT_CGROUP); in perf_cgroup_switch()
964 perf_ctx_enable(&cpuctx->ctx, true); in perf_cgroup_switch()
965 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); in perf_cgroup_switch()
971 struct perf_cpu_context *cpuctx; in perf_cgroup_ensure_storage() local
983 cpuctx = per_cpu_ptr(&perf_cpu_context, cpu); in perf_cgroup_ensure_storage()
984 if (heap_size <= cpuctx->heap_size) in perf_cgroup_ensure_storage()
994 raw_spin_lock_irq(&cpuctx->ctx.lock); in perf_cgroup_ensure_storage()
995 if (cpuctx->heap_size < heap_size) { in perf_cgroup_ensure_storage()
996 swap(cpuctx->heap, storage); in perf_cgroup_ensure_storage()
997 if (storage == cpuctx->heap_default) in perf_cgroup_ensure_storage()
999 cpuctx->heap_size = heap_size; in perf_cgroup_ensure_storage()
1001 raw_spin_unlock_irq(&cpuctx->ctx.lock); in perf_cgroup_ensure_storage()
1048 struct perf_cpu_context *cpuctx; in perf_cgroup_event_enable() local
1059 cpuctx = container_of(ctx, struct perf_cpu_context, ctx); in perf_cgroup_event_enable()
1064 cpuctx->cgrp = perf_cgroup_from_task(current, ctx); in perf_cgroup_event_enable()
1070 struct perf_cpu_context *cpuctx; in perf_cgroup_event_disable() local
1081 cpuctx = container_of(ctx, struct perf_cpu_context, ctx); in perf_cgroup_event_disable()
1086 cpuctx->cgrp = NULL; in perf_cgroup_event_disable()
1109 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx, in update_cgrp_time_from_cpuctx() argument
1122 perf_cgroup_set_timestamp(struct perf_cpu_context *cpuctx) in perf_cgroup_set_timestamp() argument
2413 __ctx_time_update(struct perf_cpu_context *cpuctx, struct perf_event_context *ctx, bool final) in __ctx_time_update() argument
2419 update_cgrp_time_from_cpuctx(cpuctx, final); in __ctx_time_update()
2424 ctx_time_update(struct perf_cpu_context *cpuctx, struct perf_event_context *ctx) in ctx_time_update() argument
2426 __ctx_time_update(cpuctx, ctx, false); in ctx_time_update()
2433 ctx_time_freeze(struct perf_cpu_context *cpuctx, struct perf_event_context *ctx) in ctx_time_freeze() argument
2435 ctx_time_update(cpuctx, ctx); in ctx_time_freeze()
2464 struct perf_cpu_context *cpuctx, in __perf_remove_from_context() argument
2472 ctx_time_update(cpuctx, ctx); in __perf_remove_from_context()
2504 if (ctx == &cpuctx->ctx) in __perf_remove_from_context()
2505 update_cgrp_time_from_cpuctx(cpuctx, true); in __perf_remove_from_context()
2509 WARN_ON_ONCE(cpuctx->task_ctx != ctx); in __perf_remove_from_context()
2510 cpuctx->task_ctx = NULL; in __perf_remove_from_context()
2552 struct perf_cpu_context *cpuctx, in __perf_event_disable() argument
2779 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); in task_ctx_sched_out() local
2781 if (!cpuctx->task_ctx) in task_ctx_sched_out()
2784 if (WARN_ON_ONCE(ctx != cpuctx->task_ctx)) in task_ctx_sched_out()
2790 static void perf_event_sched_in(struct perf_cpu_context *cpuctx, in perf_event_sched_in() argument
2794 ctx_sched_in(&cpuctx->ctx, pmu, EVENT_PINNED); in perf_event_sched_in()
2797 ctx_sched_in(&cpuctx->ctx, pmu, EVENT_FLEXIBLE); in perf_event_sched_in()
2817 static void ctx_resched(struct perf_cpu_context *cpuctx, in ctx_resched() argument
2833 for_each_epc(epc, &cpuctx->ctx, pmu, false) in ctx_resched()
2851 ctx_sched_out(&cpuctx->ctx, pmu, event_type); in ctx_resched()
2853 ctx_sched_out(&cpuctx->ctx, pmu, EVENT_FLEXIBLE); in ctx_resched()
2855 perf_event_sched_in(cpuctx, task_ctx, pmu); in ctx_resched()
2857 for_each_epc(epc, &cpuctx->ctx, pmu, false) in ctx_resched()
2868 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); in perf_pmu_resched() local
2869 struct perf_event_context *task_ctx = cpuctx->task_ctx; in perf_pmu_resched()
2871 perf_ctx_lock(cpuctx, task_ctx); in perf_pmu_resched()
2872 ctx_resched(cpuctx, task_ctx, pmu, EVENT_ALL|EVENT_CPU); in perf_pmu_resched()
2873 perf_ctx_unlock(cpuctx, task_ctx); in perf_pmu_resched()
2886 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); in __perf_install_in_context() local
2887 struct perf_event_context *task_ctx = cpuctx->task_ctx; in __perf_install_in_context()
2891 raw_spin_lock(&cpuctx->ctx.lock); in __perf_install_in_context()
2910 WARN_ON_ONCE(reprogram && cpuctx->task_ctx && cpuctx->task_ctx != ctx); in __perf_install_in_context()
2928 ctx_time_freeze(cpuctx, ctx); in __perf_install_in_context()
2930 ctx_resched(cpuctx, task_ctx, event->pmu_ctx->pmu, in __perf_install_in_context()
2937 perf_ctx_unlock(cpuctx, task_ctx); in __perf_install_in_context()
3063 struct perf_cpu_context *cpuctx, in __perf_event_enable() argument
3074 ctx_time_freeze(cpuctx, ctx); in __perf_event_enable()
3092 task_ctx = cpuctx->task_ctx; in __perf_event_enable()
3096 ctx_resched(cpuctx, task_ctx, event->pmu_ctx->pmu, get_event_type(event)); in __perf_event_enable()
3409 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); in ctx_sched_out() local
3424 WARN_ON_ONCE(cpuctx->task_ctx); in ctx_sched_out()
3438 __ctx_time_update(cpuctx, ctx, ctx == &cpuctx->ctx); in ctx_sched_out()
3459 WARN_ON_ONCE(cpuctx->task_ctx != ctx); in ctx_sched_out()
3461 cpuctx->task_ctx = NULL; in ctx_sched_out()
3718 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); in __perf_pmu_sched_task() local
3727 perf_ctx_lock(cpuctx, cpuctx->task_ctx); in __perf_pmu_sched_task()
3733 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); in __perf_pmu_sched_task()
3740 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); in perf_pmu_sched_task() local
3744 if (prev == next || cpuctx->task_ctx) in perf_pmu_sched_task()
3830 struct perf_cpu_context *cpuctx = NULL; in visit_groups_merge() local
3841 cpuctx = this_cpu_ptr(&perf_cpu_context); in visit_groups_merge()
3843 .data = cpuctx->heap, in visit_groups_merge()
3845 .size = cpuctx->heap_size, in visit_groups_merge()
3848 lockdep_assert_held(&cpuctx->ctx.lock); in visit_groups_merge()
3851 if (cpuctx->cgrp) in visit_groups_merge()
3852 css = &cpuctx->cgrp->css; in visit_groups_merge()
3984 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); in ctx_sched_in() local
3999 perf_cgroup_set_timestamp(cpuctx); in ctx_sched_in()
4010 cpuctx->task_ctx = ctx; in ctx_sched_in()
4012 WARN_ON_ONCE(cpuctx->task_ctx != ctx); in ctx_sched_in()
4035 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); in perf_event_context_sched_in() local
4043 if (cpuctx->task_ctx == ctx) { in perf_event_context_sched_in()
4044 perf_ctx_lock(cpuctx, ctx); in perf_event_context_sched_in()
4050 perf_ctx_unlock(cpuctx, ctx); in perf_event_context_sched_in()
4054 perf_ctx_lock(cpuctx, ctx); in perf_event_context_sched_in()
4072 perf_ctx_disable(&cpuctx->ctx, false); in perf_event_context_sched_in()
4073 ctx_sched_out(&cpuctx->ctx, NULL, EVENT_FLEXIBLE); in perf_event_context_sched_in()
4076 perf_event_sched_in(cpuctx, ctx, NULL); in perf_event_context_sched_in()
4078 perf_ctx_sched_task_cb(cpuctx->task_ctx, task, true); in perf_event_context_sched_in()
4081 perf_ctx_enable(&cpuctx->ctx, false); in perf_event_context_sched_in()
4086 perf_ctx_unlock(cpuctx, ctx); in perf_event_context_sched_in()
4379 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); in perf_rotate_context() local
4400 perf_ctx_lock(cpuctx, cpuctx->task_ctx); in perf_rotate_context()
4418 update_context_time(&cpuctx->ctx); in perf_rotate_context()
4420 rotate_ctx(&cpuctx->ctx, cpu_event); in perf_rotate_context()
4431 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); in perf_rotate_context()
4438 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); in perf_event_task_tick() local
4448 perf_adjust_freq_unthr_context(&cpuctx->ctx, !!throttled); in perf_event_task_tick()
4480 struct perf_cpu_context *cpuctx; in perf_event_enable_on_exec() local
4492 cpuctx = this_cpu_ptr(&perf_cpu_context); in perf_event_enable_on_exec()
4493 perf_ctx_lock(cpuctx, ctx); in perf_event_enable_on_exec()
4494 ctx_time_freeze(cpuctx, ctx); in perf_event_enable_on_exec()
4506 ctx_resched(cpuctx, ctx, NULL, event_type); in perf_event_enable_on_exec()
4508 perf_ctx_unlock(cpuctx, ctx); in perf_event_enable_on_exec()
4603 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); in __perf_event_read() local
4613 if (ctx->task && cpuctx->task_ctx != ctx) in __perf_event_read()
4888 struct perf_cpu_context *cpuctx; in find_get_context() local
4898 cpuctx = per_cpu_ptr(&perf_cpu_context, event->cpu); in find_get_context()
4899 ctx = &cpuctx->ctx; in find_get_context()
6153 struct perf_cpu_context *cpuctx, in __perf_event_period() argument
8597 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); in __perf_pmu_output_stop() local
8603 perf_iterate_ctx(&cpuctx->ctx, __perf_event_output_stop, &ro, false); in __perf_pmu_output_stop()
8604 if (cpuctx->task_ctx) in __perf_pmu_output_stop()
8605 perf_iterate_ctx(cpuctx->task_ctx, __perf_event_output_stop, in __perf_pmu_output_stop()
13239 struct perf_cpu_context *cpuctx = per_cpu_ptr(&perf_cpu_context, event->cpu); in SYSCALL_DEFINE5() local
13241 if (!cpuctx->online) { in SYSCALL_DEFINE5()
13525 struct perf_cpu_context *cpuctx = in perf_event_create_kernel_counter() local
13527 if (!cpuctx->online) { in perf_event_create_kernel_counter()
14279 struct perf_cpu_context *cpuctx; in perf_event_init_all_cpus() local
14299 cpuctx = per_cpu_ptr(&perf_cpu_context, cpu); in perf_event_init_all_cpus()
14300 __perf_event_init_context(&cpuctx->ctx); in perf_event_init_all_cpus()
14301 lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex); in perf_event_init_all_cpus()
14302 lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock); in perf_event_init_all_cpus()
14303 cpuctx->online = cpumask_test_cpu(cpu, perf_online_mask); in perf_event_init_all_cpus()
14304 cpuctx->heap_size = ARRAY_SIZE(cpuctx->heap_default); in perf_event_init_all_cpus()
14305 cpuctx->heap = cpuctx->heap_default; in perf_event_init_all_cpus()
14327 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); in __perf_event_exit_context() local
14334 __perf_remove_from_context(event, cpuctx, ctx, (void *)DETACH_GROUP); in __perf_event_exit_context()
14374 struct perf_cpu_context *cpuctx; in perf_event_exit_cpu_context() local
14384 cpuctx = per_cpu_ptr(&perf_cpu_context, cpu); in perf_event_exit_cpu_context()
14385 ctx = &cpuctx->ctx; in perf_event_exit_cpu_context()
14389 cpuctx->online = 0; in perf_event_exit_cpu_context()
14437 struct perf_cpu_context *cpuctx; in perf_event_init_cpu() local
14444 cpuctx = per_cpu_ptr(&perf_cpu_context, cpu); in perf_event_init_cpu()
14445 ctx = &cpuctx->ctx; in perf_event_init_cpu()
14448 cpuctx->online = 1; in perf_event_init_cpu()