1 /*===- InstrProfilingValue.c - Support library for PGO instrumentation ----===*\
2 |*
3 |*                     The LLVM Compiler Infrastructure
4 |*
5 |* This file is distributed under the University of Illinois Open Source
6 |* License. See LICENSE.TXT for details.
7 |*
8 \*===----------------------------------------------------------------------===*/
9 
10 #include "InstrProfiling.h"
11 #include "InstrProfilingInternal.h"
12 #include <limits.h>
13 #include <stdio.h>
14 #include <stdlib.h>
15 #include <string.h>
16 #define INSTR_PROF_VALUE_PROF_DATA
17 #define INSTR_PROF_COMMON_API_IMPL
18 #include "InstrProfData.inc"
19 
20 #define PROF_OOM(Msg) PROF_ERR(Msg ":%s\n", "Out of memory");
21 #define PROF_OOM_RETURN(Msg)                                                   \
22   {                                                                            \
23     PROF_OOM(Msg)                                                              \
24     free(ValueDataArray);                                                      \
25     return NULL;                                                               \
26   }
27 
28 /* This method is only used in value profiler mock testing.  */
29 COMPILER_RT_VISIBILITY void
30 __llvm_profile_set_num_value_sites(__llvm_profile_data *Data,
31                                    uint32_t ValueKind, uint16_t NumValueSites) {
32   *((uint16_t *)&Data->NumValueSites[ValueKind]) = NumValueSites;
33 }
34 
35 /* This method is only used in value profiler mock testing.  */
36 COMPILER_RT_VISIBILITY const __llvm_profile_data *
37 __llvm_profile_iterate_data(const __llvm_profile_data *Data) {
38   return Data + 1;
39 }
40 
41 /* This method is only used in value profiler mock testing.  */
42 COMPILER_RT_VISIBILITY void *
43 __llvm_get_function_addr(const __llvm_profile_data *Data) {
44   return Data->FunctionPointer;
45 }
46 
47 /* Allocate an array that holds the pointers to the linked lists of
48  * value profile counter nodes. The number of element of the array
49  * is the total number of value profile sites instrumented. Returns
50  * 0 if allocation fails.
51  */
52 
53 static int allocateValueProfileCounters(__llvm_profile_data *Data) {
54   uint64_t NumVSites = 0;
55   uint32_t VKI;
56   for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI)
57     NumVSites += Data->NumValueSites[VKI];
58 
59   ValueProfNode **Mem =
60       (ValueProfNode **)calloc(NumVSites, sizeof(ValueProfNode *));
61   if (!Mem)
62     return 0;
63   if (!COMPILER_RT_BOOL_CMPXCHG(&Data->Values, 0, Mem)) {
64     free(Mem);
65     return 0;
66   }
67   return 1;
68 }
69 
70 COMPILER_RT_VISIBILITY void
71 __llvm_profile_instrument_target(uint64_t TargetValue, void *Data,
72                                  uint32_t CounterIndex) {
73 
74   __llvm_profile_data *PData = (__llvm_profile_data *)Data;
75   if (!PData)
76     return;
77 
78   if (!PData->Values) {
79     if (!allocateValueProfileCounters(PData))
80       return;
81   }
82 
83   ValueProfNode **ValueCounters = (ValueProfNode **)PData->Values;
84   ValueProfNode *PrevVNode = NULL;
85   ValueProfNode *CurrentVNode = ValueCounters[CounterIndex];
86 
87   uint8_t VDataCount = 0;
88   while (CurrentVNode) {
89     if (TargetValue == CurrentVNode->VData.Value) {
90       CurrentVNode->VData.Count++;
91       return;
92     }
93     PrevVNode = CurrentVNode;
94     CurrentVNode = CurrentVNode->Next;
95     ++VDataCount;
96   }
97 
98   if (VDataCount >= INSTR_PROF_MAX_NUM_VAL_PER_SITE)
99     return;
100 
101   CurrentVNode = (ValueProfNode *)calloc(1, sizeof(ValueProfNode));
102   if (!CurrentVNode)
103     return;
104 
105   CurrentVNode->VData.Value = TargetValue;
106   CurrentVNode->VData.Count++;
107 
108   uint32_t Success = 0;
109   if (!ValueCounters[CounterIndex])
110     Success =
111         COMPILER_RT_BOOL_CMPXCHG(&ValueCounters[CounterIndex], 0, CurrentVNode);
112   else if (PrevVNode && !PrevVNode->Next)
113     Success = COMPILER_RT_BOOL_CMPXCHG(&(PrevVNode->Next), 0, CurrentVNode);
114 
115   if (!Success) {
116     free(CurrentVNode);
117     return;
118   }
119 }
120 
121 COMPILER_RT_VISIBILITY ValueProfData **
122 __llvm_profile_gather_value_data(uint64_t *ValueDataSize) {
123   size_t S = 0;
124   __llvm_profile_data *I;
125   ValueProfData **ValueDataArray;
126 
127   const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
128   const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
129 
130   if (!ValueDataSize)
131     return NULL;
132 
133   ValueDataArray = (ValueProfData **)calloc(
134       __llvm_profile_get_data_size(DataBegin, DataEnd), sizeof(void *));
135   if (!ValueDataArray)
136     PROF_OOM_RETURN("Failed to write value profile data ");
137 
138   /*
139    * Compute the total Size of the buffer to hold ValueProfData
140    * structures for functions with value profile data.
141    */
142   for (I = (__llvm_profile_data *)DataBegin; I < DataEnd; ++I) {
143     ValueProfRuntimeRecord R;
144     if (initializeValueProfRuntimeRecord(&R, I->NumValueSites, I->Values))
145       PROF_OOM_RETURN("Failed to write value profile data ");
146 
147     /* Compute the size of ValueProfData from this runtime record.  */
148     if (getNumValueKindsRT(&R) != 0) {
149       ValueProfData *VD = NULL;
150       uint32_t VS = getValueProfDataSizeRT(&R);
151       VD = (ValueProfData *)calloc(VS, sizeof(uint8_t));
152       if (!VD)
153         PROF_OOM_RETURN("Failed to write value profile data ");
154       serializeValueProfDataFromRT(&R, VD);
155       ValueDataArray[I - DataBegin] = VD;
156       S += VS;
157     }
158     finalizeValueProfRuntimeRecord(&R);
159   }
160 
161   if (!S) {
162     free(ValueDataArray);
163     ValueDataArray = NULL;
164   }
165 
166   *ValueDataSize = S;
167   return ValueDataArray;
168 }
169