1 /*===- InstrProfiling.c - Support library for PGO instrumentation ---------===*\
2 |*
3 |*                     The LLVM Compiler Infrastructure
4 |*
5 |* This file is distributed under the University of Illinois Open Source
6 |* License. See LICENSE.TXT for details.
7 |*
8 \*===----------------------------------------------------------------------===*/
9 
10 #include "InstrProfiling.h"
11 #include <limits.h>
12 #include <stdio.h>
13 #include <stdlib.h>
14 #include <string.h>
15 #define INSTR_PROF_VALUE_PROF_DATA
16 #define INSTR_PROF_COMMON_API_IMPL
17 #include "InstrProfData.inc"
18 
19 #define PROF_OOM(Msg) PROF_ERR(Msg ":%s\n", "Out of memory");
20 #define PROF_OOM_RETURN(Msg)                                                   \
21   {                                                                            \
22     PROF_OOM(Msg)                                                              \
23     return 0;                                                                  \
24   }
25 
26 #ifdef _MIPS_ARCH
27 LLVM_LIBRARY_VISIBILITY
28 uint32_t BoolCmpXchg(void **Ptr, void *OldV, void *NewV) {
29   void *R = *Ptr;
30   if (R == OldV) {
31     *Ptr = NewV;
32     return 1;
33   }
34   return 0;
35 }
36 #define BOOL_CMPXCHG(Ptr, OldV, NewV) BoolCmpXchg((void **)Ptr, OldV, NewV)
37 #else
38 #define BOOL_CMPXCHG(Ptr, OldV, NewV)                                          \
39   __sync_bool_compare_and_swap(Ptr, OldV, NewV)
40 #endif
41 
42 LLVM_LIBRARY_VISIBILITY uint64_t __llvm_profile_get_magic(void) {
43   return sizeof(void *) == sizeof(uint64_t) ? (INSTR_PROF_RAW_MAGIC_64)
44                                             : (INSTR_PROF_RAW_MAGIC_32);
45 }
46 
47 /* Return the number of bytes needed to add to SizeInBytes to make it
48  *   the result a multiple of 8.
49  */
50 LLVM_LIBRARY_VISIBILITY uint8_t
51 __llvm_profile_get_num_padding_bytes(uint64_t SizeInBytes) {
52   return 7 & (sizeof(uint64_t) - SizeInBytes % sizeof(uint64_t));
53 }
54 
55 LLVM_LIBRARY_VISIBILITY uint64_t __llvm_profile_get_version(void) {
56   return INSTR_PROF_RAW_VERSION;
57 }
58 
59 LLVM_LIBRARY_VISIBILITY void __llvm_profile_reset_counters(void) {
60   uint64_t *I = __llvm_profile_begin_counters();
61   uint64_t *E = __llvm_profile_end_counters();
62 
63   memset(I, 0, sizeof(uint64_t) * (E - I));
64 
65   const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
66   const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
67   const __llvm_profile_data *DI;
68   for (DI = DataBegin; DI != DataEnd; ++DI) {
69     uint64_t CurrentVSiteCount = 0;
70     uint32_t VKI, i;
71     if (!DI->Values)
72       continue;
73 
74     ValueProfNode **ValueCounters = (ValueProfNode **)DI->Values;
75 
76     for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI)
77       CurrentVSiteCount += DI->NumValueSites[VKI];
78 
79     for (i = 0; i < CurrentVSiteCount; ++i) {
80       ValueProfNode *CurrentVNode = ValueCounters[i];
81 
82       while (CurrentVNode) {
83         CurrentVNode->VData.Count = 0;
84         CurrentVNode = CurrentVNode->Next;
85       }
86     }
87   }
88 }
89 
90 /* This method is only used in value profiler mock testing.  */
91 LLVM_LIBRARY_VISIBILITY void
92 __llvm_profile_set_num_value_sites(__llvm_profile_data *Data,
93                                    uint32_t ValueKind, uint16_t NumValueSites) {
94   *((uint16_t *)&Data->NumValueSites[ValueKind]) = NumValueSites;
95 }
96 
97 /* This method is only used in value profiler mock testing.  */
98 LLVM_LIBRARY_VISIBILITY const __llvm_profile_data *
99 __llvm_profile_iterate_data(const __llvm_profile_data *Data) {
100   return Data + 1;
101 }
102 
103 /* This method is only used in value profiler mock testing.  */
104 LLVM_LIBRARY_VISIBILITY void *
105 __llvm_get_function_addr(const __llvm_profile_data *Data) {
106   return Data->FunctionPointer;
107 }
108 
109 /* Allocate an array that holds the pointers to the linked lists of
110  * value profile counter nodes. The number of element of the array
111  * is the total number of value profile sites instrumented. Returns
112  * 0 if allocation fails.
113  */
114 
115 static int allocateValueProfileCounters(__llvm_profile_data *Data) {
116   uint64_t NumVSites = 0;
117   uint32_t VKI;
118   for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI)
119     NumVSites += Data->NumValueSites[VKI];
120 
121   ValueProfNode **Mem =
122       (ValueProfNode **)calloc(NumVSites, sizeof(ValueProfNode *));
123   if (!Mem)
124     return 0;
125   if (!BOOL_CMPXCHG(&Data->Values, 0, Mem)) {
126     free(Mem);
127     return 0;
128   }
129   return 1;
130 }
131 
132 static void deallocateValueProfileCounters(__llvm_profile_data *Data) {
133   uint64_t NumVSites = 0, I;
134   uint32_t VKI;
135   if (!Data->Values)
136     return;
137   for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI)
138     NumVSites += Data->NumValueSites[VKI];
139   for (I = 0; I < NumVSites; I++) {
140     ValueProfNode *Node = ((ValueProfNode **)Data->Values)[I];
141     while (Node) {
142       ValueProfNode *Next = Node->Next;
143       free(Node);
144       Node = Next;
145     }
146   }
147   free(Data->Values);
148 }
149 
150 LLVM_LIBRARY_VISIBILITY void
151 __llvm_profile_instrument_target(uint64_t TargetValue, void *Data,
152                                  uint32_t CounterIndex) {
153 
154   __llvm_profile_data *PData = (__llvm_profile_data *)Data;
155   if (!PData)
156     return;
157 
158   if (!PData->Values) {
159     if (!allocateValueProfileCounters(PData))
160       return;
161   }
162 
163   ValueProfNode **ValueCounters = (ValueProfNode **)PData->Values;
164   ValueProfNode *PrevVNode = NULL;
165   ValueProfNode *CurrentVNode = ValueCounters[CounterIndex];
166 
167   uint8_t VDataCount = 0;
168   while (CurrentVNode) {
169     if (TargetValue == CurrentVNode->VData.Value) {
170       CurrentVNode->VData.Count++;
171       return;
172     }
173     PrevVNode = CurrentVNode;
174     CurrentVNode = CurrentVNode->Next;
175     ++VDataCount;
176   }
177 
178   if (VDataCount >= UCHAR_MAX)
179     return;
180 
181   CurrentVNode = (ValueProfNode *)calloc(1, sizeof(ValueProfNode));
182   if (!CurrentVNode)
183     return;
184 
185   CurrentVNode->VData.Value = TargetValue;
186   CurrentVNode->VData.Count++;
187 
188   uint32_t Success = 0;
189   if (!ValueCounters[CounterIndex])
190     Success = BOOL_CMPXCHG(&ValueCounters[CounterIndex], 0, CurrentVNode);
191   else if (PrevVNode && !PrevVNode->Next)
192     Success = BOOL_CMPXCHG(&(PrevVNode->Next), 0, CurrentVNode);
193 
194   if (!Success) {
195     free(CurrentVNode);
196     return;
197   }
198 }
199 
200 /* For multi-threaded programs, while the profile is being dumped, other
201    threads may still be updating the value profile data and creating new
202    value entries. To accommadate this, we need to add extra bytes to the
203    data buffer. The size of the extra space is controlled by an environment
204    varaible. */
205 static unsigned getVprofExtraBytes() {
206   const char *ExtraStr = getenv("LLVM_VALUE_PROF_BUFFER_EXTRA");
207   if (!ExtraStr || !ExtraStr[0])
208     return 1024;
209   return (unsigned)atoi(ExtraStr);
210 }
211 
212 LLVM_LIBRARY_VISIBILITY uint64_t
213 __llvm_profile_gather_value_data(uint8_t **VDataArray) {
214   size_t S = 0, RealSize = 0, BufferCapacity = 0, Extra = 0;
215   __llvm_profile_data *I;
216   if (!VDataArray)
217     PROF_OOM_RETURN("Failed to write value profile data ");
218 
219   const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
220   const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
221 
222   /*
223    * Compute the total Size of the buffer to hold ValueProfData
224    * structures for functions with value profile data.
225    */
226   for (I = (__llvm_profile_data *)DataBegin; I != DataEnd; ++I) {
227     ValueProfRuntimeRecord R;
228     /* Extract the value profile data info from the runtime. */
229     if (initializeValueProfRuntimeRecord(&R, I->NumValueSites, I->Values))
230       PROF_OOM_RETURN("Failed to write value profile data ");
231     /* Compute the size of ValueProfData from this runtime record.  */
232     if (getNumValueKindsRT(&R) != 0)
233       S += getValueProfDataSizeRT(&R);
234     finalizeValueProfRuntimeRecord(&R);
235   }
236   /* No value sites or no value profile data is collected. */
237   if (!S)
238     return 0;
239 
240   Extra = getVprofExtraBytes();
241   BufferCapacity = S + Extra;
242   *VDataArray = calloc(BufferCapacity, sizeof(uint8_t));
243   if (!*VDataArray)
244     PROF_OOM_RETURN("Failed to write value profile data ");
245 
246   ValueProfData *VD = (ValueProfData *)(*VDataArray);
247   /*
248    * Extract value profile data and write into ValueProfData structure
249    * one by one. Note that new value profile data added to any value
250    * site (from another thread) after the ValueProfRuntimeRecord is
251    * initialized (when the profile data snapshot is taken) won't be
252    * collected. This is not a problem as those dropped value will have
253    * very low taken count.
254    */
255   for (I = (__llvm_profile_data *)DataBegin; I != DataEnd; ++I) {
256     ValueProfRuntimeRecord R;
257     if (initializeValueProfRuntimeRecord(&R, I->NumValueSites, I->Values))
258       PROF_OOM_RETURN("Failed to write value profile data ");
259     if (getNumValueKindsRT(&R) == 0)
260       continue;
261 
262     /* Record R has taken a snapshot of the VP data at this point. Newly
263        added VP data for this function will be dropped.  */
264     /* Check if there is enough space.  */
265     if (BufferCapacity - RealSize < getValueProfDataSizeRT(&R)) {
266       PROF_ERR("Value profile data is dropped :%s \n",
267                "Out of buffer space. Use environment "
268                " LLVM_VALUE_PROF_BUFFER_EXTRA to allocate more");
269       I->Values = 0;
270     }
271 
272     serializeValueProfDataFromRT(&R, VD);
273     deallocateValueProfileCounters(I);
274     I->Values = VD;
275     finalizeValueProfRuntimeRecord(&R);
276     RealSize += VD->TotalSize;
277     VD = (ValueProfData *)((char *)VD + VD->TotalSize);
278   }
279 
280   return RealSize;
281 }
282