1 /*===- InstrProfiling.c - Support library for PGO instrumentation ---------===*\
2 |*
3 |*                     The LLVM Compiler Infrastructure
4 |*
5 |* This file is distributed under the University of Illinois Open Source
6 |* License. See LICENSE.TXT for details.
7 |*
8 \*===----------------------------------------------------------------------===*/
9 
10 #include "InstrProfiling.h"
11 #include <limits.h>
12 #include <stdlib.h>
13 #include <string.h>
14 
15 LLVM_LIBRARY_VISIBILITY uint64_t __llvm_profile_get_magic(void) {
16   return sizeof(void *) == sizeof(uint64_t) ? (INSTR_PROF_RAW_MAGIC_64)
17                                             : (INSTR_PROF_RAW_MAGIC_32);
18 }
19 
20 /* Return the number of bytes needed to add to SizeInBytes to make it
21  *   the result a multiple of 8.
22  */
23 LLVM_LIBRARY_VISIBILITY uint8_t
24 __llvm_profile_get_num_padding_bytes(uint64_t SizeInBytes) {
25   return 7 & (sizeof(uint64_t) - SizeInBytes % sizeof(uint64_t));
26 }
27 
28 LLVM_LIBRARY_VISIBILITY uint64_t __llvm_profile_get_version(void) {
29   return INSTR_PROF_RAW_VERSION;
30 }
31 
32 LLVM_LIBRARY_VISIBILITY void __llvm_profile_reset_counters(void) {
33   uint64_t *I = __llvm_profile_begin_counters();
34   uint64_t *E = __llvm_profile_end_counters();
35 
36   memset(I, 0, sizeof(uint64_t) * (E - I));
37 
38   const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
39   const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
40   const __llvm_profile_data *DI;
41   for (DI = DataBegin; DI != DataEnd; ++DI) {
42     uint64_t CurrentVSiteCount = 0;
43     uint32_t VKI, i;
44     if (!DI->Values)
45       continue;
46 
47     ValueProfNode **ValueCounters = (ValueProfNode **)DI->Values;
48 
49     for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI)
50       CurrentVSiteCount += DI->NumValueSites[VKI];
51 
52     for (i = 0; i < CurrentVSiteCount; ++i) {
53       ValueProfNode *CurrentVNode = ValueCounters[i];
54 
55       while (CurrentVNode) {
56         CurrentVNode->VData.Count = 0;
57         CurrentVNode = CurrentVNode->Next;
58       }
59     }
60   }
61 }
62 
63 /* Total number of value profile data in bytes. */
64 static uint64_t TotalValueDataSize = 0;
65 
66 #ifdef _MIPS_ARCH
67 LLVM_LIBRARY_VISIBILITY void
68 __llvm_profile_instrument_target(uint64_t TargetValue, void *Data_,
69                                  uint32_t CounterIndex) {}
70 
71 #else
72 
73 /* Allocate an array that holds the pointers to the linked lists of
74  * value profile counter nodes. The number of element of the array
75  * is the total number of value profile sites instrumented. Returns
76  *  0 if allocation fails.
77  */
78 
79 static int allocateValueProfileCounters(__llvm_profile_data *Data) {
80   uint64_t NumVSites = 0;
81   uint32_t VKI;
82   for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI)
83     NumVSites += Data->NumValueSites[VKI];
84 
85   ValueProfNode **Mem =
86       (ValueProfNode **)calloc(NumVSites, sizeof(ValueProfNode *));
87   if (!Mem)
88     return 0;
89   if (!__sync_bool_compare_and_swap(&Data->Values, 0, Mem)) {
90     free(Mem);
91     return 0;
92   }
93   /*  In the raw format, there will be an value count array preceding
94    *  the value profile data. The element type of the array is uint8_t,
95    *  and there is one element in array per value site. The element
96    *  stores the number of values profiled for the corresponding site.
97    */
98   uint8_t Padding = __llvm_profile_get_num_padding_bytes(NumVSites);
99   __sync_fetch_and_add(&TotalValueDataSize, NumVSites + Padding);
100   return 1;
101 }
102 
103 LLVM_LIBRARY_VISIBILITY void
104 __llvm_profile_instrument_target(uint64_t TargetValue, void *Data,
105                                  uint32_t CounterIndex) {
106 
107   __llvm_profile_data *PData = (__llvm_profile_data *)Data;
108   if (!PData)
109     return;
110 
111   if (!PData->Values) {
112     if (!allocateValueProfileCounters(PData))
113       return;
114   }
115 
116   ValueProfNode **ValueCounters = (ValueProfNode **)PData->Values;
117   ValueProfNode *PrevVNode = NULL;
118   ValueProfNode *CurrentVNode = ValueCounters[CounterIndex];
119 
120   uint8_t VDataCount = 0;
121   while (CurrentVNode) {
122     if (TargetValue == CurrentVNode->VData.Value) {
123       CurrentVNode->VData.Count++;
124       return;
125     }
126     PrevVNode = CurrentVNode;
127     CurrentVNode = CurrentVNode->Next;
128     ++VDataCount;
129   }
130 
131   if (VDataCount >= UCHAR_MAX)
132     return;
133 
134   CurrentVNode = (ValueProfNode *)calloc(1, sizeof(ValueProfNode));
135   if (!CurrentVNode)
136     return;
137 
138   CurrentVNode->VData.Value = TargetValue;
139   CurrentVNode->VData.Count++;
140 
141   uint32_t Success = 0;
142   if (!ValueCounters[CounterIndex])
143     Success = __sync_bool_compare_and_swap(&ValueCounters[CounterIndex], 0,
144                                            CurrentVNode);
145   else if (PrevVNode && !PrevVNode->Next)
146     Success = __sync_bool_compare_and_swap(&(PrevVNode->Next), 0, CurrentVNode);
147 
148   if (!Success) {
149     free(CurrentVNode);
150     return;
151   }
152   __sync_fetch_and_add(&TotalValueDataSize, Success * sizeof(ValueProfNode));
153 }
154 #endif
155 
156 LLVM_LIBRARY_VISIBILITY uint64_t
157 __llvm_profile_gather_value_data(uint8_t **VDataArray) {
158 
159   if (!VDataArray || 0 == TotalValueDataSize)
160     return 0;
161 
162   uint64_t NumData = TotalValueDataSize;
163   *VDataArray = (uint8_t *)calloc(NumData, sizeof(uint8_t));
164   if (!*VDataArray)
165     return 0;
166 
167   uint8_t *VDataEnd = *VDataArray + NumData;
168   uint8_t *PerSiteCountsHead = *VDataArray;
169   const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
170   const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
171   __llvm_profile_data *I;
172   for (I = (__llvm_profile_data *)DataBegin; I != DataEnd; ++I) {
173 
174     uint64_t NumVSites = 0;
175     uint32_t VKI, i;
176 
177     if (!I->Values)
178       continue;
179 
180     ValueProfNode **ValueCounters = (ValueProfNode **)I->Values;
181 
182     for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI)
183       NumVSites += I->NumValueSites[VKI];
184     uint8_t Padding = __llvm_profile_get_num_padding_bytes(NumVSites);
185 
186     uint8_t *PerSiteCountPtr = PerSiteCountsHead;
187     InstrProfValueData *VDataPtr =
188         (InstrProfValueData *)(PerSiteCountPtr + NumVSites + Padding);
189 
190     for (i = 0; i < NumVSites; ++i) {
191 
192       ValueProfNode *VNode = ValueCounters[i];
193 
194       uint8_t VDataCount = 0;
195       while (VNode && ((uint8_t *)(VDataPtr + 1) <= VDataEnd)) {
196         *VDataPtr = VNode->VData;
197         VNode = VNode->Next;
198         ++VDataPtr;
199         if (++VDataCount == UCHAR_MAX)
200           break;
201       }
202       *PerSiteCountPtr = VDataCount;
203       ++PerSiteCountPtr;
204     }
205     I->Values = (void *)PerSiteCountsHead;
206     PerSiteCountsHead = (uint8_t *)VDataPtr;
207   }
208   return PerSiteCountsHead - *VDataArray;
209 }
210