1/*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\ 2|* 3|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4|* See https://llvm.org/LICENSE.txt for license information. 5|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6|* 7\*===----------------------------------------------------------------------===*/ 8/* 9 * This is the master file that defines all the data structure, signature, 10 * constant literals that are shared across profiling runtime library, 11 * compiler (instrumentation), and host tools (reader/writer). The entities 12 * defined in this file affect the profile runtime ABI, the raw profile format, 13 * or both. 14 * 15 * The file has two identical copies. The master copy lives in LLVM and 16 * the other one sits in compiler-rt/lib/profile directory. To make changes 17 * in this file, first modify the master copy and copy it over to compiler-rt. 18 * Testing of any change in this file can start only after the two copies are 19 * synced up. 20 * 21 * The first part of the file includes macros that defines types, names, and 22 * initializers for the member fields of the core data structures. The field 23 * declarations for one structure is enabled by defining the field activation 24 * macro associated with that structure. Only one field activation record 25 * can be defined at one time and the rest definitions will be filtered out by 26 * the preprocessor. 27 * 28 * Examples of how the template is used to instantiate structure definition: 29 * 1. To declare a structure: 30 * 31 * struct ProfData { 32 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ 33 * Type Name; 34 * #include "llvm/ProfileData/InstrProfData.inc" 35 * }; 36 * 37 * 2. To construct LLVM type arrays for the struct type: 38 * 39 * Type *DataTypes[] = { 40 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ 41 * LLVMType, 42 * #include "llvm/ProfileData/InstrProfData.inc" 43 * }; 44 * 45 * 4. To construct constant array for the initializers: 46 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ 47 * Initializer, 48 * Constant *ConstantVals[] = { 49 * #include "llvm/ProfileData/InstrProfData.inc" 50 * }; 51 * 52 * 53 * The second part of the file includes definitions all other entities that 54 * are related to runtime ABI and format. When no field activation macro is 55 * defined, this file can be included to introduce the definitions. 56 * 57\*===----------------------------------------------------------------------===*/ 58 59/* Functions marked with INSTR_PROF_VISIBILITY must have hidden visibility in 60 * the compiler runtime. */ 61#ifndef INSTR_PROF_VISIBILITY 62#define INSTR_PROF_VISIBILITY 63#endif 64 65/* INSTR_PROF_DATA start. */ 66/* Definition of member fields of the per-function control structure. */ 67#ifndef INSTR_PROF_DATA 68#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) 69#else 70#define INSTR_PROF_DATA_DEFINED 71#endif 72INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \ 73 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ 74 IndexedInstrProf::ComputeHash(getPGOFuncNameVarInitializer(Inc->getName())))) 75INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ 76 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ 77 Inc->getHash()->getZExtValue())) 78INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt64PtrTy(Ctx), CounterPtr, \ 79 ConstantExpr::getBitCast(CounterPtr, \ 80 llvm::Type::getInt64PtrTy(Ctx))) 81/* This is used to map function pointers for the indirect call targets to 82 * function name hashes during the conversion from raw to merged profile 83 * data. 84 */ 85INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), FunctionPointer, \ 86 FunctionAddr) 87INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \ 88 ValuesPtrExpr) 89INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \ 90 ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters)) 91INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \ 92 ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) 93#undef INSTR_PROF_DATA 94/* INSTR_PROF_DATA end. */ 95 96 97/* This is an internal data structure used by value profiler. It 98 * is defined here to allow serialization code sharing by LLVM 99 * to be used in unit test. 100 * 101 * typedef struct ValueProfNode { 102 * // InstrProfValueData VData; 103 * uint64_t Value; 104 * uint64_t Count; 105 * struct ValueProfNode *Next; 106 * } ValueProfNode; 107 */ 108/* INSTR_PROF_VALUE_NODE start. */ 109#ifndef INSTR_PROF_VALUE_NODE 110#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer) 111#else 112#define INSTR_PROF_DATA_DEFINED 113#endif 114INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Value, \ 115 ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0)) 116INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Count, \ 117 ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0)) 118INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::Type::getInt8PtrTy(Ctx), Next, \ 119 ConstantInt::get(llvm::Type::GetInt8PtrTy(Ctx), 0)) 120#undef INSTR_PROF_VALUE_NODE 121/* INSTR_PROF_VALUE_NODE end. */ 122 123/* INSTR_PROF_RAW_HEADER start */ 124/* Definition of member fields of the raw profile header data structure. */ 125#ifndef INSTR_PROF_RAW_HEADER 126#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer) 127#else 128#define INSTR_PROF_DATA_DEFINED 129#endif 130INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic()) 131INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version()) 132INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) 133INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters) 134INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) 135INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters) 136INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) 137INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) 138INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) 139INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) 140#undef INSTR_PROF_RAW_HEADER 141/* INSTR_PROF_RAW_HEADER end */ 142 143/* VALUE_PROF_FUNC_PARAM start */ 144/* Definition of parameter types of the runtime API used to do value profiling 145 * for a given value site. 146 */ 147#ifndef VALUE_PROF_FUNC_PARAM 148#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType) 149#define INSTR_PROF_COMMA 150#else 151#define INSTR_PROF_DATA_DEFINED 152#define INSTR_PROF_COMMA , 153#endif 154VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \ 155 INSTR_PROF_COMMA 156VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA 157#ifndef VALUE_RANGE_PROF 158VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) 159#else /* VALUE_RANGE_PROF */ 160/* FIXME: This is to be removed after switching to the new memop value 161 * profiling. */ 162VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) \ 163 INSTR_PROF_COMMA 164VALUE_PROF_FUNC_PARAM(uint64_t, PreciseRangeStart, Type::getInt64Ty(Ctx)) \ 165 INSTR_PROF_COMMA 166VALUE_PROF_FUNC_PARAM(uint64_t, PreciseRangeLast, Type::getInt64Ty(Ctx)) \ 167 INSTR_PROF_COMMA 168VALUE_PROF_FUNC_PARAM(uint64_t, LargeValue, Type::getInt64Ty(Ctx)) 169#endif /*VALUE_RANGE_PROF */ 170#undef VALUE_PROF_FUNC_PARAM 171#undef INSTR_PROF_COMMA 172/* VALUE_PROF_FUNC_PARAM end */ 173 174/* VALUE_PROF_KIND start */ 175#ifndef VALUE_PROF_KIND 176#define VALUE_PROF_KIND(Enumerator, Value, Descr) 177#else 178#define INSTR_PROF_DATA_DEFINED 179#endif 180/* For indirect function call value profiling, the addresses of the target 181 * functions are profiled by the instrumented code. The target addresses are 182 * written in the raw profile data and converted to target function name's MD5 183 * hash by the profile reader during deserialization. Typically, this happens 184 * when the raw profile data is read during profile merging. 185 * 186 * For this remapping the ProfData is used. ProfData contains both the function 187 * name hash and the function address. 188 */ 189VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target") 190/* For memory intrinsic functions size profiling. */ 191VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size") 192/* These two kinds must be the last to be 193 * declared. This is to make sure the string 194 * array created with the template can be 195 * indexed with the kind value. 196 */ 197VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first") 198VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last") 199 200#undef VALUE_PROF_KIND 201/* VALUE_PROF_KIND end */ 202 203#undef COVMAP_V2_OR_V3 204#ifdef COVMAP_V2 205#define COVMAP_V2_OR_V3 206#endif 207#ifdef COVMAP_V3 208#define COVMAP_V2_OR_V3 209#endif 210 211/* COVMAP_FUNC_RECORD start */ 212/* Definition of member fields of the function record structure in coverage 213 * map. 214 */ 215#ifndef COVMAP_FUNC_RECORD 216#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer) 217#else 218#define INSTR_PROF_DATA_DEFINED 219#endif 220#ifdef COVMAP_V1 221COVMAP_FUNC_RECORD(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), \ 222 NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \ 223 llvm::Type::getInt8PtrTy(Ctx))) 224COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \ 225 llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \ 226 NameValue.size())) 227#endif 228#ifdef COVMAP_V2_OR_V3 229COVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \ 230 llvm::ConstantInt::get( \ 231 llvm::Type::getInt64Ty(Ctx), NameHash)) 232#endif 233COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \ 234 llvm::ConstantInt::get( \ 235 llvm::Type::getInt32Ty(Ctx), CoverageMapping.size())) 236COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ 237 llvm::ConstantInt::get( \ 238 llvm::Type::getInt64Ty(Ctx), FuncHash)) 239#ifdef COVMAP_V3 240COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FilenamesRef, \ 241 llvm::ConstantInt::get( \ 242 llvm::Type::getInt64Ty(Ctx), FilenamesRef)) 243COVMAP_FUNC_RECORD(const char, \ 244 llvm::ArrayType::get(llvm::Type::getInt8Ty(Ctx), \ 245 CoverageMapping.size()), \ 246 CoverageMapping, 247 llvm::ConstantDataArray::getRaw( \ 248 CoverageMapping, CoverageMapping.size(), \ 249 llvm::Type::getInt8Ty(Ctx))) 250#endif 251#undef COVMAP_FUNC_RECORD 252/* COVMAP_FUNC_RECORD end. */ 253 254/* COVMAP_HEADER start */ 255/* Definition of member fields of coverage map header. 256 */ 257#ifndef COVMAP_HEADER 258#define COVMAP_HEADER(Type, LLVMType, Name, Initializer) 259#else 260#define INSTR_PROF_DATA_DEFINED 261#endif 262COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \ 263 llvm::ConstantInt::get(Int32Ty, NRecords)) 264COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \ 265 llvm::ConstantInt::get(Int32Ty, FilenamesSize)) 266COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \ 267 llvm::ConstantInt::get(Int32Ty, CoverageMappingSize)) 268COVMAP_HEADER(uint32_t, Int32Ty, Version, \ 269 llvm::ConstantInt::get(Int32Ty, CovMapVersion::CurrentVersion)) 270#undef COVMAP_HEADER 271/* COVMAP_HEADER end. */ 272 273 274#ifdef INSTR_PROF_SECT_ENTRY 275#define INSTR_PROF_DATA_DEFINED 276INSTR_PROF_SECT_ENTRY(IPSK_data, \ 277 INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \ 278 INSTR_PROF_DATA_COFF, "__DATA,") 279INSTR_PROF_SECT_ENTRY(IPSK_cnts, \ 280 INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \ 281 INSTR_PROF_CNTS_COFF, "__DATA,") 282INSTR_PROF_SECT_ENTRY(IPSK_name, \ 283 INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \ 284 INSTR_PROF_NAME_COFF, "__DATA,") 285INSTR_PROF_SECT_ENTRY(IPSK_vals, \ 286 INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \ 287 INSTR_PROF_VALS_COFF, "__DATA,") 288INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \ 289 INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \ 290 INSTR_PROF_VNODES_COFF, "__DATA,") 291INSTR_PROF_SECT_ENTRY(IPSK_covmap, \ 292 INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \ 293 INSTR_PROF_COVMAP_COFF, "__LLVM_COV,") 294INSTR_PROF_SECT_ENTRY(IPSK_covfun, \ 295 INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON), \ 296 INSTR_PROF_COVFUN_COFF, "__LLVM_COV,") 297INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \ 298 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \ 299 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,") 300 301#undef INSTR_PROF_SECT_ENTRY 302#endif 303 304 305#ifdef INSTR_PROF_VALUE_PROF_DATA 306#define INSTR_PROF_DATA_DEFINED 307 308#define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255 309/*! 310 * This is the header of the data structure that defines the on-disk 311 * layout of the value profile data of a particular kind for one function. 312 */ 313typedef struct ValueProfRecord { 314 /* The kind of the value profile record. */ 315 uint32_t Kind; 316 /* 317 * The number of value profile sites. It is guaranteed to be non-zero; 318 * otherwise the record for this kind won't be emitted. 319 */ 320 uint32_t NumValueSites; 321 /* 322 * The first element of the array that stores the number of profiled 323 * values for each value site. The size of the array is NumValueSites. 324 * Since NumValueSites is greater than zero, there is at least one 325 * element in the array. 326 */ 327 uint8_t SiteCountArray[1]; 328 329 /* 330 * The fake declaration is for documentation purpose only. 331 * Align the start of next field to be on 8 byte boundaries. 332 uint8_t Padding[X]; 333 */ 334 335 /* The array of value profile data. The size of the array is the sum 336 * of all elements in SiteCountArray[]. 337 InstrProfValueData ValueData[]; 338 */ 339 340#ifdef __cplusplus 341 /*! 342 * Return the number of value sites. 343 */ 344 uint32_t getNumValueSites() const { return NumValueSites; } 345 /*! 346 * Read data from this record and save it to Record. 347 */ 348 void deserializeTo(InstrProfRecord &Record, 349 InstrProfSymtab *SymTab); 350 /* 351 * In-place byte swap: 352 * Do byte swap for this instance. \c Old is the original order before 353 * the swap, and \c New is the New byte order. 354 */ 355 void swapBytes(support::endianness Old, support::endianness New); 356#endif 357} ValueProfRecord; 358 359/*! 360 * Per-function header/control data structure for value profiling 361 * data in indexed format. 362 */ 363typedef struct ValueProfData { 364 /* 365 * Total size in bytes including this field. It must be a multiple 366 * of sizeof(uint64_t). 367 */ 368 uint32_t TotalSize; 369 /* 370 *The number of value profile kinds that has value profile data. 371 * In this implementation, a value profile kind is considered to 372 * have profile data if the number of value profile sites for the 373 * kind is not zero. More aggressively, the implementation can 374 * choose to check the actual data value: if none of the value sites 375 * has any profiled values, the kind can be skipped. 376 */ 377 uint32_t NumValueKinds; 378 379 /* 380 * Following are a sequence of variable length records. The prefix/header 381 * of each record is defined by ValueProfRecord type. The number of 382 * records is NumValueKinds. 383 * ValueProfRecord Record_1; 384 * ValueProfRecord Record_N; 385 */ 386 387#if __cplusplus 388 /*! 389 * Return the total size in bytes of the on-disk value profile data 390 * given the data stored in Record. 391 */ 392 static uint32_t getSize(const InstrProfRecord &Record); 393 /*! 394 * Return a pointer to \c ValueProfData instance ready to be streamed. 395 */ 396 static std::unique_ptr<ValueProfData> 397 serializeFrom(const InstrProfRecord &Record); 398 /*! 399 * Check the integrity of the record. 400 */ 401 Error checkIntegrity(); 402 /*! 403 * Return a pointer to \c ValueProfileData instance ready to be read. 404 * All data in the instance are properly byte swapped. The input 405 * data is assumed to be in little endian order. 406 */ 407 static Expected<std::unique_ptr<ValueProfData>> 408 getValueProfData(const unsigned char *SrcBuffer, 409 const unsigned char *const SrcBufferEnd, 410 support::endianness SrcDataEndianness); 411 /*! 412 * Swap byte order from \c Endianness order to host byte order. 413 */ 414 void swapBytesToHost(support::endianness Endianness); 415 /*! 416 * Swap byte order from host byte order to \c Endianness order. 417 */ 418 void swapBytesFromHost(support::endianness Endianness); 419 /*! 420 * Return the total size of \c ValueProfileData. 421 */ 422 uint32_t getSize() const { return TotalSize; } 423 /*! 424 * Read data from this data and save it to \c Record. 425 */ 426 void deserializeTo(InstrProfRecord &Record, 427 InstrProfSymtab *SymTab); 428 void operator delete(void *ptr) { ::operator delete(ptr); } 429#endif 430} ValueProfData; 431 432/* 433 * The closure is designed to abstact away two types of value profile data: 434 * - InstrProfRecord which is the primary data structure used to 435 * represent profile data in host tools (reader, writer, and profile-use) 436 * - value profile runtime data structure suitable to be used by C 437 * runtime library. 438 * 439 * Both sources of data need to serialize to disk/memory-buffer in common 440 * format: ValueProfData. The abstraction allows compiler-rt's raw profiler 441 * writer to share the same format and code with indexed profile writer. 442 * 443 * For documentation of the member methods below, refer to corresponding methods 444 * in class InstrProfRecord. 445 */ 446typedef struct ValueProfRecordClosure { 447 const void *Record; 448 uint32_t (*GetNumValueKinds)(const void *Record); 449 uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind); 450 uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind); 451 uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S); 452 453 /* 454 * After extracting the value profile data from the value profile record, 455 * this method is used to map the in-memory value to on-disk value. If 456 * the method is null, value will be written out untranslated. 457 */ 458 uint64_t (*RemapValueData)(uint32_t, uint64_t Value); 459 void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K, 460 uint32_t S); 461 ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes); 462} ValueProfRecordClosure; 463 464INSTR_PROF_VISIBILITY ValueProfRecord * 465getFirstValueProfRecord(ValueProfData *VPD); 466INSTR_PROF_VISIBILITY ValueProfRecord * 467getValueProfRecordNext(ValueProfRecord *VPR); 468INSTR_PROF_VISIBILITY InstrProfValueData * 469getValueProfRecordValueData(ValueProfRecord *VPR); 470INSTR_PROF_VISIBILITY uint32_t 471getValueProfRecordHeaderSize(uint32_t NumValueSites); 472 473#undef INSTR_PROF_VALUE_PROF_DATA 474#endif /* INSTR_PROF_VALUE_PROF_DATA */ 475 476 477#ifdef INSTR_PROF_COMMON_API_IMPL 478#define INSTR_PROF_DATA_DEFINED 479#ifdef __cplusplus 480#define INSTR_PROF_INLINE inline 481#define INSTR_PROF_NULLPTR nullptr 482#else 483#define INSTR_PROF_INLINE 484#define INSTR_PROF_NULLPTR NULL 485#endif 486 487#ifndef offsetof 488#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) 489#endif 490 491/*! 492 * Return the \c ValueProfRecord header size including the 493 * padding bytes. 494 */ 495INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 496uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) { 497 uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) + 498 sizeof(uint8_t) * NumValueSites; 499 /* Round the size to multiple of 8 bytes. */ 500 Size = (Size + 7) & ~7; 501 return Size; 502} 503 504/*! 505 * Return the total size of the value profile record including the 506 * header and the value data. 507 */ 508INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 509uint32_t getValueProfRecordSize(uint32_t NumValueSites, 510 uint32_t NumValueData) { 511 return getValueProfRecordHeaderSize(NumValueSites) + 512 sizeof(InstrProfValueData) * NumValueData; 513} 514 515/*! 516 * Return the pointer to the start of value data array. 517 */ 518INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 519InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) { 520 return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize( 521 This->NumValueSites)); 522} 523 524/*! 525 * Return the total number of value data for \c This record. 526 */ 527INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 528uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) { 529 uint32_t NumValueData = 0; 530 uint32_t I; 531 for (I = 0; I < This->NumValueSites; I++) 532 NumValueData += This->SiteCountArray[I]; 533 return NumValueData; 534} 535 536/*! 537 * Use this method to advance to the next \c This \c ValueProfRecord. 538 */ 539INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 540ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) { 541 uint32_t NumValueData = getValueProfRecordNumValueData(This); 542 return (ValueProfRecord *)((char *)This + 543 getValueProfRecordSize(This->NumValueSites, 544 NumValueData)); 545} 546 547/*! 548 * Return the first \c ValueProfRecord instance. 549 */ 550INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 551ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) { 552 return (ValueProfRecord *)((char *)This + sizeof(ValueProfData)); 553} 554 555/* Closure based interfaces. */ 556 557/*! 558 * Return the total size in bytes of the on-disk value profile data 559 * given the data stored in Record. 560 */ 561INSTR_PROF_VISIBILITY uint32_t 562getValueProfDataSize(ValueProfRecordClosure *Closure) { 563 uint32_t Kind; 564 uint32_t TotalSize = sizeof(ValueProfData); 565 const void *Record = Closure->Record; 566 567 for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { 568 uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind); 569 if (!NumValueSites) 570 continue; 571 TotalSize += getValueProfRecordSize(NumValueSites, 572 Closure->GetNumValueData(Record, Kind)); 573 } 574 return TotalSize; 575} 576 577/*! 578 * Extract value profile data of a function for the profile kind \c ValueKind 579 * from the \c Closure and serialize the data into \c This record instance. 580 */ 581INSTR_PROF_VISIBILITY void 582serializeValueProfRecordFrom(ValueProfRecord *This, 583 ValueProfRecordClosure *Closure, 584 uint32_t ValueKind, uint32_t NumValueSites) { 585 uint32_t S; 586 const void *Record = Closure->Record; 587 This->Kind = ValueKind; 588 This->NumValueSites = NumValueSites; 589 InstrProfValueData *DstVD = getValueProfRecordValueData(This); 590 591 for (S = 0; S < NumValueSites; S++) { 592 uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S); 593 This->SiteCountArray[S] = ND; 594 Closure->GetValueForSite(Record, DstVD, ValueKind, S); 595 DstVD += ND; 596 } 597} 598 599/*! 600 * Extract value profile data of a function from the \c Closure 601 * and serialize the data into \c DstData if it is not NULL or heap 602 * memory allocated by the \c Closure's allocator method. If \c 603 * DstData is not null, the caller is expected to set the TotalSize 604 * in DstData. 605 */ 606INSTR_PROF_VISIBILITY ValueProfData * 607serializeValueProfDataFrom(ValueProfRecordClosure *Closure, 608 ValueProfData *DstData) { 609 uint32_t Kind; 610 uint32_t TotalSize = 611 DstData ? DstData->TotalSize : getValueProfDataSize(Closure); 612 613 ValueProfData *VPD = 614 DstData ? DstData : Closure->AllocValueProfData(TotalSize); 615 616 VPD->TotalSize = TotalSize; 617 VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record); 618 ValueProfRecord *VR = getFirstValueProfRecord(VPD); 619 for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { 620 uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind); 621 if (!NumValueSites) 622 continue; 623 serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites); 624 VR = getValueProfRecordNext(VR); 625 } 626 return VPD; 627} 628 629#undef INSTR_PROF_COMMON_API_IMPL 630#endif /* INSTR_PROF_COMMON_API_IMPL */ 631 632/*============================================================================*/ 633 634#ifndef INSTR_PROF_DATA_DEFINED 635 636#ifndef INSTR_PROF_DATA_INC 637#define INSTR_PROF_DATA_INC 638 639/* Helper macros. */ 640#define INSTR_PROF_SIMPLE_QUOTE(x) #x 641#define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x) 642#define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y 643#define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y) 644 645/* Magic number to detect file format and endianness. 646 * Use 255 at one end, since no UTF-8 file can use that character. Avoid 0, 647 * so that utilities, like strings, don't grab it as a string. 129 is also 648 * invalid UTF-8, and high enough to be interesting. 649 * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR" 650 * for 32-bit platforms. 651 */ 652#define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ 653 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ 654 (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129 655#define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ 656 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ 657 (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129 658 659/* Raw profile format version (start from 1). */ 660#define INSTR_PROF_RAW_VERSION 5 661/* Indexed profile format version (start from 1). */ 662#define INSTR_PROF_INDEX_VERSION 6 663/* Coverage mapping format version (start from 0). */ 664#define INSTR_PROF_COVMAP_VERSION 3 665 666/* Profile version is always of type uint64_t. Reserve the upper 8 bits in the 667 * version for other variants of profile. We set the lowest bit of the upper 8 668 * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton 669 * generated profile, and 0 if this is a Clang FE generated profile. 670 * 1 in bit 57 indicates there are context-sensitive records in the profile. 671 */ 672#define VARIANT_MASKS_ALL 0xff00000000000000ULL 673#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) 674#define VARIANT_MASK_IR_PROF (0x1ULL << 56) 675#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57) 676#define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version 677#define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime 678 679/* The variable that holds the name of the profile data 680 * specified via command line. */ 681#define INSTR_PROF_PROFILE_NAME_VAR __llvm_profile_filename 682 683/* section name strings common to all targets other 684 than WIN32 */ 685#define INSTR_PROF_DATA_COMMON __llvm_prf_data 686#define INSTR_PROF_NAME_COMMON __llvm_prf_names 687#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts 688#define INSTR_PROF_VALS_COMMON __llvm_prf_vals 689#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds 690#define INSTR_PROF_COVMAP_COMMON __llvm_covmap 691#define INSTR_PROF_COVFUN_COMMON __llvm_covfun 692#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile 693/* Windows section names. Because these section names contain dollar characters, 694 * they must be quoted. 695 */ 696#define INSTR_PROF_DATA_COFF ".lprfd$M" 697#define INSTR_PROF_NAME_COFF ".lprfn$M" 698#define INSTR_PROF_CNTS_COFF ".lprfc$M" 699#define INSTR_PROF_VALS_COFF ".lprfv$M" 700#define INSTR_PROF_VNODES_COFF ".lprfnd$M" 701#define INSTR_PROF_COVMAP_COFF ".lcovmap$M" 702#define INSTR_PROF_COVFUN_COFF ".lcovfun$M" 703#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M" 704 705#ifdef _WIN32 706/* Runtime section names and name strings. */ 707#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF 708#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF 709#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF 710/* Array of pointers. Each pointer points to a list 711 * of value nodes associated with one value site. 712 */ 713#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF 714/* Value profile nodes section. */ 715#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF 716#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF 717#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF 718#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF 719#else 720/* Runtime section names and name strings. */ 721#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON) 722#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON) 723#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON) 724/* Array of pointers. Each pointer points to a list 725 * of value nodes associated with one value site. 726 */ 727#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON) 728/* Value profile nodes section. */ 729#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON) 730#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON) 731#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON) 732/* Order file instrumentation. */ 733#define INSTR_PROF_ORDERFILE_SECT_NAME \ 734 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON) 735#endif 736 737#define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer 738#define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR \ 739 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME) 740#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx 741#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \ 742 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME) 743 744/* Macros to define start/stop section symbol for a given 745 * section on Linux. For instance 746 * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will 747 * expand to __start___llvm_prof_data 748 */ 749#define INSTR_PROF_SECT_START(Sect) \ 750 INSTR_PROF_CONCAT(__start_,Sect) 751#define INSTR_PROF_SECT_STOP(Sect) \ 752 INSTR_PROF_CONCAT(__stop_,Sect) 753 754/* Value Profiling API linkage name. */ 755#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target 756#define INSTR_PROF_VALUE_PROF_FUNC_STR \ 757 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC) 758/* FIXME: This is to be removed after switching to the new memop value 759 * profiling. */ 760#define INSTR_PROF_VALUE_RANGE_PROF_FUNC __llvm_profile_instrument_range 761#define INSTR_PROF_VALUE_RANGE_PROF_FUNC_STR \ 762 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_RANGE_PROF_FUNC) 763#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop 764#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \ 765 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC) 766 767/* InstrProfile per-function control data alignment. */ 768#define INSTR_PROF_DATA_ALIGNMENT 8 769 770/* The data structure that represents a tracked value by the 771 * value profiler. 772 */ 773typedef struct InstrProfValueData { 774 /* Profiled value. */ 775 uint64_t Value; 776 /* Number of times the value appears in the training run. */ 777 uint64_t Count; 778} InstrProfValueData; 779 780#endif /* INSTR_PROF_DATA_INC */ 781 782#ifndef INSTR_ORDER_FILE_INC 783/* The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB). */ 784#define INSTR_ORDER_FILE_BUFFER_SIZE 131072 785#define INSTR_ORDER_FILE_BUFFER_BITS 17 786#define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff 787#endif /* INSTR_ORDER_FILE_INC */ 788#else 789#undef INSTR_PROF_DATA_DEFINED 790#endif 791 792#undef COVMAP_V2_OR_V3 793 794#ifdef INSTR_PROF_VALUE_PROF_MEMOP_API 795 796#ifdef __cplusplus 797#define INSTR_PROF_INLINE inline 798#else 799#define INSTR_PROF_INLINE 800#endif 801 802/* The value range buckets (22 buckets) for the memop size value profiling looks 803 * like: 804 * 805 * [0, 0] 806 * [1, 1] 807 * [2, 2] 808 * [3, 3] 809 * [4, 4] 810 * [5, 5] 811 * [6, 6] 812 * [7, 7] 813 * [8, 8] 814 * [9, 15] 815 * [16, 16] 816 * [17, 31] 817 * [32, 32] 818 * [33, 63] 819 * [64, 64] 820 * [65, 127] 821 * [128, 128] 822 * [129, 255] 823 * [256, 256] 824 * [257, 511] 825 * [512, 512] 826 * [513, UINT64_MAX] 827 * 828 * Each range has a 'representative value' which is the lower end value of the 829 * range and used to store in the runtime profile data records and the VP 830 * metadata. For example, it's 2 for [2, 2] and 64 for [65, 127]. 831 */ 832 833/* 834 * Clz and Popcount. This code was copied from 835 * compiler-rt/lib/fuzzer/{FuzzerBuiltins.h,FuzzerBuiltinsMsvc.h} and 836 * llvm/include/llvm/Support/MathExtras.h. 837 */ 838#if defined(_MSC_VER) && !defined(__clang__) 839 840#include <intrin.h> 841INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 842int InstProfClzll(unsigned long long X) { 843 unsigned long LeadZeroIdx = 0; 844#if !defined(_M_ARM64) && !defined(_M_X64) 845 // Scan the high 32 bits. 846 if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X >> 32))) 847 return (int)(63 - (LeadZeroIdx + 32)); // Create a bit offset 848 // from the MSB. 849 // Scan the low 32 bits. 850 if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X))) 851 return (int)(63 - LeadZeroIdx); 852#else 853 if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx; 854#endif 855 return 64; 856} 857INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 858int InstProfPopcountll(unsigned long long X) { 859 // This code originates from https://reviews.llvm.org/rG30626254510f. 860 unsigned long long v = X; 861 v = v - ((v >> 1) & 0x5555555555555555ULL); 862 v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL); 863 v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL; 864 return (int)((unsigned long long)(v * 0x0101010101010101ULL) >> 56); 865} 866 867#else 868 869INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 870int InstProfClzll(unsigned long long X) { return __builtin_clzll(X); } 871INSTR_PROF_VISIBILITY INSTR_PROF_INLINE 872int InstProfPopcountll(unsigned long long X) { return __builtin_popcountll(X); } 873 874#endif /* defined(_MSC_VER) && !defined(__clang__) */ 875 876/* Map an (observed) memop size value to the representative value of its range. 877 * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */ 878INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t 879InstrProfGetRangeRepValue(uint64_t Value) { 880 if (Value <= 8) 881 // The first ranges are individually tracked. Use the value as is. 882 return Value; 883 else if (Value >= 513) 884 // The last range is mapped to its lowest value. 885 return 513; 886 else if (InstProfPopcountll(Value) == 1) 887 // If it's a power of two, use it as is. 888 return Value; 889 else 890 // Otherwise, take to the previous power of two + 1. 891 return (1 << (64 - InstProfClzll(Value) - 1)) + 1; 892} 893 894/* Return true if the range that an (observed) memop size value belongs to has 895 * only a single value in the range. For example, 0 -> true, 8 -> true, 10 -> 896 * false, 64 -> true, 100 -> false, 513 -> false. */ 897INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned 898InstrProfIsSingleValRange(uint64_t Value) { 899 if (Value <= 8) 900 // The first ranges are individually tracked. 901 return 1; 902 else if (InstProfPopcountll(Value) == 1) 903 // If it's a power of two, there's only one value. 904 return 1; 905 else 906 // Otherwise, there's more than one value in the range. 907 return 0; 908} 909 910#endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */ 911