tools/llvm-profgen/PerfReader.cpp

//===-- PerfReader.cpp - perfscript reader  ---------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "PerfReader.h"
#include "ProfileGenerator.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Process.h"

#define DEBUG_TYPE "perf-reader"

static cl::opt<bool> ShowMmapEvents("show-mmap-events", cl::ReallyHidden,
                                    cl::init(false), cl::ZeroOrMore,
                                    cl::desc("Print binary load events."));

cl::opt<bool> SkipSymbolization("skip-symbolization", cl::ReallyHidden,
                                cl::init(false), cl::ZeroOrMore,
                                cl::desc("Dump the unsymbolized profile to the "
                                         "output file. It will show unwinder "
                                         "output for CS profile generation."));
cl::opt<bool> UseOffset("use-offset", cl::ReallyHidden, cl::init(true),
                        cl::ZeroOrMore,
                        cl::desc("Work with `--skip-symbolization` to dump the "
                                 "offset instead of virtual address."));
cl::opt<bool>
    IgnoreStackSamples("ignore-stack-samples", cl::ReallyHidden,
                       cl::init(false), cl::ZeroOrMore,
                       cl::desc("Ignore call stack samples for hybrid samples "
                                "and produce context-insensitive profile."));

extern cl::opt<std::string> PerfTraceFilename;
extern cl::opt<bool> ShowDisassemblyOnly;
extern cl::opt<bool> ShowSourceLocations;
extern cl::opt<std::string> OutputFilename;

namespace llvm {
namespace sampleprof {

void VirtualUnwinder::unwindCall(UnwindState &State) {
  // The 2nd frame after leaf could be missing if stack sample is
  // taken when IP is within prolog/epilog, as frame chain isn't
  // setup yet. Fill in the missing frame in that case.
  // TODO: Currently we just assume all the addr that can't match the
  // 2nd frame is in prolog/epilog. In the future, we will switch to
  // pro/epi tracker(Dwarf CFI) for the precise check.
  uint64_t Source = State.getCurrentLBRSource();
  auto *ParentFrame = State.getParentFrame();
  if (ParentFrame == State.getDummyRootPtr() ||
      ParentFrame->Address != Source) {
    State.switchToFrame(Source);
  } else {
    State.popFrame();
  }
  State.InstPtr.update(Source);
}

void VirtualUnwinder::unwindLinear(UnwindState &State, uint64_t Repeat) {
  InstructionPointer &IP = State.InstPtr;
  uint64_t Target = State.getCurrentLBRTarget();
  uint64_t End = IP.Address;
  if (Binary->usePseudoProbes()) {
    // We don't need to top frame probe since it should be extracted
    // from the range.
    // The outcome of the virtual unwinding with pseudo probes is a
    // map from a context key to the address range being unwound.
    // This means basically linear unwinding is not needed for pseudo
    // probes. The range will be simply recorded here and will be
    // converted to a list of pseudo probes to report in ProfileGenerator.
    State.getParentFrame()->recordRangeCount(Target, End, Repeat);
  } else {
    // Unwind linear execution part.
    // Split and record the range by different inline context. For example:
    // [0x01] ... main:1          # Target
    // [0x02] ... main:2
    // [0x03] ... main:3 @ foo:1
    // [0x04] ... main:3 @ foo:2
    // [0x05] ... main:3 @ foo:3
    // [0x06] ... main:4
    // [0x07] ... main:5          # End
    // It will be recorded:
    // [main:*]         : [0x06, 0x07], [0x01, 0x02]
    // [main:3 @ foo:*] : [0x03, 0x05]
    while (IP.Address > Target) {
      uint64_t PrevIP = IP.Address;
      IP.backward();
      // Break into segments for implicit call/return due to inlining
      bool SameInlinee = Binary->inlineContextEqual(PrevIP, IP.Address);
      if (!SameInlinee) {
        State.switchToFrame(PrevIP);
        State.CurrentLeafFrame->recordRangeCount(PrevIP, End, Repeat);
        End = IP.Address;
      }
    }
    assert(IP.Address == Target && "The last one must be the target address.");
    // Record the remaining range, [0x01, 0x02] in the example
    State.switchToFrame(IP.Address);
    State.CurrentLeafFrame->recordRangeCount(IP.Address, End, Repeat);
  }
}

void VirtualUnwinder::unwindReturn(UnwindState &State) {
  // Add extra frame as we unwind through the return
  const LBREntry &LBR = State.getCurrentLBR();
  uint64_t CallAddr = Binary->getCallAddrFromFrameAddr(LBR.Target);
  State.switchToFrame(CallAddr);
  State.pushFrame(LBR.Source);
  State.InstPtr.update(LBR.Source);
}

void VirtualUnwinder::unwindBranchWithinFrame(UnwindState &State) {
  // TODO: Tolerate tail call for now, as we may see tail call from libraries.
  // This is only for intra function branches, excluding tail calls.
  uint64_t Source = State.getCurrentLBRSource();
  State.switchToFrame(Source);
  State.InstPtr.update(Source);
}

std::shared_ptr<StringBasedCtxKey> FrameStack::getContextKey() {
  std::shared_ptr<StringBasedCtxKey> KeyStr =
      std::make_shared<StringBasedCtxKey>();
  KeyStr->Context = Binary->getExpandedContext(Stack, KeyStr->WasLeafInlined);
  if (KeyStr->Context.empty())
    return nullptr;
  KeyStr->genHashCode();
  return KeyStr;
}

std::shared_ptr<ProbeBasedCtxKey> ProbeStack::getContextKey() {
  std::shared_ptr<ProbeBasedCtxKey> ProbeBasedKey =
      std::make_shared<ProbeBasedCtxKey>();
  for (auto CallProbe : Stack) {
    ProbeBasedKey->Probes.emplace_back(CallProbe);
  }
  CSProfileGenerator::compressRecursionContext<const MCDecodedPseudoProbe *>(
      ProbeBasedKey->Probes);
  CSProfileGenerator::trimContext<const MCDecodedPseudoProbe *>(
      ProbeBasedKey->Probes);

  ProbeBasedKey->genHashCode();
  return ProbeBasedKey;
}

template <typename T>
void VirtualUnwinder::collectSamplesFromFrame(UnwindState::ProfiledFrame *Cur,
                                              T &Stack) {
  if (Cur->RangeSamples.empty() && Cur->BranchSamples.empty())
    return;

  std::shared_ptr<ContextKey> Key = Stack.getContextKey();
  if (Key == nullptr)
    return;
  auto Ret = CtxCounterMap->emplace(Hashable<ContextKey>(Key), SampleCounter());
  SampleCounter &SCounter = Ret.first->second;
  for (auto &Item : Cur->RangeSamples) {
    uint64_t StartOffset = Binary->virtualAddrToOffset(std::get<0>(Item));
    uint64_t EndOffset = Binary->virtualAddrToOffset(std::get<1>(Item));
    SCounter.recordRangeCount(StartOffset, EndOffset, std::get<2>(Item));
  }

  for (auto &Item : Cur->BranchSamples) {
    uint64_t SourceOffset = Binary->virtualAddrToOffset(std::get<0>(Item));
    uint64_t TargetOffset = Binary->virtualAddrToOffset(std::get<1>(Item));
    SCounter.recordBranchCount(SourceOffset, TargetOffset, std::get<2>(Item));
  }
}

template <typename T>
void VirtualUnwinder::collectSamplesFromFrameTrie(
    UnwindState::ProfiledFrame *Cur, T &Stack) {
  if (!Cur->isDummyRoot()) {
    if (!Stack.pushFrame(Cur)) {
      // Process truncated context
      // Start a new traversal ignoring its bottom context
      T EmptyStack(Binary);
      collectSamplesFromFrame(Cur, EmptyStack);
      for (const auto &Item : Cur->Children) {
        collectSamplesFromFrameTrie(Item.second.get(), EmptyStack);
      }

      // Keep note of untracked call site and deduplicate them
      // for warning later.
      if (!Cur->isLeafFrame())
        UntrackedCallsites.insert(Cur->Address);

      return;
    }
  }

  collectSamplesFromFrame(Cur, Stack);
  // Process children frame
  for (const auto &Item : Cur->Children) {
    collectSamplesFromFrameTrie(Item.second.get(), Stack);
  }
  // Recover the call stack
  Stack.popFrame();
}

void VirtualUnwinder::collectSamplesFromFrameTrie(
    UnwindState::ProfiledFrame *Cur) {
  if (Binary->usePseudoProbes()) {
    ProbeStack Stack(Binary);
    collectSamplesFromFrameTrie<ProbeStack>(Cur, Stack);
  } else {
    FrameStack Stack(Binary);
    collectSamplesFromFrameTrie<FrameStack>(Cur, Stack);
  }
}

void VirtualUnwinder::recordBranchCount(const LBREntry &Branch,
                                        UnwindState &State, uint64_t Repeat) {
  if (Branch.IsArtificial)
    return;

  if (Binary->usePseudoProbes()) {
    // Same as recordRangeCount, We don't need to top frame probe since we will
    // extract it from branch's source address
    State.getParentFrame()->recordBranchCount(Branch.Source, Branch.Target,
                                              Repeat);
  } else {
    State.CurrentLeafFrame->recordBranchCount(Branch.Source, Branch.Target,
                                              Repeat);
  }
}

bool VirtualUnwinder::unwind(const PerfSample *Sample, uint64_t Repeat) {
  // Capture initial state as starting point for unwinding.
  UnwindState State(Sample, Binary);

  // Sanity check - making sure leaf of LBR aligns with leaf of stack sample
  // Stack sample sometimes can be unreliable, so filter out bogus ones.
  if (!State.validateInitialState())
    return false;

  // Also do not attempt linear unwind for the leaf range as it's incomplete.
  bool IsLeaf = true;

  // Now process the LBR samples in parrallel with stack sample
  // Note that we do not reverse the LBR entry order so we can
  // unwind the sample stack as we walk through LBR entries.
  while (State.hasNextLBR()) {
    State.checkStateConsistency();

    // Unwind implicit calls/returns from inlining, along the linear path,
    // break into smaller sub section each with its own calling context.
    if (!IsLeaf) {
      unwindLinear(State, Repeat);
    }
    IsLeaf = false;

    // Save the LBR branch before it gets unwound.
    const LBREntry &Branch = State.getCurrentLBR();

    if (isCallState(State)) {
      // Unwind calls - we know we encountered call if LBR overlaps with
      // transition between leaf the 2nd frame. Note that for calls that
      // were not in the original stack sample, we should have added the
      // extra frame when processing the return paired with this call.
      unwindCall(State);
    } else if (isReturnState(State)) {
      // Unwind returns - check whether the IP is indeed at a return instruction
      unwindReturn(State);
    } else {
      // Unwind branches - for regular intra function branches, we only
      // need to record branch with context.
      unwindBranchWithinFrame(State);
    }
    State.advanceLBR();
    // Record `branch` with calling context after unwinding.
    recordBranchCount(Branch, State, Repeat);
  }
  // As samples are aggregated on trie, record them into counter map
  collectSamplesFromFrameTrie(State.getDummyRootPtr());

  return true;
}

std::unique_ptr<PerfReaderBase> PerfReaderBase::create(ProfiledBinary *Binary,
                                                       StringRef PerfInputFile,
                                                       bool IsPerfData) {
  // For perf data input, we need to convert them into perf script first.
  if (IsPerfData) {
    std::string ConvertedPerfScript =
        convertPerfDataToTrace(Binary, PerfInputFile);
    // Let commoand opt own the string for converted perf trace file name
    PerfTraceFilename = ConvertedPerfScript;
    PerfInputFile = PerfTraceFilename;
  }

  PerfScriptType PerfType = checkPerfScriptType(PerfInputFile);
  std::unique_ptr<PerfReaderBase> PerfReader;
  if (PerfType == PERF_LBR_STACK) {
    PerfReader.reset(new HybridPerfReader(Binary, PerfInputFile));
  } else if (PerfType == PERF_LBR) {
    PerfReader.reset(new LBRPerfReader(Binary, PerfInputFile));
  } else {
    exitWithError("Unsupported perfscript!");
  }

  return PerfReader;
}

std::string PerfReaderBase::convertPerfDataToTrace(ProfiledBinary *Binary,
                                                   StringRef PerfData) {
  // Run perf script to retrieve PIDs matching binary we're interested in.
  auto PerfExecutable = sys::Process::FindInEnvPath("PATH", "perf");
  if (!PerfExecutable) {
    exitWithError("Perf not found.");
  }
  std::string PerfPath = *PerfExecutable;
  std::string PerfTraceFile = PerfData.str() + ".script.tmp";
  StringRef ScriptMMapArgs[] = {PerfPath, "script",   "--show-mmap-events",
                                "-F",     "comm,pid", "-i",
                                PerfData};
  Optional<StringRef> Redirects[] = {llvm::None,                // Stdin
                                     StringRef(PerfTraceFile),  // Stdout
                                     StringRef(PerfTraceFile)}; // Stderr
  sys::ExecuteAndWait(PerfPath, ScriptMMapArgs, llvm::None, Redirects);

  // Collect the PIDs
  TraceStream TraceIt(PerfTraceFile);
  std::string PIDs;
  while (!TraceIt.isAtEoF()) {
    MMapEvent MMap;
    if (isMMap2Event(TraceIt.getCurrentLine()) &&
        extractMMap2EventForBinary(Binary, TraceIt.getCurrentLine(), MMap)) {
      if (!PIDs.empty()) {
        PIDs.append(",");
      }
      PIDs.append(utostr(MMap.PID));
    }
    TraceIt.advance();
  }

  // Run perf script again to retrieve events for PIDs collected above
  StringRef ScriptSampleArgs[] = {PerfPath, "script",     "--show-mmap-events",
                                  "-F",     "ip,brstack", "--pid",
                                  PIDs,     "-i",         PerfData};
  sys::ExecuteAndWait(PerfPath, ScriptSampleArgs, llvm::None, Redirects);

  return PerfTraceFile;
}

void PerfReaderBase::updateBinaryAddress(const MMapEvent &Event) {
  // Drop the event which doesn't belong to user-provided binary
  StringRef BinaryName = llvm::sys::path::filename(Event.BinaryPath);
  if (Binary->getName() != BinaryName)
    return;

  // Drop the event if its image is loaded at the same address
  if (Event.Address == Binary->getBaseAddress()) {
    Binary->setIsLoadedByMMap(true);
    return;
  }

  if (Event.Offset == Binary->getTextSegmentOffset()) {
    // A binary image could be unloaded and then reloaded at different
    // place, so update binary load address.
    // Only update for the first executable segment and assume all other
    // segments are loaded at consecutive memory addresses, which is the case on
    // X64.
    Binary->setBaseAddress(Event.Address);
    Binary->setIsLoadedByMMap(true);
  } else {
    // Verify segments are loaded consecutively.
    const auto &Offsets = Binary->getTextSegmentOffsets();
    auto It = std::lower_bound(Offsets.begin(), Offsets.end(), Event.Offset);
    if (It != Offsets.end() && *It == Event.Offset) {
      // The event is for loading a separate executable segment.
      auto I = std::distance(Offsets.begin(), It);
      const auto &PreferredAddrs = Binary->getPreferredTextSegmentAddresses();
      if (PreferredAddrs[I] - Binary->getPreferredBaseAddress() !=
          Event.Address - Binary->getBaseAddress())
        exitWithError("Executable segments not loaded consecutively");
    } else {
      if (It == Offsets.begin())
        exitWithError("File offset not found");
      else {
        // Find the segment the event falls in. A large segment could be loaded
        // via multiple mmap calls with consecutive memory addresses.
        --It;
        assert(*It < Event.Offset);
        if (Event.Offset - *It != Event.Address - Binary->getBaseAddress())
          exitWithError("Segment not loaded by consecutive mmaps");
      }
    }
  }
}

static std::string getContextKeyStr(ContextKey *K,
                                    const ProfiledBinary *Binary) {
  if (const auto *CtxKey = dyn_cast<StringBasedCtxKey>(K)) {
    return SampleContext::getContextString(CtxKey->Context);
  } else if (const auto *CtxKey = dyn_cast<ProbeBasedCtxKey>(K)) {
    SampleContextFrameVector ContextStack;
    for (const auto *Probe : CtxKey->Probes) {
      Binary->getInlineContextForProbe(Probe, ContextStack, true);
    }
    // Probe context key at this point does not have leaf probe, so do not
    // include the leaf inline location.
    return SampleContext::getContextString(ContextStack, true);
  } else {
    llvm_unreachable("unexpected key type");
  }
}

void HybridPerfReader::unwindSamples() {
  std::set<uint64_t> AllUntrackedCallsites;
  for (const auto &Item : AggregatedSamples) {
    const PerfSample *Sample = Item.first.getPtr();
    VirtualUnwinder Unwinder(&SampleCounters, Binary);
    Unwinder.unwind(Sample, Item.second);
    auto &CurrUntrackedCallsites = Unwinder.getUntrackedCallsites();
    AllUntrackedCallsites.insert(CurrUntrackedCallsites.begin(),
                                 CurrUntrackedCallsites.end());
  }

  // Warn about untracked frames due to missing probes.
  for (auto Address : AllUntrackedCallsites)
    WithColor::warning() << "Profile context truncated due to missing probe "
                         << "for call instruction at "
                         << format("0x%" PRIx64, Address) << "\n";
}

bool PerfReaderBase::extractLBRStack(TraceStream &TraceIt,
                                     SmallVectorImpl<LBREntry> &LBRStack) {
  // The raw format of LBR stack is like:
  // 0x4005c8/0x4005dc/P/-/-/0 0x40062f/0x4005b0/P/-/-/0 ...
  //                           ... 0x4005c8/0x4005dc/P/-/-/0
  // It's in FIFO order and seperated by whitespace.
  SmallVector<StringRef, 32> Records;
  TraceIt.getCurrentLine().split(Records, " ", -1, false);
  auto WarnInvalidLBR = [](TraceStream &TraceIt) {
    WithColor::warning() << "Invalid address in LBR record at line "
                         << TraceIt.getLineNumber() << ": "
                         << TraceIt.getCurrentLine() << "\n";
  };

  // Skip the leading instruction pointer.
  size_t Index = 0;
  uint64_t LeadingAddr;
  if (!Records.empty() && Records[0].find('/') == StringRef::npos) {
    if (Records[0].getAsInteger(16, LeadingAddr)) {
      WarnInvalidLBR(TraceIt);
      TraceIt.advance();
      return false;
    }
    Index = 1;
  }
  // Now extract LBR samples - note that we do not reverse the
  // LBR entry order so we can unwind the sample stack as we walk
  // through LBR entries.
  uint64_t PrevTrDst = 0;

  while (Index < Records.size()) {
    auto &Token = Records[Index++];
    if (Token.size() == 0)
      continue;

    SmallVector<StringRef, 8> Addresses;
    Token.split(Addresses, "/");
    uint64_t Src;
    uint64_t Dst;

    // Stop at broken LBR records.
    if (Addresses.size() < 2 || Addresses[0].substr(2).getAsInteger(16, Src) ||
        Addresses[1].substr(2).getAsInteger(16, Dst)) {
      WarnInvalidLBR(TraceIt);
      break;
    }

    bool SrcIsInternal = Binary->addressIsCode(Src);
    bool DstIsInternal = Binary->addressIsCode(Dst);
    bool IsExternal = !SrcIsInternal && !DstIsInternal;
    bool IsIncoming = !SrcIsInternal && DstIsInternal;
    bool IsOutgoing = SrcIsInternal && !DstIsInternal;
    bool IsArtificial = false;

    // Ignore branches outside the current binary. Ignore all remaining branches
    // if there's no incoming branch before the external branch in reverse
    // order.
    if (IsExternal) {
      if (PrevTrDst)
        continue;
      else if (!LBRStack.empty()) {
        WithColor::warning()
            << "Invalid transfer to external code in LBR record at line "
            << TraceIt.getLineNumber() << ": " << TraceIt.getCurrentLine()
            << "\n";
        break;
      }
    }

    if (IsOutgoing) {
      if (!PrevTrDst) {
        // This is unpaired outgoing jump which is likely due to interrupt or
        // incomplete LBR trace. Ignore current and subsequent entries since
        // they are likely in different contexts.
        break;
      }

      if (Binary->addressIsReturn(Src)) {
        // In a callback case, a return from internal code, say A, to external
        // runtime can happen. The external runtime can then call back to
        // another internal routine, say B. Making an artificial branch that
        // looks like a return from A to B can confuse the unwinder to treat
        // the instruction before B as the call instruction.
        break;
      }

      // For transition to external code, group the Source with the next
      // availabe transition target.
      Dst = PrevTrDst;
      PrevTrDst = 0;
      IsArtificial = true;
    } else {
      if (PrevTrDst) {
        // If we have seen an incoming transition from external code to internal
        // code, but not a following outgoing transition, the incoming
        // transition is likely due to interrupt which is usually unpaired.
        // Ignore current and subsequent entries since they are likely in
        // different contexts.
        break;
      }

      if (IsIncoming) {
        // For transition from external code (such as dynamic libraries) to
        // the current binary, keep track of the branch target which will be
        // grouped with the Source of the last transition from the current
        // binary.
        PrevTrDst = Dst;
        continue;
      }
    }

    // TODO: filter out buggy duplicate branches on Skylake

    LBRStack.emplace_back(LBREntry(Src, Dst, IsArtificial));
  }
  TraceIt.advance();
  return !LBRStack.empty();
}

bool PerfReaderBase::extractCallstack(TraceStream &TraceIt,
                                      SmallVectorImpl<uint64_t> &CallStack) {
  // The raw format of call stack is like:
  //            4005dc      # leaf frame
  //	          400634
  //	          400684      # root frame
  // It's in bottom-up order with each frame in one line.

  // Extract stack frames from sample
  while (!TraceIt.isAtEoF() && !TraceIt.getCurrentLine().startswith(" 0x")) {
    StringRef FrameStr = TraceIt.getCurrentLine().ltrim();
    uint64_t FrameAddr = 0;
    if (FrameStr.getAsInteger(16, FrameAddr)) {
      // We might parse a non-perf sample line like empty line and comments,
      // skip it
      TraceIt.advance();
      return false;
    }
    TraceIt.advance();
    // Currently intermixed frame from different binaries is not supported.
    // Ignore caller frames not from binary of interest.
    if (!Binary->addressIsCode(FrameAddr))
      break;

    // We need to translate return address to call address for non-leaf frames.
    if (!CallStack.empty()) {
      auto CallAddr = Binary->getCallAddrFromFrameAddr(FrameAddr);
      if (!CallAddr) {
        // Stop at an invalid return address caused by bad unwinding. This could
        // happen to frame-pointer-based unwinding and the callee functions that
        // do not have the frame pointer chain set up.
        InvalidReturnAddresses.insert(FrameAddr);
        break;
      }
      FrameAddr = CallAddr;
    }

    CallStack.emplace_back(FrameAddr);
  }

  // Skip other unrelated line, find the next valid LBR line
  // Note that even for empty call stack, we should skip the address at the
  // bottom, otherwise the following pass may generate a truncated callstack
  while (!TraceIt.isAtEoF() && !TraceIt.getCurrentLine().startswith(" 0x")) {
    TraceIt.advance();
  }
  // Filter out broken stack sample. We may not have complete frame info
  // if sample end up in prolog/epilog, the result is dangling context not
  // connected to entry point. This should be relatively rare thus not much
  // impact on overall profile quality. However we do want to filter them
  // out to reduce the number of different calling contexts. One instance
  // of such case - when sample landed in prolog/epilog, somehow stack
  // walking will be broken in an unexpected way that higher frames will be
  // missing.
  return !CallStack.empty() &&
         !Binary->addressInPrologEpilog(CallStack.front());
}

void PerfReaderBase::warnIfMissingMMap() {
  if (!Binary->getMissingMMapWarned() && !Binary->getIsLoadedByMMap()) {
    WithColor::warning() << "No relevant mmap event is matched for "
                         << Binary->getName()
                         << ", will use preferred address ("
                         << format("0x%" PRIx64,
                                   Binary->getPreferredBaseAddress())
                         << ") as the base loading address!\n";
    // Avoid redundant warning, only warn at the first unmatched sample.
    Binary->setMissingMMapWarned(true);
  }
}

void HybridPerfReader::parseSample(TraceStream &TraceIt, uint64_t Count) {
  // The raw hybird sample started with call stack in FILO order and followed
  // intermediately by LBR sample
  // e.g.
  // 	          4005dc    # call stack leaf
  //	          400634
  //	          400684    # call stack root
  // 0x4005c8/0x4005dc/P/-/-/0   0x40062f/0x4005b0/P/-/-/0 ...
  //          ... 0x4005c8/0x4005dc/P/-/-/0    # LBR Entries
  //
  std::shared_ptr<PerfSample> Sample = std::make_shared<PerfSample>();

  // Parsing call stack and populate into PerfSample.CallStack
  if (!extractCallstack(TraceIt, Sample->CallStack)) {
    // Skip the next LBR line matched current call stack
    if (!TraceIt.isAtEoF() && TraceIt.getCurrentLine().startswith(" 0x"))
      TraceIt.advance();
    return;
  }

  warnIfMissingMMap();

  if (!TraceIt.isAtEoF() && TraceIt.getCurrentLine().startswith(" 0x")) {
    // Parsing LBR stack and populate into PerfSample.LBRStack
    if (extractLBRStack(TraceIt, Sample->LBRStack)) {
      // Canonicalize stack leaf to avoid 'random' IP from leaf frame skew LBR
      // ranges
      Sample->CallStack.front() = Sample->LBRStack[0].Target;
      // Record samples by aggregation
      AggregatedSamples[Hashable<PerfSample>(Sample)] += Count;
    }
  } else {
    // LBR sample is encoded in single line after stack sample
    exitWithError("'Hybrid perf sample is corrupted, No LBR sample line");
  }
}

void PerfReaderBase::writeRawProfile(StringRef Filename) {
  std::error_code EC;
  raw_fd_ostream OS(Filename, EC, llvm::sys::fs::OF_TextWithCRLF);
  if (EC)
    exitWithError(EC, Filename);
  writeRawProfile(OS);
}

// Use ordered map to make the output deterministic
using OrderedCounterForPrint = std::map<std::string, SampleCounter *>;

void PerfReaderBase::writeRawProfile(raw_fd_ostream &OS) {
  /*
     Format:
     [context string]
     number of entries in RangeCounter
     from_1-to_1:count_1
     from_2-to_2:count_2
     ......
     from_n-to_n:count_n
     number of entries in BranchCounter
     src_1->dst_1:count_1
     src_2->dst_2:count_2
     ......
     src_n->dst_n:count_n
  */

  OrderedCounterForPrint OrderedCounters;
  for (auto &CI : SampleCounters) {
    OrderedCounters[getContextKeyStr(CI.first.getPtr(), Binary)] = &CI.second;
  }

  auto SCounterPrinter = [&](RangeSample Counter, StringRef Separator,
                             uint32_t Indent) {
    OS.indent(Indent);
    OS << Counter.size() << "\n";
    for (auto I : Counter) {
      uint64_t Start = UseOffset ? I.first.first
                                 : Binary->offsetToVirtualAddr(I.first.first);
      uint64_t End = UseOffset ? I.first.second
                               : Binary->offsetToVirtualAddr(I.first.second);
      OS.indent(Indent);
      OS << Twine::utohexstr(Start) << Separator << Twine::utohexstr(End) << ":"
         << I.second << "\n";
    }
  };

  for (auto &CI : OrderedCounters) {
    uint32_t Indent = 0;
    if (!CI.first.empty()) {
      // Context string key
      OS << "[" << CI.first << "]\n";
      Indent = 2;
    }

    SampleCounter &Counter = *CI.second;
    SCounterPrinter(Counter.RangeCounter, "-", Indent);
    SCounterPrinter(Counter.BranchCounter, "->", Indent);
  }
}

void LBRPerfReader::computeCounterFromLBR(const PerfSample *Sample,
                                          uint64_t Repeat) {
  SampleCounter &Counter = SampleCounters.begin()->second;
  uint64_t EndOffeset = 0;
  for (const LBREntry &LBR : Sample->LBRStack) {
    uint64_t SourceOffset = Binary->virtualAddrToOffset(LBR.Source);
    uint64_t TargetOffset = Binary->virtualAddrToOffset(LBR.Target);

    if (!LBR.IsArtificial) {
      Counter.recordBranchCount(SourceOffset, TargetOffset, Repeat);
    }

    // If this not the first LBR, update the range count between TO of current
    // LBR and FROM of next LBR.
    uint64_t StartOffset = TargetOffset;
    if (EndOffeset != 0)
      Counter.recordRangeCount(StartOffset, EndOffeset, Repeat);
    EndOffeset = SourceOffset;
  }
}

void LBRPerfReader::parseSample(TraceStream &TraceIt, uint64_t Count) {
  std::shared_ptr<PerfSample> Sample = std::make_shared<PerfSample>();
  // Parsing LBR stack and populate into PerfSample.LBRStack
  if (extractLBRStack(TraceIt, Sample->LBRStack)) {
    warnIfMissingMMap();
    // Record LBR only samples by aggregation
    AggregatedSamples[Hashable<PerfSample>(Sample)] += Count;
  }
}

void LBRPerfReader::generateRawProfile() {
  // There is no context for LBR only sample, so initialize one entry with
  // fake "empty" context key.
  assert(SampleCounters.empty() &&
         "Sample counter map should be empty before raw profile generation");
  std::shared_ptr<StringBasedCtxKey> Key =
      std::make_shared<StringBasedCtxKey>();
  Key->genHashCode();
  SampleCounters.emplace(Hashable<ContextKey>(Key), SampleCounter());
  for (const auto &Item : AggregatedSamples) {
    const PerfSample *Sample = Item.first.getPtr();
    computeCounterFromLBR(Sample, Item.second);
  }
}

uint64_t PerfReaderBase::parseAggregatedCount(TraceStream &TraceIt) {
  // The aggregated count is optional, so do not skip the line and return 1 if
  // it's unmatched
  uint64_t Count = 1;
  if (!TraceIt.getCurrentLine().getAsInteger(10, Count))
    TraceIt.advance();
  return Count;
}

void PerfReaderBase::parseSample(TraceStream &TraceIt) {
  uint64_t Count = parseAggregatedCount(TraceIt);
  assert(Count >= 1 && "Aggregated count should be >= 1!");
  parseSample(TraceIt, Count);
}

bool PerfReaderBase::extractMMap2EventForBinary(ProfiledBinary *Binary,
                                                StringRef Line,
                                                MMapEvent &MMap) {
  // Parse a line like:
  //  PERF_RECORD_MMAP2 2113428/2113428: [0x7fd4efb57000(0x204000) @ 0
  //  08:04 19532229 3585508847]: r-xp /usr/lib64/libdl-2.17.so
  constexpr static const char *const Pattern =
      "PERF_RECORD_MMAP2 ([0-9]+)/[0-9]+: "
      "\\[(0x[a-f0-9]+)\\((0x[a-f0-9]+)\\) @ "
      "(0x[a-f0-9]+|0) .*\\]: [-a-z]+ (.*)";
  // Field 0 - whole line
  // Field 1 - PID
  // Field 2 - base address
  // Field 3 - mmapped size
  // Field 4 - page offset
  // Field 5 - binary path
  enum EventIndex {
    WHOLE_LINE = 0,
    PID = 1,
    MMAPPED_ADDRESS = 2,
    MMAPPED_SIZE = 3,
    PAGE_OFFSET = 4,
    BINARY_PATH = 5
  };

  Regex RegMmap2(Pattern);
  SmallVector<StringRef, 6> Fields;
  bool R = RegMmap2.match(Line, &Fields);
  if (!R) {
    std::string ErrorMsg = "Cannot parse mmap event: " + Line.str() + " \n";
    exitWithError(ErrorMsg);
  }
  Fields[PID].getAsInteger(10, MMap.PID);
  Fields[MMAPPED_ADDRESS].getAsInteger(0, MMap.Address);
  Fields[MMAPPED_SIZE].getAsInteger(0, MMap.Size);
  Fields[PAGE_OFFSET].getAsInteger(0, MMap.Offset);
  MMap.BinaryPath = Fields[BINARY_PATH];
  if (ShowMmapEvents) {
    outs() << "Mmap: Binary " << MMap.BinaryPath << " loaded at "
           << format("0x%" PRIx64 ":", MMap.Address) << " \n";
  }

  StringRef BinaryName = llvm::sys::path::filename(MMap.BinaryPath);
  return Binary->getName() == BinaryName;
}

void PerfReaderBase::parseMMap2Event(TraceStream &TraceIt) {
  MMapEvent MMap;
  if (extractMMap2EventForBinary(Binary, TraceIt.getCurrentLine(), MMap))
    updateBinaryAddress(MMap);
  TraceIt.advance();
}

void PerfReaderBase::parseEventOrSample(TraceStream &TraceIt) {
  if (isMMap2Event(TraceIt.getCurrentLine()))
    parseMMap2Event(TraceIt);
  else
    parseSample(TraceIt);
}

void PerfReaderBase::parseAndAggregateTrace() {
  // Trace line iterator
  TraceStream TraceIt(PerfTraceFile);
  while (!TraceIt.isAtEoF())
    parseEventOrSample(TraceIt);
}

// A LBR sample is like:
// 40062f 0x5c6313f/0x5c63170/P/-/-/0  0x5c630e7/0x5c63130/P/-/-/0 ...
// A heuristic for fast detection by checking whether a
// leading "  0x" and the '/' exist.
bool PerfReaderBase::isLBRSample(StringRef Line) {
  // Skip the leading instruction pointer
  SmallVector<StringRef, 32> Records;
  Line.trim().split(Records, " ", 2, false);
  if (Records.size() < 2)
    return false;
  if (Records[1].startswith("0x") && Records[1].find('/') != StringRef::npos)
    return true;
  return false;
}

bool PerfReaderBase::isMMap2Event(StringRef Line) {
  // Short cut to avoid string find is possible.
  if (Line.empty() || Line.size() < 50)
    return false;

  if (std::isdigit(Line[0]))
    return false;

  // PERF_RECORD_MMAP2 does not appear at the beginning of the line
  // for ` perf script  --show-mmap-events  -i ...`
  return Line.find("PERF_RECORD_MMAP2") != StringRef::npos;
}

// The raw hybird sample is like
// e.g.
// 	          4005dc    # call stack leaf
//	          400634
//	          400684    # call stack root
// 0x4005c8/0x4005dc/P/-/-/0   0x40062f/0x4005b0/P/-/-/0 ...
//          ... 0x4005c8/0x4005dc/P/-/-/0    # LBR Entries
// Determine the perfscript contains hybrid samples(call stack + LBRs) by
// checking whether there is a non-empty call stack immediately followed by
// a LBR sample
PerfScriptType PerfReaderBase::checkPerfScriptType(StringRef FileName) {
  TraceStream TraceIt(FileName);
  uint64_t FrameAddr = 0;
  while (!TraceIt.isAtEoF()) {
    // Skip the aggregated count
    if (!TraceIt.getCurrentLine().getAsInteger(10, FrameAddr))
      TraceIt.advance();

    // Detect sample with call stack
    int32_t Count = 0;
    while (!TraceIt.isAtEoF() &&
           !TraceIt.getCurrentLine().ltrim().getAsInteger(16, FrameAddr)) {
      Count++;
      TraceIt.advance();
    }
    if (!TraceIt.isAtEoF()) {
      if (isLBRSample(TraceIt.getCurrentLine())) {
        if (Count > 0)
          return PERF_LBR_STACK;
        else
          return PERF_LBR;
      }
      TraceIt.advance();
    }
  }

  exitWithError("Invalid perf script input!");
  return PERF_INVALID;
}

void HybridPerfReader::generateRawProfile() {
  ProfileIsCS = !IgnoreStackSamples;
  if (ProfileIsCS)
    unwindSamples();
  else
    LBRPerfReader::generateRawProfile();
}

void PerfReaderBase::warnTruncatedStack() {
  for (auto Address : InvalidReturnAddresses) {
    WithColor::warning()
        << "Truncated stack sample due to invalid return address at "
        << format("0x%" PRIx64, Address)
        << ", likely caused by frame pointer omission\n";
  }
}

void PerfReaderBase::parsePerfTraces() {
  // Parse perf traces and do aggregation.
  parseAndAggregateTrace();

  // Generate unsymbolized profile.
  warnTruncatedStack();
  generateRawProfile();

  if (SkipSymbolization)
    writeRawProfile(OutputFilename);
}

} // end namespace sampleprof
} // end namespace llvm