Process/Linux/IntelPTCollector.cpp

//===-- IntelPTCollector.cpp ------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include "IntelPTCollector.h"

#include "Perf.h"

#include "Plugins/Process/POSIX/ProcessPOSIXLog.h"
#include "lldb/Host/linux/Support.h"
#include "lldb/Utility/StreamString.h"

#include "llvm/ADT/StringRef.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/MathExtras.h"

#include <algorithm>
#include <cstddef>
#include <fstream>
#include <linux/perf_event.h>
#include <sstream>
#include <sys/ioctl.h>
#include <sys/syscall.h>

using namespace lldb;
using namespace lldb_private;
using namespace process_linux;
using namespace llvm;

const char *kOSEventIntelPTTypeFile =
    "/sys/bus/event_source/devices/intel_pt/type";

const char *kPSBPeriodCapFile =
    "/sys/bus/event_source/devices/intel_pt/caps/psb_cyc";

const char *kPSBPeriodValidValuesFile =
    "/sys/bus/event_source/devices/intel_pt/caps/psb_periods";

const char *kTSCBitOffsetFile =
    "/sys/bus/event_source/devices/intel_pt/format/tsc";

const char *kPSBPeriodBitOffsetFile =
    "/sys/bus/event_source/devices/intel_pt/format/psb_period";

enum IntelPTConfigFileType {
  Hex = 0,
  // 0 or 1
  ZeroOne,
  Decimal,
  // a bit index file always starts with the prefix config: following by an int,
  // which represents the offset of the perf_event_attr.config value where to
  // store a given configuration.
  BitOffset
};

/// Get the content of /proc/cpuinfo that can be later used to decode traces.
static Expected<ArrayRef<uint8_t>> GetCPUInfo() {
  static llvm::Optional<std::vector<uint8_t>> cpu_info;
  if (!cpu_info) {
    auto buffer_or_error = errorOrToExpected(getProcFile("cpuinfo"));
    if (!buffer_or_error)
      return buffer_or_error.takeError();
    MemoryBuffer &buffer = **buffer_or_error;
    cpu_info = std::vector<uint8_t>(
        reinterpret_cast<const uint8_t *>(buffer.getBufferStart()),
        reinterpret_cast<const uint8_t *>(buffer.getBufferEnd()));
  }
  return *cpu_info;
}

static Expected<uint32_t> ReadIntelPTConfigFile(const char *file,
                                                IntelPTConfigFileType type) {
  ErrorOr<std::unique_ptr<MemoryBuffer>> stream =
      MemoryBuffer::getFileAsStream(file);

  if (!stream)
    return createStringError(inconvertibleErrorCode(),
                             "Can't open the file '%s'", file);

  uint32_t value = 0;
  StringRef text_buffer = stream.get()->getBuffer();

  if (type == BitOffset) {
    const char *prefix = "config:";
    if (!text_buffer.startswith(prefix))
      return createStringError(inconvertibleErrorCode(),
                               "The file '%s' contents doesn't start with '%s'",
                               file, prefix);
    text_buffer = text_buffer.substr(strlen(prefix));
  }

  auto getRadix = [&]() {
    switch (type) {
    case Hex:
      return 16;
    case ZeroOne:
    case Decimal:
    case BitOffset:
      return 10;
    }
  };

  auto createError = [&](const char *expected_value_message) {
    return createStringError(
        inconvertibleErrorCode(),
        "The file '%s' has an invalid value. It should be %s.", file,
        expected_value_message);
  };

  if (text_buffer.trim().consumeInteger(getRadix(), value) ||
      (type == ZeroOne && value != 0 && value != 1)) {
    switch (type) {
    case Hex:
      return createError("an unsigned hexadecimal int");
    case ZeroOne:
      return createError("0 or 1");
    case Decimal:
    case BitOffset:
      return createError("an unsigned decimal int");
    }
  }
  return value;
}

/// Return the Linux perf event type for Intel PT.
static Expected<uint32_t> GetOSEventType() {
  return ReadIntelPTConfigFile(kOSEventIntelPTTypeFile,
                               IntelPTConfigFileType::Decimal);
}

static Error CheckPsbPeriod(size_t psb_period) {
  Expected<uint32_t> cap =
      ReadIntelPTConfigFile(kPSBPeriodCapFile, IntelPTConfigFileType::ZeroOne);
  if (!cap)
    return cap.takeError();
  if (*cap == 0)
    return createStringError(inconvertibleErrorCode(),
                             "psb_period is unsupported in the system.");

  Expected<uint32_t> valid_values = ReadIntelPTConfigFile(
      kPSBPeriodValidValuesFile, IntelPTConfigFileType::Hex);
  if (!valid_values)
    return valid_values.takeError();

  if (valid_values.get() & (1 << psb_period))
    return Error::success();

  std::ostringstream error;
  // 0 is always a valid value
  error << "Invalid psb_period. Valid values are: 0";
  uint32_t mask = valid_values.get();
  while (mask) {
    int index = __builtin_ctz(mask);
    if (index > 0)
      error << ", " << index;
    // clear the lowest bit
    mask &= mask - 1;
  }
  error << ".";
  return createStringError(inconvertibleErrorCode(), error.str().c_str());
}

size_t IntelPTThreadTrace::GetTraceBufferSize() const {
#ifndef PERF_ATTR_SIZE_VER5
  llvm_unreachable("Intel PT Linux perf event not supported");
#else
  return m_perf_event.GetAuxBuffer().size();
#endif
}

static Expected<uint64_t>
GeneratePerfEventConfigValue(bool enable_tsc, Optional<size_t> psb_period) {
  uint64_t config = 0;
  // tsc is always supported
  if (enable_tsc) {
    if (Expected<uint32_t> offset = ReadIntelPTConfigFile(
            kTSCBitOffsetFile, IntelPTConfigFileType::BitOffset))
      config |= 1 << *offset;
    else
      return offset.takeError();
  }
  if (psb_period) {
    if (Error error = CheckPsbPeriod(*psb_period))
      return std::move(error);

    if (Expected<uint32_t> offset = ReadIntelPTConfigFile(
            kPSBPeriodBitOffsetFile, IntelPTConfigFileType::BitOffset))
      config |= *psb_period << *offset;
    else
      return offset.takeError();
  }
  return config;
}

llvm::Expected<perf_event_attr>
IntelPTThreadTrace::CreateIntelPTPerfEventConfiguration(
    bool enable_tsc, Optional<size_t> psb_period) {
  perf_event_attr attr;
  memset(&attr, 0, sizeof(attr));
  attr.size = sizeof(attr);
  attr.exclude_kernel = 1;
  attr.sample_type = PERF_SAMPLE_TIME;
  attr.sample_id_all = 1;
  attr.exclude_hv = 1;
  attr.exclude_idle = 1;
  attr.mmap = 1;

  if (Expected<uint64_t> config_value =
          GeneratePerfEventConfigValue(enable_tsc, psb_period)) {
    attr.config = *config_value;
  } else {
    return config_value.takeError();
  }

  if (Expected<uint32_t> intel_pt_type = GetOSEventType()) {
    attr.type = *intel_pt_type;
  } else {
    return intel_pt_type.takeError();
  }

  return attr;
}

llvm::Expected<IntelPTThreadTraceUP>
IntelPTThreadTrace::Create(lldb::pid_t pid, lldb::tid_t tid, size_t buffer_size,
                           bool enable_tsc, Optional<size_t> psb_period) {
#ifndef PERF_ATTR_SIZE_VER5
  llvm_unreachable("Intel PT Linux perf event not supported");
#else
  Log *log = GetLog(POSIXLog::Ptrace);

  LLDB_LOG(log, "called thread id {0}", tid);

  if (__builtin_popcount(buffer_size) != 1 || buffer_size < 4096) {
    return createStringError(
        inconvertibleErrorCode(),
        "The trace buffer size must be a power of 2 greater than or equal to "
        "4096 (2^12) bytes. It was %" PRIu64 ".",
        buffer_size);
  }
  uint64_t page_size = getpagesize();
  uint64_t buffer_numpages = static_cast<uint64_t>(
      llvm::PowerOf2Floor((buffer_size + page_size - 1) / page_size));

  Expected<perf_event_attr> attr =
      IntelPTThreadTrace::CreateIntelPTPerfEventConfiguration(enable_tsc,
                                                              psb_period);
  if (!attr)
    return attr.takeError();

  LLDB_LOG(log, "buffer size {0} ", buffer_size);

  if (Expected<PerfEvent> perf_event = PerfEvent::Init(*attr, tid)) {
    if (Error mmap_err = perf_event->MmapMetadataAndBuffers(buffer_numpages,
                                                            buffer_numpages)) {
      return std::move(mmap_err);
    }
    return IntelPTThreadTraceUP(
        new IntelPTThreadTrace(std::move(*perf_event), tid));
  } else {
    return perf_event.takeError();
  }
#endif
}

Expected<std::vector<uint8_t>>
IntelPTThreadTrace::GetIntelPTBuffer(size_t offset, size_t size) const {
  std::vector<uint8_t> data(size, 0);
  MutableArrayRef<uint8_t> buffer_ref(data);
  Status error = ReadPerfTraceAux(buffer_ref, 0);
  if (error.Fail())
    return error.ToError();
  return data;
}

Status
IntelPTThreadTrace::ReadPerfTraceAux(llvm::MutableArrayRef<uint8_t> &buffer,
                                     size_t offset) const {
#ifndef PERF_ATTR_SIZE_VER5
  llvm_unreachable("perf event not supported");
#else
  auto fd = m_perf_event.GetFd();
  perf_event_mmap_page &mmap_metadata = m_perf_event.GetMetadataPage();
  // Disable the perf event to force a flush out of the CPU's internal buffer.
  // Besides, we can guarantee that the CPU won't override any data as we are
  // reading the buffer.
  //
  // The Intel documentation says:
  //
  // Packets are first buffered internally and then written out asynchronously.
  // To collect packet output for postprocessing, a collector needs first to
  // ensure that all packet data has been flushed from internal buffers.
  // Software can ensure this by stopping packet generation by clearing
  // IA32_RTIT_CTL.TraceEn (see “Disabling Packet Generation” in
  // Section 35.2.7.2).
  //
  // This is achieved by the PERF_EVENT_IOC_DISABLE ioctl request, as mentioned
  // in the man page of perf_event_open.
  ioctl(fd, PERF_EVENT_IOC_DISABLE);

  Log *log = GetLog(POSIXLog::Ptrace);
  Status error;
  uint64_t head = mmap_metadata.aux_head;

  LLDB_LOG(log, "Aux size -{0} , Head - {1}", mmap_metadata.aux_size, head);

  /**
   * When configured as ring buffer, the aux buffer keeps wrapping around
   * the buffer and its not possible to detect how many times the buffer
   * wrapped. Initially the buffer is filled with zeros,as shown below
   * so in order to get complete buffer we first copy firstpartsize, followed
   * by any left over part from beginning to aux_head
   *
   * aux_offset [d,d,d,d,d,d,d,d,0,0,0,0,0,0,0,0,0,0,0] aux_size
   *                 aux_head->||<- firstpartsize  ->|
   *
   * */

  ReadCyclicBuffer(buffer, m_perf_event.GetAuxBuffer(),
                   static_cast<size_t>(head), offset);
  LLDB_LOG(log, "ReadCyclic Buffer Done");

  // Reenable tracing now we have read the buffer
  ioctl(fd, PERF_EVENT_IOC_ENABLE);
  return error;
#endif
}

Status
IntelPTThreadTrace::ReadPerfTraceData(llvm::MutableArrayRef<uint8_t> &buffer,
                                      size_t offset) const {
#ifndef PERF_ATTR_SIZE_VER5
  llvm_unreachable("perf event not supported");
#else
  Log *log = GetLog(POSIXLog::Ptrace);
  uint64_t bytes_remaining = buffer.size();
  Status error;

  perf_event_mmap_page &mmap_metadata = m_perf_event.GetMetadataPage();
  uint64_t head = mmap_metadata.data_head;

  /*
   * The data buffer and aux buffer have different implementations
   * with respect to their definition of head pointer. In the case
   * of Aux data buffer the head always wraps around the aux buffer
   * and we don't need to care about it, whereas the data_head keeps
   * increasing and needs to be wrapped by modulus operator
   */

  LLDB_LOG(log, "bytes_remaining - {0}", bytes_remaining);

  auto data_buffer = m_perf_event.GetDataBuffer();

  if (head > data_buffer.size()) {
    head = head % data_buffer.size();
    LLDB_LOG(log, "Data size -{0} Head - {1}", mmap_metadata.data_size, head);

    ReadCyclicBuffer(buffer, data_buffer, static_cast<size_t>(head), offset);
    bytes_remaining -= buffer.size();
  } else {
    LLDB_LOG(log, "Head - {0}", head);
    if (offset >= head) {
      LLDB_LOG(log, "Invalid Offset ");
      error.SetErrorString("invalid offset");
      buffer = buffer.slice(buffer.size());
      return error;
    }

    auto data = data_buffer.slice(offset, (head - offset));
    auto remaining = std::copy(data.begin(), data.end(), buffer.begin());
    bytes_remaining -= (remaining - buffer.begin());
  }
  buffer = buffer.drop_back(bytes_remaining);
  return error;
#endif
}

void IntelPTThreadTrace::ReadCyclicBuffer(llvm::MutableArrayRef<uint8_t> &dst,
                                          llvm::ArrayRef<uint8_t> src,
                                          size_t src_cyc_index, size_t offset) {

  Log *log = GetLog(POSIXLog::Ptrace);

  if (dst.empty() || src.empty()) {
    dst = dst.drop_back(dst.size());
    return;
  }

  if (dst.data() == nullptr || src.data() == nullptr) {
    dst = dst.drop_back(dst.size());
    return;
  }

  if (src_cyc_index > src.size()) {
    dst = dst.drop_back(dst.size());
    return;
  }

  if (offset >= src.size()) {
    LLDB_LOG(log, "Too Big offset ");
    dst = dst.drop_back(dst.size());
    return;
  }

  llvm::SmallVector<ArrayRef<uint8_t>, 2> parts = {
      src.slice(src_cyc_index), src.take_front(src_cyc_index)};

  if (offset > parts[0].size()) {
    parts[1] = parts[1].slice(offset - parts[0].size());
    parts[0] = parts[0].drop_back(parts[0].size());
  } else if (offset == parts[0].size()) {
    parts[0] = parts[0].drop_back(parts[0].size());
  } else {
    parts[0] = parts[0].slice(offset);
  }
  auto next = dst.begin();
  auto bytes_left = dst.size();
  for (auto part : parts) {
    size_t chunk_size = std::min(part.size(), bytes_left);
    next = std::copy_n(part.begin(), chunk_size, next);
    bytes_left -= chunk_size;
  }
  dst = dst.drop_back(bytes_left);
}

TraceThreadState IntelPTThreadTrace::GetState() const {
  return {static_cast<int64_t>(m_tid),
          {TraceBinaryData{"threadTraceBuffer",
                           static_cast<int64_t>(GetTraceBufferSize())}}};
}

/// IntelPTThreadTraceCollection

bool IntelPTThreadTraceCollection::TracesThread(lldb::tid_t tid) const {
  return m_thread_traces.count(tid);
}

Error IntelPTThreadTraceCollection::TraceStop(lldb::tid_t tid) {
  auto it = m_thread_traces.find(tid);
  if (it == m_thread_traces.end())
    return createStringError(inconvertibleErrorCode(),
                             "Thread %" PRIu64 " not currently traced", tid);
  m_total_buffer_size -= it->second->GetTraceBufferSize();
  m_thread_traces.erase(tid);
  return Error::success();
}

Error IntelPTThreadTraceCollection::TraceStart(
    lldb::tid_t tid, const TraceIntelPTStartRequest &request) {
  if (TracesThread(tid))
    return createStringError(inconvertibleErrorCode(),
                             "Thread %" PRIu64 " already traced", tid);

  Expected<IntelPTThreadTraceUP> trace_up = IntelPTThreadTrace::Create(
      m_pid, tid, request.threadBufferSize, request.enableTsc,
      request.psbPeriod.map([](int64_t period) { return (size_t)period; }));
  if (!trace_up)
    return trace_up.takeError();

  m_total_buffer_size += (*trace_up)->GetTraceBufferSize();
  m_thread_traces.try_emplace(tid, std::move(*trace_up));
  return Error::success();
}

size_t IntelPTThreadTraceCollection::GetTotalBufferSize() const {
  return m_total_buffer_size;
}

std::vector<TraceThreadState>
IntelPTThreadTraceCollection::GetThreadStates() const {
  std::vector<TraceThreadState> states;
  for (const auto &it : m_thread_traces)
    states.push_back(it.second->GetState());
  return states;
}

Expected<const IntelPTThreadTrace &>
IntelPTThreadTraceCollection::GetTracedThread(lldb::tid_t tid) const {
  auto it = m_thread_traces.find(tid);
  if (it == m_thread_traces.end())
    return createStringError(inconvertibleErrorCode(),
                             "Thread %" PRIu64 " not currently traced", tid);
  return *it->second.get();
}

void IntelPTThreadTraceCollection::Clear() {
  m_thread_traces.clear();
  m_total_buffer_size = 0;
}

/// IntelPTProcessTrace

bool IntelPTProcessTrace::TracesThread(lldb::tid_t tid) const {
  return m_thread_traces.TracesThread(tid);
}

Error IntelPTProcessTrace::TraceStop(lldb::tid_t tid) {
  return m_thread_traces.TraceStop(tid);
}

Error IntelPTProcessTrace::TraceStart(lldb::tid_t tid) {
  if (m_thread_traces.GetTotalBufferSize() + m_tracing_params.threadBufferSize >
      static_cast<size_t>(*m_tracing_params.processBufferSizeLimit))
    return createStringError(
        inconvertibleErrorCode(),
        "Thread %" PRIu64 " can't be traced as the process trace size limit "
        "has been reached. Consider retracing with a higher "
        "limit.",
        tid);

  return m_thread_traces.TraceStart(tid, m_tracing_params);
}

const IntelPTThreadTraceCollection &
IntelPTProcessTrace::GetThreadTraces() const {
  return m_thread_traces;
}

/// IntelPTCollector

IntelPTCollector::IntelPTCollector(lldb::pid_t pid)
    : m_pid(pid), m_thread_traces(pid) {
  if (Expected<LinuxPerfZeroTscConversion> tsc_conversion =
          LoadPerfTscConversionParameters())
    m_tsc_conversion =
        std::make_unique<LinuxPerfZeroTscConversion>(*tsc_conversion);
  else
    LLDB_LOG_ERROR(GetLog(POSIXLog::Trace), tsc_conversion.takeError(),
                   "unable to load TSC to wall time conversion: {0}");
}

Error IntelPTCollector::TraceStop(lldb::tid_t tid) {
  if (IsProcessTracingEnabled() && m_process_trace->TracesThread(tid))
    return m_process_trace->TraceStop(tid);
  return m_thread_traces.TraceStop(tid);
}

Error IntelPTCollector::TraceStop(const TraceStopRequest &request) {
  if (request.IsProcessTracing()) {
    Clear();
    return Error::success();
  } else {
    Error error = Error::success();
    for (int64_t tid : *request.tids)
      error = joinErrors(std::move(error),
                         TraceStop(static_cast<lldb::tid_t>(tid)));
    return error;
  }
}

Error IntelPTCollector::TraceStart(
    const TraceIntelPTStartRequest &request,
    const std::vector<lldb::tid_t> &process_threads) {
  if (request.IsProcessTracing()) {
    if (IsProcessTracingEnabled()) {
      return createStringError(
          inconvertibleErrorCode(),
          "Process currently traced. Stop process tracing first");
    }
    m_process_trace = IntelPTProcessTrace(m_pid, request);

    Error error = Error::success();
    for (lldb::tid_t tid : process_threads)
      error = joinErrors(std::move(error), m_process_trace->TraceStart(tid));
    return error;
  } else {
    Error error = Error::success();
    for (int64_t tid : *request.tids)
      error = joinErrors(std::move(error),
                         m_thread_traces.TraceStart(tid, request));
    return error;
  }
}

Error IntelPTCollector::OnThreadCreated(lldb::tid_t tid) {
  if (!IsProcessTracingEnabled())
    return Error::success();
  return m_process_trace->TraceStart(tid);
}

Error IntelPTCollector::OnThreadDestroyed(lldb::tid_t tid) {
  if (IsProcessTracingEnabled() && m_process_trace->TracesThread(tid))
    return m_process_trace->TraceStop(tid);
  else if (m_thread_traces.TracesThread(tid))
    return m_thread_traces.TraceStop(tid);
  return Error::success();
}

Expected<json::Value> IntelPTCollector::GetState() const {
  Expected<ArrayRef<uint8_t>> cpu_info = GetCPUInfo();
  if (!cpu_info)
    return cpu_info.takeError();

  TraceGetStateResponse state;
  state.processBinaryData.push_back(
      {"cpuInfo", static_cast<int64_t>(cpu_info->size())});

  std::vector<TraceThreadState> thread_states =
      m_thread_traces.GetThreadStates();
  state.tracedThreads.insert(state.tracedThreads.end(), thread_states.begin(),
                             thread_states.end());

  if (IsProcessTracingEnabled()) {
    thread_states = m_process_trace->GetThreadTraces().GetThreadStates();
    state.tracedThreads.insert(state.tracedThreads.end(), thread_states.begin(),
                               thread_states.end());
  }
  return toJSON(state);
}

Expected<const IntelPTThreadTrace &>
IntelPTCollector::GetTracedThread(lldb::tid_t tid) const {
  if (IsProcessTracingEnabled() && m_process_trace->TracesThread(tid))
    return m_process_trace->GetThreadTraces().GetTracedThread(tid);
  return m_thread_traces.GetTracedThread(tid);
}

Expected<std::vector<uint8_t>>
IntelPTCollector::GetBinaryData(const TraceGetBinaryDataRequest &request) const {
  if (request.kind == "threadTraceBuffer") {
    if (Expected<const IntelPTThreadTrace &> trace =
            GetTracedThread(*request.tid))
      return trace->GetIntelPTBuffer(request.offset, request.size);
    else
      return trace.takeError();
  } else if (request.kind == "cpuInfo") {
    return GetCPUInfo();
  }
  return createStringError(inconvertibleErrorCode(),
                           "Unsuported trace binary data kind: %s",
                           request.kind.c_str());
}

void IntelPTCollector::ClearProcessTracing() { m_process_trace = None; }

bool IntelPTCollector::IsSupported() {
  Expected<uint32_t> intel_pt_type = GetOSEventType();
  if (!intel_pt_type) {
    llvm::consumeError(intel_pt_type.takeError());
    return false;
  }
  return true;
}

bool IntelPTCollector::IsProcessTracingEnabled() const {
  return (bool)m_process_trace;
}

void IntelPTCollector::Clear() {
  ClearProcessTracing();
  m_thread_traces.Clear();
}