1 //===- PDBFileBuilder.cpp - PDB File Creation -------------------*- C++ -*-===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "llvm/DebugInfo/PDB/Native/PDBFileBuilder.h"
11 
12 #include "llvm/ADT/BitVector.h"
13 
14 #include "llvm/DebugInfo/MSF/MSFBuilder.h"
15 #include "llvm/DebugInfo/PDB/GenericError.h"
16 #include "llvm/DebugInfo/PDB/Native/DbiStream.h"
17 #include "llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h"
18 #include "llvm/DebugInfo/PDB/Native/GSIStreamBuilder.h"
19 #include "llvm/DebugInfo/PDB/Native/InfoStream.h"
20 #include "llvm/DebugInfo/PDB/Native/InfoStreamBuilder.h"
21 #include "llvm/DebugInfo/PDB/Native/PDBStringTableBuilder.h"
22 #include "llvm/DebugInfo/PDB/Native/RawError.h"
23 #include "llvm/DebugInfo/PDB/Native/TpiStream.h"
24 #include "llvm/DebugInfo/PDB/Native/TpiStreamBuilder.h"
25 #include "llvm/Support/BinaryStream.h"
26 #include "llvm/Support/BinaryStreamWriter.h"
27 #include "llvm/Support/JamCRC.h"
28 #include "llvm/Support/Path.h"
29 
30 using namespace llvm;
31 using namespace llvm::codeview;
32 using namespace llvm::msf;
33 using namespace llvm::pdb;
34 using namespace llvm::support;
35 
36 PDBFileBuilder::PDBFileBuilder(BumpPtrAllocator &Allocator)
37     : Allocator(Allocator), InjectedSourceHashTraits(Strings),
38       InjectedSourceTable(2, InjectedSourceHashTraits) {}
39 
40 PDBFileBuilder::~PDBFileBuilder() {}
41 
42 Error PDBFileBuilder::initialize(uint32_t BlockSize) {
43   auto ExpectedMsf = MSFBuilder::create(Allocator, BlockSize);
44   if (!ExpectedMsf)
45     return ExpectedMsf.takeError();
46   Msf = llvm::make_unique<MSFBuilder>(std::move(*ExpectedMsf));
47   return Error::success();
48 }
49 
50 MSFBuilder &PDBFileBuilder::getMsfBuilder() { return *Msf; }
51 
52 InfoStreamBuilder &PDBFileBuilder::getInfoBuilder() {
53   if (!Info)
54     Info = llvm::make_unique<InfoStreamBuilder>(*Msf, NamedStreams);
55   return *Info;
56 }
57 
58 DbiStreamBuilder &PDBFileBuilder::getDbiBuilder() {
59   if (!Dbi)
60     Dbi = llvm::make_unique<DbiStreamBuilder>(*Msf);
61   return *Dbi;
62 }
63 
64 TpiStreamBuilder &PDBFileBuilder::getTpiBuilder() {
65   if (!Tpi)
66     Tpi = llvm::make_unique<TpiStreamBuilder>(*Msf, StreamTPI);
67   return *Tpi;
68 }
69 
70 TpiStreamBuilder &PDBFileBuilder::getIpiBuilder() {
71   if (!Ipi)
72     Ipi = llvm::make_unique<TpiStreamBuilder>(*Msf, StreamIPI);
73   return *Ipi;
74 }
75 
76 PDBStringTableBuilder &PDBFileBuilder::getStringTableBuilder() {
77   return Strings;
78 }
79 
80 GSIStreamBuilder &PDBFileBuilder::getGsiBuilder() {
81   if (!Gsi)
82     Gsi = llvm::make_unique<GSIStreamBuilder>(*Msf);
83   return *Gsi;
84 }
85 
86 Expected<uint32_t> PDBFileBuilder::allocateNamedStream(StringRef Name,
87                                                        uint32_t Size) {
88   auto ExpectedStream = Msf->addStream(Size);
89   if (ExpectedStream)
90     NamedStreams.set(Name, *ExpectedStream);
91   return ExpectedStream;
92 }
93 
94 Error PDBFileBuilder::addNamedStream(StringRef Name, StringRef Data) {
95   Expected<uint32_t> ExpectedIndex = allocateNamedStream(Name, Data.size());
96   if (!ExpectedIndex)
97     return ExpectedIndex.takeError();
98   assert(NamedStreamData.count(*ExpectedIndex) == 0);
99   NamedStreamData[*ExpectedIndex] = Data;
100   return Error::success();
101 }
102 
103 void PDBFileBuilder::addInjectedSource(StringRef Name,
104                                        std::unique_ptr<MemoryBuffer> Buffer) {
105   // Stream names must be exact matches, since they get looked up in a hash
106   // table and the hash value is dependent on the exact contents of the string.
107   // link.exe lowercases a path and converts / to \, so we must do the same.
108   SmallString<64> VName;
109   sys::path::native(Name.lower(), VName);
110 
111   uint32_t NI = getStringTableBuilder().insert(Name);
112   uint32_t VNI = getStringTableBuilder().insert(VName);
113 
114   InjectedSourceDescriptor Desc;
115   Desc.Content = std::move(Buffer);
116   Desc.NameIndex = NI;
117   Desc.VNameIndex = VNI;
118   Desc.StreamName = "/src/files/";
119 
120   Desc.StreamName += VName;
121 
122   InjectedSources.push_back(std::move(Desc));
123 }
124 
125 Expected<msf::MSFLayout> PDBFileBuilder::finalizeMsfLayout() {
126 
127   if (Ipi && Ipi->getRecordCount() > 0) {
128     // In theory newer PDBs always have an ID stream, but by saying that we're
129     // only going to *really* have an ID stream if there is at least one ID
130     // record, we leave open the opportunity to test older PDBs such as those
131     // that don't have an ID stream.
132     auto &Info = getInfoBuilder();
133     Info.addFeature(PdbRaw_FeatureSig::VC140);
134   }
135 
136   uint32_t StringsLen = Strings.calculateSerializedSize();
137 
138   Expected<uint32_t> SN = allocateNamedStream("/LinkInfo", 0);
139   if (!SN)
140     return SN.takeError();
141 
142   if (Gsi) {
143     if (auto EC = Gsi->finalizeMsfLayout())
144       return std::move(EC);
145     if (Dbi) {
146       Dbi->setPublicsStreamIndex(Gsi->getPublicsStreamIndex());
147       Dbi->setGlobalsStreamIndex(Gsi->getGlobalsStreamIndex());
148       Dbi->setSymbolRecordStreamIndex(Gsi->getRecordStreamIdx());
149     }
150   }
151   if (Tpi) {
152     if (auto EC = Tpi->finalizeMsfLayout())
153       return std::move(EC);
154   }
155   if (Dbi) {
156     if (auto EC = Dbi->finalizeMsfLayout())
157       return std::move(EC);
158   }
159   SN = allocateNamedStream("/names", StringsLen);
160   if (!SN)
161     return SN.takeError();
162 
163   if (Ipi) {
164     if (auto EC = Ipi->finalizeMsfLayout())
165       return std::move(EC);
166   }
167 
168   // Do this last, since it relies on the named stream map being complete, and
169   // that can be updated by previous steps in the finalization.
170   if (Info) {
171     if (auto EC = Info->finalizeMsfLayout())
172       return std::move(EC);
173   }
174 
175   if (!InjectedSources.empty()) {
176     for (const auto &IS : InjectedSources) {
177       JamCRC CRC(0);
178       CRC.update(makeArrayRef(IS.Content->getBufferStart(),
179                               IS.Content->getBufferSize()));
180 
181       SrcHeaderBlockEntry Entry;
182       ::memset(&Entry, 0, sizeof(SrcHeaderBlockEntry));
183       Entry.Size = sizeof(SrcHeaderBlockEntry);
184       Entry.FileSize = IS.Content->getBufferSize();
185       Entry.FileNI = IS.NameIndex;
186       Entry.VFileNI = IS.VNameIndex;
187       Entry.ObjNI = 1;
188       Entry.IsVirtual = 0;
189       Entry.Version =
190           static_cast<uint32_t>(PdbRaw_SrcHeaderBlockVer::SrcVerOne);
191       Entry.CRC = CRC.getCRC();
192       StringRef VName = getStringTableBuilder().getStringForId(IS.VNameIndex);
193       InjectedSourceTable.set_as(VName, std::move(Entry));
194     }
195 
196     uint32_t SrcHeaderBlockSize =
197         sizeof(SrcHeaderBlockHeader) +
198         InjectedSourceTable.calculateSerializedLength();
199     SN = allocateNamedStream("/src/headerblock", SrcHeaderBlockSize);
200     if (!SN)
201       return SN.takeError();
202     for (const auto &IS : InjectedSources) {
203       SN = allocateNamedStream(IS.StreamName, IS.Content->getBufferSize());
204       if (!SN)
205         return SN.takeError();
206     }
207   }
208 
209   // Do this last, since it relies on the named stream map being complete, and
210   // that can be updated by previous steps in the finalization.
211   if (Info) {
212     if (auto EC = Info->finalizeMsfLayout())
213       return std::move(EC);
214   }
215 
216   return Msf->build();
217 }
218 
219 Expected<uint32_t> PDBFileBuilder::getNamedStreamIndex(StringRef Name) const {
220   uint32_t SN = 0;
221   if (!NamedStreams.get(Name, SN))
222     return llvm::make_error<pdb::RawError>(raw_error_code::no_stream);
223   return SN;
224 }
225 
226 void PDBFileBuilder::commitFpm(WritableBinaryStream &MsfBuffer,
227                                const MSFLayout &Layout) {
228   auto FpmStream =
229       WritableMappedBlockStream::createFpmStream(Layout, MsfBuffer, Allocator);
230 
231   // We only need to create the alt fpm stream so that it gets initialized.
232   WritableMappedBlockStream::createFpmStream(Layout, MsfBuffer, Allocator,
233                                              true);
234 
235   uint32_t BI = 0;
236   BinaryStreamWriter FpmWriter(*FpmStream);
237   while (BI < Layout.SB->NumBlocks) {
238     uint8_t ThisByte = 0;
239     for (uint32_t I = 0; I < 8; ++I) {
240       bool IsFree =
241           (BI < Layout.SB->NumBlocks) ? Layout.FreePageMap.test(BI) : true;
242       uint8_t Mask = uint8_t(IsFree) << I;
243       ThisByte |= Mask;
244       ++BI;
245     }
246     cantFail(FpmWriter.writeObject(ThisByte));
247   }
248   assert(FpmWriter.bytesRemaining() == 0);
249 }
250 
251 void PDBFileBuilder::commitSrcHeaderBlock(WritableBinaryStream &MsfBuffer,
252                                           const msf::MSFLayout &Layout) {
253   assert(!InjectedSourceTable.empty());
254 
255   uint32_t SN = cantFail(getNamedStreamIndex("/src/headerblock"));
256   auto Stream = WritableMappedBlockStream::createIndexedStream(
257       Layout, MsfBuffer, SN, Allocator);
258   BinaryStreamWriter Writer(*Stream);
259 
260   SrcHeaderBlockHeader Header;
261   ::memset(&Header, 0, sizeof(Header));
262   Header.Version = static_cast<uint32_t>(PdbRaw_SrcHeaderBlockVer::SrcVerOne);
263   Header.Size = Writer.bytesRemaining();
264 
265   cantFail(Writer.writeObject(Header));
266   cantFail(InjectedSourceTable.commit(Writer));
267 
268   assert(Writer.bytesRemaining() == 0);
269 }
270 
271 void PDBFileBuilder::commitInjectedSources(WritableBinaryStream &MsfBuffer,
272                                            const msf::MSFLayout &Layout) {
273   if (InjectedSourceTable.empty())
274     return;
275 
276   commitSrcHeaderBlock(MsfBuffer, Layout);
277 
278   for (const auto &IS : InjectedSources) {
279     uint32_t SN = cantFail(getNamedStreamIndex(IS.StreamName));
280 
281     auto SourceStream = WritableMappedBlockStream::createIndexedStream(
282         Layout, MsfBuffer, SN, Allocator);
283     BinaryStreamWriter SourceWriter(*SourceStream);
284     assert(SourceWriter.bytesRemaining() == IS.Content->getBufferSize());
285     cantFail(SourceWriter.writeBytes(
286         arrayRefFromStringRef(IS.Content->getBuffer())));
287   }
288 }
289 
290 Error PDBFileBuilder::commit(StringRef Filename) {
291   assert(!Filename.empty());
292   auto ExpectedLayout = finalizeMsfLayout();
293   if (!ExpectedLayout)
294     return ExpectedLayout.takeError();
295   auto &Layout = *ExpectedLayout;
296 
297   uint64_t Filesize = Layout.SB->BlockSize * Layout.SB->NumBlocks;
298   auto OutFileOrError = FileOutputBuffer::create(Filename, Filesize);
299   if (auto E = OutFileOrError.takeError())
300     return E;
301   FileOutputBuffer *FOB = OutFileOrError->get();
302 
303   FileBufferByteStream Buffer(std::move(*OutFileOrError),
304                               llvm::support::little);
305   BinaryStreamWriter Writer(Buffer);
306 
307   if (auto EC = Writer.writeObject(*Layout.SB))
308     return EC;
309 
310   commitFpm(Buffer, Layout);
311 
312   uint32_t BlockMapOffset =
313       msf::blockToOffset(Layout.SB->BlockMapAddr, Layout.SB->BlockSize);
314   Writer.setOffset(BlockMapOffset);
315   if (auto EC = Writer.writeArray(Layout.DirectoryBlocks))
316     return EC;
317 
318   auto DirStream = WritableMappedBlockStream::createDirectoryStream(
319       Layout, Buffer, Allocator);
320   BinaryStreamWriter DW(*DirStream);
321   if (auto EC = DW.writeInteger<uint32_t>(Layout.StreamSizes.size()))
322     return EC;
323 
324   if (auto EC = DW.writeArray(Layout.StreamSizes))
325     return EC;
326 
327   for (const auto &Blocks : Layout.StreamMap) {
328     if (auto EC = DW.writeArray(Blocks))
329       return EC;
330   }
331 
332   auto ExpectedSN = getNamedStreamIndex("/names");
333   if (!ExpectedSN)
334     return ExpectedSN.takeError();
335 
336   auto NS = WritableMappedBlockStream::createIndexedStream(
337       Layout, Buffer, *ExpectedSN, Allocator);
338   BinaryStreamWriter NSWriter(*NS);
339   if (auto EC = Strings.commit(NSWriter))
340     return EC;
341 
342   for (const auto &NSE : NamedStreamData) {
343     if (NSE.second.empty())
344       continue;
345 
346     auto NS = WritableMappedBlockStream::createIndexedStream(
347         Layout, Buffer, NSE.first, Allocator);
348     BinaryStreamWriter NSW(*NS);
349     if (auto EC = NSW.writeBytes(arrayRefFromStringRef(NSE.second)))
350       return EC;
351   }
352 
353   if (Info) {
354     if (auto EC = Info->commit(Layout, Buffer))
355       return EC;
356   }
357 
358   if (Dbi) {
359     if (auto EC = Dbi->commit(Layout, Buffer))
360       return EC;
361   }
362 
363   if (Tpi) {
364     if (auto EC = Tpi->commit(Layout, Buffer))
365       return EC;
366   }
367 
368   if (Ipi) {
369     if (auto EC = Ipi->commit(Layout, Buffer))
370       return EC;
371   }
372 
373   if (Gsi) {
374     if (auto EC = Gsi->commit(Layout, Buffer))
375       return EC;
376   }
377 
378   auto InfoStreamBlocks = Layout.StreamMap[StreamPDB];
379   assert(!InfoStreamBlocks.empty());
380   uint64_t InfoStreamFileOffset =
381       blockToOffset(InfoStreamBlocks.front(), Layout.SB->BlockSize);
382   InfoStreamHeader *H = reinterpret_cast<InfoStreamHeader *>(
383       FOB->getBufferStart() + InfoStreamFileOffset);
384 
385   commitInjectedSources(Buffer, Layout);
386 
387   // Set the build id at the very end, after every other byte of the PDB
388   // has been written.
389   // FIXME: Use a hash of the PDB rather than time(nullptr) for the signature.
390   H->Age = Info->getAge();
391   H->Guid = Info->getGuid();
392   Optional<uint32_t> Sig = Info->getSignature();
393   H->Signature = Sig.hasValue() ? *Sig : time(nullptr);
394 
395   return Buffer.commit();
396 }
397