1 //== BackgroundIndexStorage.cpp - Provide caching support to BackgroundIndex ==/
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "GlobalCompilationDatabase.h"
10 #include "index/Background.h"
11 #include "support/Logger.h"
12 #include "support/Path.h"
13 #include "llvm/ADT/Optional.h"
14 #include "llvm/ADT/SmallString.h"
15 #include "llvm/ADT/StringRef.h"
16 #include "llvm/Support/Error.h"
17 #include "llvm/Support/FileSystem.h"
18 #include "llvm/Support/FileUtilities.h"
19 #include "llvm/Support/MemoryBuffer.h"
20 #include "llvm/Support/Path.h"
21 #include <functional>
22
23 namespace clang {
24 namespace clangd {
25 namespace {
26
getShardPathFromFilePath(llvm::StringRef ShardRoot,llvm::StringRef FilePath)27 std::string getShardPathFromFilePath(llvm::StringRef ShardRoot,
28 llvm::StringRef FilePath) {
29 llvm::SmallString<128> ShardRootSS(ShardRoot);
30 llvm::sys::path::append(ShardRootSS, llvm::sys::path::filename(FilePath) +
31 "." + llvm::toHex(digest(FilePath)) +
32 ".idx");
33 return std::string(ShardRootSS.str());
34 }
35
36 // Uses disk as a storage for index shards.
37 class DiskBackedIndexStorage : public BackgroundIndexStorage {
38 std::string DiskShardRoot;
39
40 public:
41 // Creates `DiskShardRoot` and any parents during construction.
DiskBackedIndexStorage(llvm::StringRef Directory)42 DiskBackedIndexStorage(llvm::StringRef Directory) : DiskShardRoot(Directory) {
43 std::error_code OK;
44 std::error_code EC = llvm::sys::fs::create_directories(DiskShardRoot);
45 if (EC != OK) {
46 elog("Failed to create directory {0} for index storage: {1}",
47 DiskShardRoot, EC.message());
48 }
49 }
50
51 std::unique_ptr<IndexFileIn>
loadShard(llvm::StringRef ShardIdentifier) const52 loadShard(llvm::StringRef ShardIdentifier) const override {
53 const std::string ShardPath =
54 getShardPathFromFilePath(DiskShardRoot, ShardIdentifier);
55 auto Buffer = llvm::MemoryBuffer::getFile(ShardPath);
56 if (!Buffer)
57 return nullptr;
58 if (auto I =
59 readIndexFile(Buffer->get()->getBuffer(), SymbolOrigin::Background))
60 return std::make_unique<IndexFileIn>(std::move(*I));
61 else
62 elog("Error while reading shard {0}: {1}", ShardIdentifier,
63 I.takeError());
64 return nullptr;
65 }
66
storeShard(llvm::StringRef ShardIdentifier,IndexFileOut Shard) const67 llvm::Error storeShard(llvm::StringRef ShardIdentifier,
68 IndexFileOut Shard) const override {
69 auto ShardPath = getShardPathFromFilePath(DiskShardRoot, ShardIdentifier);
70 return llvm::writeFileAtomically(ShardPath + ".tmp.%%%%%%%%", ShardPath,
71 [&Shard](llvm::raw_ostream &OS) {
72 OS << Shard;
73 return llvm::Error::success();
74 });
75 }
76 };
77
78 // Doesn't persist index shards anywhere (used when the CDB dir is unknown).
79 // We could consider indexing into ~/.clangd/ or so instead.
80 class NullStorage : public BackgroundIndexStorage {
81 public:
82 std::unique_ptr<IndexFileIn>
loadShard(llvm::StringRef ShardIdentifier) const83 loadShard(llvm::StringRef ShardIdentifier) const override {
84 return nullptr;
85 }
86
storeShard(llvm::StringRef ShardIdentifier,IndexFileOut Shard) const87 llvm::Error storeShard(llvm::StringRef ShardIdentifier,
88 IndexFileOut Shard) const override {
89 vlog("Couldn't find project for {0}, indexing in-memory only",
90 ShardIdentifier);
91 return llvm::Error::success();
92 }
93 };
94
95 // Creates and owns IndexStorages for multiple CDBs.
96 // When a CDB root is found, shards are stored in $ROOT/.cache/clangd/index/.
97 // When no root is found, the fallback path is ~/.cache/clangd/index/.
98 class DiskBackedIndexStorageManager {
99 public:
DiskBackedIndexStorageManager(std::function<llvm::Optional<ProjectInfo> (PathRef)> GetProjectInfo)100 DiskBackedIndexStorageManager(
101 std::function<llvm::Optional<ProjectInfo>(PathRef)> GetProjectInfo)
102 : IndexStorageMapMu(std::make_unique<std::mutex>()),
103 GetProjectInfo(std::move(GetProjectInfo)) {
104 llvm::SmallString<128> FallbackDir;
105 if (llvm::sys::path::cache_directory(FallbackDir))
106 llvm::sys::path::append(FallbackDir, "clangd", "index");
107 this->FallbackDir = FallbackDir.str().str();
108 }
109
110 // Creates or fetches to storage from cache for the specified project.
operator ()(PathRef File)111 BackgroundIndexStorage *operator()(PathRef File) {
112 std::lock_guard<std::mutex> Lock(*IndexStorageMapMu);
113 llvm::SmallString<128> StorageDir(FallbackDir);
114 if (auto PI = GetProjectInfo(File)) {
115 StorageDir = PI->SourceRoot;
116 llvm::sys::path::append(StorageDir, ".cache", "clangd", "index");
117 }
118 auto &IndexStorage = IndexStorageMap[StorageDir];
119 if (!IndexStorage)
120 IndexStorage = create(StorageDir);
121 return IndexStorage.get();
122 }
123
124 private:
create(PathRef CDBDirectory)125 std::unique_ptr<BackgroundIndexStorage> create(PathRef CDBDirectory) {
126 if (CDBDirectory.empty()) {
127 elog("Tried to create storage for empty directory!");
128 return std::make_unique<NullStorage>();
129 }
130 return std::make_unique<DiskBackedIndexStorage>(CDBDirectory);
131 }
132
133 Path FallbackDir;
134
135 llvm::StringMap<std::unique_ptr<BackgroundIndexStorage>> IndexStorageMap;
136 std::unique_ptr<std::mutex> IndexStorageMapMu;
137
138 std::function<llvm::Optional<ProjectInfo>(PathRef)> GetProjectInfo;
139 };
140
141 } // namespace
142
143 BackgroundIndexStorage::Factory
createDiskBackedStorageFactory(std::function<llvm::Optional<ProjectInfo> (PathRef)> GetProjectInfo)144 BackgroundIndexStorage::createDiskBackedStorageFactory(
145 std::function<llvm::Optional<ProjectInfo>(PathRef)> GetProjectInfo) {
146 return DiskBackedIndexStorageManager(std::move(GetProjectInfo));
147 }
148
149 } // namespace clangd
150 } // namespace clang
151