Add Cache Pruning support
Incremental LTO will usea cache to store object files. This patch handles the pruning part of the cache, exposing a few knobs: - Pruning interval: the implementation keeps a "timestamp" file in the directory and will scan it only after a given interval since the last modification of the timestamp file. This is for performance purpose, we don't want to scan continuously the folder. - Entry expiration: this is the time after which a file that hasn't been used is remove from the cache. - Maximum size: expressed in percentage of the available disk space, it helps to avoid that we blow up the disk space. http://reviews.llvm.org/D18422 From: Mehdi Amini <mehdi.amini@apple.com> llvm-svn: 265209
This commit is contained in:
parent
4fda708624
commit
27814980a3
|
@ -0,0 +1,69 @@
|
|||
//=- CachePruning.h - Helper to manage the pruning of a cache dir -*- C++ -*-=//
|
||||
//
|
||||
// The LLVM Compiler Infrastructure
|
||||
//
|
||||
// This file is distributed under the University of Illinois Open Source
|
||||
// License. See LICENSE.TXT for details.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// This file implements pruning of a directory intended for cache storage, using
|
||||
// various policies.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef LLVM_SUPPORT_CACHE_PRUNING_H
|
||||
#define LLVM_SUPPORT_CACHE_PRUNING_H
|
||||
|
||||
#include "llvm/ADT/StringRef.h"
|
||||
|
||||
namespace llvm {
|
||||
|
||||
/// Handle pruning a directory provided a path and some options to control what
|
||||
/// to prune.
|
||||
class CachePruning {
|
||||
public:
|
||||
/// Prepare to prune \p Path.
|
||||
CachePruning(StringRef Path) : Path(Path) {}
|
||||
|
||||
/// Define the pruning interval. This is intended to be used to avoid scanning
|
||||
/// the directory too often. It does not impact the decision of which file to
|
||||
/// prune. A value of 0 forces the scan to occurs.
|
||||
CachePruning &setPruningInterval(int PruningInterval) {
|
||||
Interval = PruningInterval;
|
||||
return *this;
|
||||
}
|
||||
|
||||
/// Define the expiration for a file. When a file hasn't been accessed for
|
||||
/// \p ExpireAfter seconds, it is removed from the cache. A value of 0 disable
|
||||
/// the expiration-based pruning.
|
||||
CachePruning &setEntryExpiration(unsigned ExpireAfter) {
|
||||
Expiration = ExpireAfter;
|
||||
return *this;
|
||||
}
|
||||
|
||||
/// Define the maximum size for the cache directory, in terms of percentage of
|
||||
/// the available space on the the disk. Set to 100 to indicate no limit, 50
|
||||
/// to indicate that the cache size will not be left over half the
|
||||
/// available disk space. A value over 100 will be reduced to 100. A value of
|
||||
/// 0 disable the size-based pruning.
|
||||
CachePruning &setMaxSize(unsigned Percentage) {
|
||||
PercentageOfAvailableSpace = std::min(100u, Percentage);
|
||||
return *this;
|
||||
}
|
||||
|
||||
/// Peform pruning using the supplied options, returns true if pruning
|
||||
/// occured, i.e. if PruningInterval was expired.
|
||||
bool prune();
|
||||
|
||||
private:
|
||||
// Options that matches the setters above.
|
||||
std::string Path;
|
||||
unsigned Expiration = 0;
|
||||
unsigned Interval = 0;
|
||||
unsigned PercentageOfAvailableSpace = 0;
|
||||
};
|
||||
|
||||
} // namespace llvm
|
||||
|
||||
#endif
|
|
@ -35,6 +35,7 @@ add_llvm_library(LLVMSupport
|
|||
Allocator.cpp
|
||||
BlockFrequency.cpp
|
||||
BranchProbability.cpp
|
||||
CachePruning.cpp
|
||||
circular_raw_ostream.cpp
|
||||
COM.cpp
|
||||
CommandLine.cpp
|
||||
|
|
|
@ -0,0 +1,130 @@
|
|||
//===-CachePruning.cpp - LLVM Cache Directory Pruning ---------------------===//
|
||||
//
|
||||
// The LLVM Compiler Infrastructure
|
||||
//
|
||||
// This file is distributed under the University of Illinois Open Source
|
||||
// License. See LICENSE.TXT for details.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// This file implements the pruning of a directory based on least recently used.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include "llvm/Support/CachePruning.h"
|
||||
|
||||
#include "llvm/Support/FileSystem.h"
|
||||
#include "llvm/Support/Path.h"
|
||||
#include "llvm/Support/raw_ostream.h"
|
||||
|
||||
#include <set>
|
||||
|
||||
using namespace llvm;
|
||||
|
||||
/// Write a new timestamp file with the given path. This is used for the pruning
|
||||
/// interval option.
|
||||
static void writeTimestampFile(StringRef TimestampFile) {
|
||||
std::error_code EC;
|
||||
raw_fd_ostream Out(TimestampFile.str(), EC, sys::fs::F_None);
|
||||
}
|
||||
|
||||
/// Prune the cache of files that haven't been accessed in a long time.
|
||||
bool CachePruning::prune() {
|
||||
SmallString<128> TimestampFile(Path);
|
||||
sys::path::append(TimestampFile, "llvmcache.timestamp");
|
||||
|
||||
if (Expiration == 0 && PercentageOfAvailableSpace == 0)
|
||||
// Nothing will be pruned, early exit
|
||||
return false;
|
||||
|
||||
// Try to stat() the timestamp file.
|
||||
sys::fs::file_status FileStatus;
|
||||
sys::TimeValue CurrentTime = sys::TimeValue::now();
|
||||
if (sys::fs::status(TimestampFile, FileStatus)) {
|
||||
if (errno == ENOENT) {
|
||||
// If the timestamp file wasn't there, create one now.
|
||||
writeTimestampFile(TimestampFile);
|
||||
} else {
|
||||
// Unknown error?
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
if (Interval) {
|
||||
// Check whether the time stamp is older than our pruning interval.
|
||||
// If not, do nothing.
|
||||
sys::TimeValue TimeStampModTime = FileStatus.getLastModificationTime();
|
||||
auto TimeInterval = sys::TimeValue(sys::TimeValue::SecondsType(Interval));
|
||||
if (CurrentTime - TimeStampModTime <= TimeInterval)
|
||||
return false;
|
||||
}
|
||||
// Write a new timestamp file so that nobody else attempts to prune.
|
||||
// There is a benign race condition here, if two processes happen to
|
||||
// notice at the same time that the timestamp is out-of-date.
|
||||
writeTimestampFile(TimestampFile);
|
||||
}
|
||||
|
||||
bool ShouldComputeSize = (PercentageOfAvailableSpace > 0);
|
||||
|
||||
// Keep track of space
|
||||
std::set<std::pair<uint64_t, std::string>> FileSizes;
|
||||
uint64_t TotalSize = 0;
|
||||
// Helper to add a path to the set of files to consider for size-based
|
||||
// pruning, sorted by last accessed time.
|
||||
auto AddToFileListForSizePruning =
|
||||
[&](StringRef Path, sys::TimeValue FileAccessTime) {
|
||||
if (!ShouldComputeSize)
|
||||
return;
|
||||
TotalSize += FileStatus.getSize();
|
||||
FileSizes.insert(
|
||||
std::make_pair(FileAccessTime.seconds(), std::string(Path)));
|
||||
};
|
||||
|
||||
// Walk the entire directory cache, looking for unused files.
|
||||
std::error_code EC;
|
||||
SmallString<128> CachePathNative;
|
||||
sys::path::native(Path, CachePathNative);
|
||||
auto TimeExpiration = sys::TimeValue(sys::TimeValue::SecondsType(Expiration));
|
||||
// Walk all of the files within this directory.
|
||||
for (sys::fs::directory_iterator File(CachePathNative, EC), FileEnd;
|
||||
File != FileEnd && !EC; File.increment(EC)) {
|
||||
// Do not touch the timestamp.
|
||||
if (File->path() == TimestampFile)
|
||||
continue;
|
||||
|
||||
// Look at this file. If we can't stat it, there's nothing interesting
|
||||
// there.
|
||||
if (sys::fs::status(File->path(), FileStatus))
|
||||
continue;
|
||||
|
||||
// If the file hasn't been used recently enough, delete it
|
||||
sys::TimeValue FileAccessTime = FileStatus.getLastAccessedTime();
|
||||
if (CurrentTime - FileAccessTime > TimeExpiration) {
|
||||
sys::fs::remove(File->path());
|
||||
continue;
|
||||
}
|
||||
|
||||
// Leave it here for now, but add it to the list of size-based pruning.
|
||||
AddToFileListForSizePruning(File->path(), FileAccessTime);
|
||||
}
|
||||
|
||||
// Prune for size now if needed
|
||||
if (ShouldComputeSize) {
|
||||
auto ErrOrSpaceInfo = sys::fs::disk_space(Path);
|
||||
if (!ErrOrSpaceInfo) {
|
||||
report_fatal_error("Can't get available size");
|
||||
}
|
||||
sys::fs::space_info SpaceInfo = ErrOrSpaceInfo.get();
|
||||
auto AvailableSpace = TotalSize + SpaceInfo.free;
|
||||
auto FileAndSize = FileSizes.rbegin();
|
||||
// Remove the oldest accessed files first, till we get below the threshold
|
||||
while (((100 * TotalSize) / AvailableSpace) > PercentageOfAvailableSpace &&
|
||||
FileAndSize != FileSizes.rend()) {
|
||||
// Remove the file.
|
||||
sys::fs::remove(FileAndSize->second);
|
||||
// Update size
|
||||
TotalSize -= FileAndSize->first;
|
||||
++FileAndSize;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
Loading…
Reference in New Issue