bahman/scylla
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

exceptions: Shutdown communications on non file I/O errors

Browse Source 
Apply the same treatment to non file filesystem I/O errors.

Signed-off-by: Benoît Canet <benoit@scylladb.com>
Message-Id: <1458154098-9977-2-git-send-email-benoit@scylladb.com>
This commit is contained in:
Benoît Canet
2016-03-16 18:48:18 +00:00
committed by Avi Kivity
parent 1fb9a48ac5
commit 3b1d3d977d
7 changed files with 113 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
checked-file-impl.hh
View File

@@ -134,3 +134,14 @@ inline open_checked_file_dma(disk_error_signal_type& signal,
});
});
}
future<file>
inline open_checked_directory(disk_error_signal_type& signal,
sstring name)
{
return do_io_check(signal, [&] {
return engine().open_directory(name).then([&] (file f) {
return make_ready_future<file>(make_checked_file(signal, f));
});
});
}

37
database.cc
View File

@@ -61,6 +61,7 @@
#include "service/priority_manager.hh"
#include "checked-file-impl.hh"
#include "disk-error-handler.hh"
using namespace std::chrono_literals;
@@ -432,9 +433,9 @@ private:
future<> lister::scan_dir(sstring name, lister::dir_entry_types type, walker_type walker, filter_type filter) {
return engine().open_directory(name).then([type, walker = std::move(walker), filter = std::move(filter), name] (file f) {
auto l = make_lw_shared<lister>(std::move(f), type, walker, filter, name);
return l->done().then([l] { });
return open_checked_directory(general_disk_error, name).then([type, walker = std::move(walker), filter = std::move(filter), name] (file f) {
auto l = make_lw_shared<lister>(std::move(f), type, walker, filter, name);
return l->done().then([l] { });
});
}
@@ -1202,7 +1203,7 @@ database::init_system_keyspace() {
db::system_keyspace::make(*this, durable, _cfg->volatile_system_keyspace_for_testing());
// FIXME support multiple directories
return touch_directory(_cfg->data_file_directories()[0] + "/" + db::system_keyspace::NAME).then([this] {
return io_check(touch_directory, _cfg->data_file_directories()[0] + "/" + db::system_keyspace::NAME).then([this] {
return populate_keyspace(_cfg->data_file_directories()[0], db::system_keyspace::NAME).then([this]() {
return init_commitlog();
});
@@ -1441,7 +1442,7 @@ keyspace::column_family_directory(const sstring& name, utils::UUID uuid) const {
future<>
keyspace::make_directory_for_column_family(const sstring& name, utils::UUID uuid) {
return touch_directory(column_family_directory(name, uuid));
return io_check(touch_directory, column_family_directory(name, uuid));
}
no_such_keyspace::no_such_keyspace(const sstring& ks_name)
@@ -1507,7 +1508,7 @@ database::create_keyspace(const lw_shared_ptr<keyspace_metadata>& ksm) {
create_in_memory_keyspace(ksm);
auto& datadir = _keyspaces.at(ksm->name()).datadir();
if (datadir != "") {
return touch_directory(datadir);
return io_check(touch_directory, datadir);
} else {
return make_ready_future<>();
}
@@ -2062,7 +2063,7 @@ seal_snapshot(sstring jsondir) {
dblog.debug("Storing manifest {}", jsonfile);
return recursive_touch_directory(jsondir).then([jsonfile, json = std::move(json)] {
return io_check(recursive_touch_directory, jsondir).then([jsonfile, json = std::move(json)] {
return open_checked_file_dma(general_disk_error, jsonfile, open_flags::wo | open_flags::create | open_flags::truncate).then([json](file f) {
return do_with(make_file_output_stream(std::move(f)), [json] (output_stream<char>& out) {
return out.write(json.c_str(), json.size()).then([&out] {
@@ -2073,7 +2074,7 @@ seal_snapshot(sstring jsondir) {
});
});
}).then([jsondir] {
return sync_directory(std::move(jsondir));
return io_check(sync_directory, std::move(jsondir));
}).finally([jsondir] {
pending_snapshots.erase(jsondir);
return make_ready_future<>();
@@ -2088,7 +2089,7 @@ future<> column_family::snapshot(sstring name) {
return parallel_for_each(tables, [name](sstables::shared_sstable sstable) {
auto dir = sstable->get_dir() + "/snapshots/" + name;
return recursive_touch_directory(dir).then([sstable, dir] {
return io_check(recursive_touch_directory, dir).then([sstable, dir] {
return sstable->create_links(dir).then_wrapped([] (future<> f) {
// If the SSTables are shared, one of the CPUs will fail here.
// That is completely fine, though. We only need one link.
@@ -2106,7 +2107,7 @@ future<> column_family::snapshot(sstring name) {
// This is not just an optimization. If we have no files, jsondir may not have been created,
// and sync_directory would throw.
if (tables.size()) {
return sync_directory(std::move(jsondir));
return io_check(sync_directory, std::move(jsondir));
} else {
return make_ready_future<>();
}
@@ -2151,7 +2152,7 @@ future<> column_family::snapshot(sstring name) {
future<bool> column_family::snapshot_exists(sstring tag) {
sstring jsondir = _config.datadir + "/snapshots/" + tag;
return engine().open_directory(std::move(jsondir)).then_wrapped([] (future<file> f) {
return open_checked_directory(general_disk_error, std::move(jsondir)).then_wrapped([] (future<file> f) {
try {
f.get0();
return make_ready_future<bool>(true);
@@ -2200,20 +2201,20 @@ future<> column_family::clear_snapshot(sstring tag) {
}
auto newdir = curr_dir + "/" + de.name;
recurse = lister::scan_dir(newdir, dir_and_files, [this, curr_dir = newdir] (directory_entry de) {
return remove_file(curr_dir + "/" + de.name);
return io_check(remove_file, curr_dir + "/" + de.name);
});
}
return recurse.then([fname = curr_dir + "/" + de.name] {
return remove_file(fname);
return io_check(remove_file, fname);
});
}).then_wrapped([jsondir] (future<> f) {
// Fine if directory does not exist. If it did, we delete it
if (file_missing(std::move(f)) == missing::no) {
return remove_file(jsondir);
return io_check(remove_file, jsondir);
}
return make_ready_future<>();
}).then([parent] {
return sync_directory(parent).then_wrapped([] (future<> f) {
return io_check(sync_directory, parent).then_wrapped([] (future<> f) {
// Should always exist for empty tags, but may not exist for a single tag if we never took
// snapshots. We will check this here just to mask out the exception, without silencing
// unexpected ones.
@@ -2226,7 +2227,7 @@ future<> column_family::clear_snapshot(sstring tag) {
future<std::unordered_map<sstring, column_family::snapshot_details>> column_family::get_snapshot_details() {
std::unordered_map<sstring, snapshot_details> all_snapshots;
return do_with(std::move(all_snapshots), [this] (auto& all_snapshots) {
return engine().file_exists(_config.datadir + "/snapshots").then([this, &all_snapshots](bool file_exists) {
return io_check([&] { return engine().file_exists(_config.datadir + "/snapshots"); }).then([this, &all_snapshots](bool file_exists) {
if (!file_exists) {
return make_ready_future<>();
}
@@ -2235,7 +2236,7 @@ future<std::unordered_map<sstring, column_family::snapshot_details>> column_fami
auto snapshot = _config.datadir + "/snapshots/" + snapshot_name;
all_snapshots.emplace(snapshot_name, snapshot_details());
return lister::scan_dir(snapshot, { directory_entry_type::regular }, [this, &all_snapshots, snapshot, snapshot_name] (directory_entry de) {
return file_size(snapshot + "/" + de.name).then([this, &all_snapshots, snapshot_name, name = de.name] (auto size) {
return io_check(file_size, snapshot + "/" + de.name).then([this, &all_snapshots, snapshot_name, name = de.name] (auto size) {
// The manifest is the only file expected to be in this directory not belonging to the SSTable.
// For it, we account the total size, but zero it for the true size calculation.
//
@@ -2251,7 +2252,7 @@ future<std::unordered_map<sstring, column_family::snapshot_details>> column_fami
}).then([this, &all_snapshots, snapshot_name, name = de.name] (auto size) {
// FIXME: When we support multiple data directories, the file may not necessarily
// live in this same location. May have to test others as well.
return file_size(_config.datadir + "/" + name).then_wrapped([&all_snapshots, snapshot_name, size] (auto fut) {
return io_check(file_size, _config.datadir + "/" + name).then_wrapped([&all_snapshots, snapshot_name, size] (auto fut) {
try {
// File exists in the main SSTable directory. Snapshots are not contributing to size
fut.get0();

9
db/commitlog/commitlog.cc
View File

@@ -71,6 +71,7 @@
#include <boost/range/adaptor/transformed.hpp>
#include "checked-file-impl.hh"
#include "disk-error-handler.hh"
static logging::logger logger("commitlog");
@@ -464,7 +465,7 @@ public:
++_segment_manager->totals.segments_destroyed;
_segment_manager->totals.total_size_on_disk -= size_on_disk();
_segment_manager->totals.total_size -= (size_on_disk() + _buffer.size());
::unlink(
commit_io_check(::unlink,
(_segment_manager->cfg.commit_log_location + "/" + _desc.filename()).c_str());
} else {
logger.warn("Segment {} is dirty and is left on disk.", *this);
@@ -897,7 +898,7 @@ db::commitlog::segment_manager::list_descriptors(sstring dirname) {
}
};
return engine().open_directory(dirname).then([this, dirname](file dir) {
return open_checked_directory(commit_error, dirname).then([this, dirname](file dir) {
auto h = make_lw_shared<helper>(std::move(dirname), std::move(dir));
return h->done().then([h]() {
return make_ready_future<std::vector<db::commitlog::descriptor>>(std::move(h->_result));
@@ -1443,6 +1444,8 @@ const db::commitlog::config& db::commitlog::active_config() const {
return _segment_manager->cfg;
}
// No commit_io_check needed in the log reader since the database will fail
// on error at startup if required
future<std::unique_ptr<subscription<temporary_buffer<char>, db::replay_position>>>
db::commitlog::read_log_file(const sstring& filename, commit_load_reader_func next, position_type off) {
return open_checked_file_dma(commit_error, filename, open_flags::ro).then([next = std::move(next), off](file f) {
@@ -1451,6 +1454,8 @@ db::commitlog::read_log_file(const sstring& filename, commit_load_reader_func ne
});
}
// No commit_io_check needed in the log reader since the database will fail
// on error at startup if required
subscription<temporary_buffer<char>, db::replay_position>
db::commitlog::read_log_file(file f, commit_load_reader_func next, position_type off) {
struct work {

21
disk-error-handler.hh
View File

@@ -77,3 +77,24 @@ auto do_io_check(disk_error_signal_type& signal, Func&& func, Args&&... args) {
throw;
}
}
template<typename Func, typename... Args>
auto commit_io_check(Func&& func, Args&&... args) {
return do_io_check(commit_error, func, std::forward<Args>(args)...);
}
template<typename Func, typename... Args>
auto sstable_read_io_check(Func&& func, Args&&... args) {
return do_io_check(sstable_read_error, func, std::forward<Args>(args)...);
}
template<typename Func, typename... Args>
auto sstable_write_io_check(Func&& func, Args&&... args) {
return do_io_check(sstable_write_error, func, std::forward<Args>(args)...);
}
template<typename Func, typename... Args>
auto io_check(Func&& func, Args&&... args) {
return do_io_check(general_disk_error, func, std::forward<Args>(args)...);
}

2
main.cc
View File

@@ -162,7 +162,7 @@ static void apply_logger_settings(sstring default_level, db::config::string_map
class directories {
public:
future<> touch_and_lock(sstring path) {
return recursive_touch_directory(path).then_wrapped([this, path] (future<> f) {
return io_check(recursive_touch_directory, path).then_wrapped([this, path] (future<> f) {
try {
f.get();
return utils::file_lock::acquire(path + "/.lock").then([this](utils::file_lock lock) {

6
sstables/filter.cc
View File

@@ -29,6 +29,8 @@
#include "sstables.hh"
#include "utils/bloom_filter.hh"
#include "disk-error-handler.hh"
namespace sstables {
future<> sstable::read_filter(const io_priority_class& pc) {
@@ -43,7 +45,9 @@ future<> sstable::read_filter(const io_priority_class& pc) {
bs.load(filter.buckets.elements.begin(), filter.buckets.elements.end());
_filter = utils::filter::create_filter(filter.hashes, std::move(bs));
}).then([this] {
return engine().file_size(this->filename(sstable::component_type::Filter));
return io_check([&] {
return engine().file_size(this->filename(sstable::component_type::Filter));
});
});
}).then([this] (auto size) {
_filter_file_size = size;

84
sstables/sstables.cc
View File

@@ -123,6 +123,7 @@ public:
return random_access_reader::close().then([this] {
return _file.close().handle_exception([save = _file] (auto ep) {
sstlog.warn("sstable close failed: {}", ep);
general_disk_error();
});
});
}
@@ -792,23 +793,28 @@ void sstable::write_toc(const io_priority_class& pc) {
// Flushing parent directory to guarantee that temporary TOC file reached
// the disk.
file dir_f = engine().open_directory(_dir).get0();
dir_f.flush().get();
dir_f.close().get();
file dir_f = open_checked_directory(sstable_write_error, _dir).get0();
sstable_write_io_check([&] {
dir_f.flush().get();
dir_f.close().get();
});
}
void sstable::seal_sstable() {
// SSTable sealing is about renaming temporary TOC file after guaranteeing
// that each component reached the disk safely.
file dir_f = engine().open_directory(_dir).get0();
// Guarantee that every component of this sstable reached the disk.
dir_f.flush().get();
// Rename TOC because it's no longer temporary.
engine().rename_file(filename(sstable::component_type::TemporaryTOC), filename(sstable::component_type::TOC)).get();
// Guarantee that the changes above reached the disk.
dir_f.flush().get();
dir_f.close().get();
file dir_f = open_checked_directory(sstable_write_error, _dir).get0();
sstable_write_io_check([&] {
// Guarantee that every component of this sstable reached the disk.
dir_f.flush().get();
// Rename TOC because it's no longer temporary.
engine().rename_file(filename(sstable::component_type::TemporaryTOC), filename(sstable::component_type::TOC)).get();
// Guarantee that the changes above reached the disk.
dir_f.flush().get();
dir_f.close().get();
});
// If this point was reached, sstable should be safe in disk.
sstlog.debug("SSTable with generation {} of {}.{} was sealed successfully.", _generation, _ks, _cf);
}
@@ -961,7 +967,9 @@ future<> sstable::open_data() {
// Get disk usage for this sstable (includes all components).
_bytes_on_disk = 0;
return do_for_each(_components, [this] (component_type c) {
return engine().file_size(this->filename(c)).then([this] (uint64_t bytes) {
return sstable_write_io_check([&] {
return engine().file_size(this->filename(c));
}).then([this] (uint64_t bytes) {
_bytes_on_disk += bytes;
});
});
@@ -1467,7 +1475,7 @@ future<> sstable::write_components(::mutation_reader mr,
if (backup) {
auto dir = get_dir() + "/backups/";
touch_directory(dir).get();
sstable_write_io_check(touch_directory, dir).get();
create_links(dir).get();
}
});
@@ -1525,8 +1533,8 @@ const sstring sstable::filename(sstring dir, sstring ks, sstring cf, version_typ
future<> sstable::create_links(sstring dir, int64_t generation) const {
// TemporaryTOC is always first, TOC is always last
auto dst = sstable::filename(dir, _ks, _cf, _version, generation, _format, component_type::TemporaryTOC);
return ::link_file(filename(component_type::TOC), dst).then([dir] {
return sync_directory(dir);
return sstable_write_io_check(::link_file, filename(component_type::TOC), dst).then([dir] {
return sstable_write_io_check(sync_directory, dir);
}).then([this, dir, generation] {
// FIXME: Should clean already-created links if we failed midway.
return parallel_for_each(_components, [this, dir, generation] (auto comp) {
@@ -1534,23 +1542,25 @@ future<> sstable::create_links(sstring dir, int64_t generation) const {
return make_ready_future<>();
}
auto dst = sstable::filename(dir, _ks, _cf, _version, generation, _format, comp);
return ::link_file(this->filename(comp), dst);
return sstable_write_io_check(::link_file, this->filename(comp), dst);
});
}).then([dir] {
return sync_directory(dir);
return sstable_write_io_check(sync_directory, dir);
}).then([dir, this, generation] {
auto src = sstable::filename(dir, _ks, _cf, _version, generation, _format, component_type::TemporaryTOC);
auto dst = sstable::filename(dir, _ks, _cf, _version, generation, _format, component_type::TOC);
return engine().rename_file(src, dst);
return sstable_write_io_check([&] {
return engine().rename_file(src, dst);
});
}).then([dir] {
return sync_directory(dir);
return sstable_write_io_check(sync_directory, dir);
});
}
future<> sstable::set_generation(int64_t new_generation) {
return create_links(_dir, new_generation).then([this] {
return remove_file(filename(component_type::TOC)).then([this] {
return sync_directory(_dir);
return sstable_write_io_check(sync_directory, _dir);
}).then([this] {
return parallel_for_each(_components, [this] (auto comp) {
if (comp == component_type::TOC) {
@@ -1722,11 +1732,13 @@ sstable::~sstable() {
if (_index_file) {
_index_file.close().handle_exception([save = _index_file, op = background_jobs().start()] (auto ep) {
sstlog.warn("sstable close index_file failed: {}", ep);
general_disk_error();
});
}
if (_data_file) {
_data_file.close().handle_exception([save = _data_file, op = background_jobs().start()] (auto ep) {
sstlog.warn("sstable close data_file failed: {}", ep);
general_disk_error();
});
}
@@ -1776,9 +1788,11 @@ sstable::shared_remove_by_toc_name(sstring toc_name, bool shared) {
future<>
fsync_directory(sstring fname) {
return open_directory(dirname(fname)).then([] (file f) {
return do_with(std::move(f), [] (file& f) {
return f.flush();
return sstable_write_io_check([&] {
return open_checked_directory(sstable_write_error ,dirname(fname)).then([] (file f) {
return do_with(std::move(f), [] (file& f) {
return f.flush();
});
});
});
}
@@ -1794,8 +1808,8 @@ remove_by_toc_name(sstring sstable_toc_name) {
in.close().get();
sstring prefix = sstable_toc_name.substr(0, sstable_toc_name.size() - TOC_SUFFIX.size());
auto new_toc_name = prefix + TEMPORARY_TOC_SUFFIX;
rename_file(sstable_toc_name, new_toc_name).get();
fsync_directory(dir).get();
sstable_write_io_check(rename_file, sstable_toc_name, new_toc_name).get();
sstable_write_io_check(fsync_directory, dir).get();
std::vector<sstring> components;
sstring all(text.begin(), text.end());
boost::split(components, all, boost::is_any_of("\n"));
@@ -1808,21 +1822,21 @@ remove_by_toc_name(sstring sstable_toc_name) {
// already deleted
return make_ready_future<>();
}
return remove_file(prefix + component);
return sstable_write_io_check(remove_file, prefix + component);
}).get();
fsync_directory(dir).get();
remove_file(new_toc_name).get();
sstable_write_io_check(fsync_directory, dir).get();
sstable_write_io_check(remove_file, new_toc_name).get();
});
}
future<>
sstable::remove_sstable_with_temp_toc(sstring ks, sstring cf, sstring dir, int64_t generation, version_types v, format_types f) {
return seastar::async([ks, cf, dir, generation, v, f] {
auto toc = file_exists(filename(dir, ks, cf, v, generation, f, component_type::TOC)).get0();
auto toc = sstable_write_io_check(file_exists, filename(dir, ks, cf, v, generation, f, component_type::TOC)).get0();
// assert that toc doesn't exist for sstable with temporary toc.
assert(toc == false);
auto tmptoc = file_exists(filename(dir, ks, cf, v, generation, f, component_type::TemporaryTOC)).get0();
auto tmptoc = sstable_write_io_check(file_exists, filename(dir, ks, cf, v, generation, f, component_type::TemporaryTOC)).get0();
// assert that temporary toc exists for this sstable.
assert(tmptoc == true);
@@ -1842,17 +1856,17 @@ sstable::remove_sstable_with_temp_toc(sstring ks, sstring cf, sstring dir, int64
auto file_path = filename(dir, ks, cf, v, generation, f, entry.first);
// Skip component that doesn't exist.
auto exists = file_exists(file_path).get0();
auto exists = sstable_write_io_check(file_exists, file_path).get0();
if (!exists) {
continue;
}
remove_file(file_path).get();
sstable_write_io_check(remove_file, file_path).get();
}
fsync_directory(dir).get();
sstable_write_io_check(fsync_directory, dir).get();
// Removing temporary
remove_file(filename(dir, ks, cf, v, generation, f, component_type::TemporaryTOC)).get();
sstable_write_io_check(remove_file, filename(dir, ks, cf, v, generation, f, component_type::TemporaryTOC)).get();
// Fsync'ing column family dir to guarantee that deletion completed.
fsync_directory(dir).get();
sstable_write_io_check(fsync_directory, dir).get();
});
}