bahman/scylla
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

keys: Make from_exploded() and components() work without schema

Browse Source 
For simplicity, we want to have keys serializable and deserializable
without schema for now. We will serialize keys in a generic form of a
vector of components where the format of components is specified by
CQL binary protocol. So conversion between keys and vector of
components needs to be possible to do without schema.

We may want to make keys schema-dependent back in the future to apply
space optimizations specific to column types. Existing code should
still pass schema& to construct and access the key when possible.

One optimization had to be reverted in this change - avoidance of
storing key length (2 bytes) for single-component partition keys. One
consequence of this, in addition to a bit larger keys, is that we can
no longer avoid copy when constructing single-component partition keys
from a ready "bytes" object.

I haven't noticed any significant performance difference in:

  tests/perf/perf_simple_query -c1 --write

It does ~130K tps on my machine.
This commit is contained in:
Tomasz Grabiec
2016-02-09 13:42:49 +01:00
parent 31312722d1
commit 095efd01d6
3 changed files with 68 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
compound.hh
View File

@@ -68,7 +68,7 @@ public:
, _byte_order_equal(std::all_of(_types.begin(), _types.end(), [] (auto t) {
return t->is_byte_order_equal();
}))
, _byte_order_comparable(!is_prefixable && _types.size() == 1 && _types[0]->is_byte_order_comparable())
, _byte_order_comparable(false)
, _is_reversed(_types.size() == 1 && _types[0]->is_reversed())
{ }
@@ -88,51 +88,30 @@ public:
/*
* Format:
* <len(value1)><value1><len(value2)><value2>...<len(value_n-1)><value_n-1>(len(value_n))?<value_n>
* <len(value1)><value1><len(value2)><value2>...<len(value_n)><value_n>
*
* For non-prefixable compounds, the value corresponding to the last component of types doesn't
* have its length encoded, its length is deduced from the input range.
*
* serialize_value() and serialize_optionals() for single element rely on the fact that for a single-element
* compounds their serialized form is equal to the serialized form of the component.
*/
template<typename Wrapped>
void serialize_value(const std::vector<Wrapped>& values, bytes::iterator& out) {
if (AllowPrefixes == allow_prefixes::yes) {
assert(values.size() <= _types.size());
} else {
assert(values.size() == _types.size());
}
size_t n_left = _types.size();
static void serialize_value(const std::vector<Wrapped>& values, bytes::iterator& out) {
for (auto&& wrapped : values) {
auto&& val = value_traits<Wrapped>::unwrap(wrapped);
assert(val.size() <= std::numeric_limits<uint16_t>::max());
if (--n_left || AllowPrefixes == allow_prefixes::yes) {
write<uint16_t>(out, uint16_t(val.size()));
}
write<uint16_t>(out, uint16_t(val.size()));
out = std::copy(val.begin(), val.end(), out);
}
}
template <typename Wrapped>
size_t serialized_size(const std::vector<Wrapped>& values) {
static size_t serialized_size(const std::vector<Wrapped>& values) {
size_t len = 0;
size_t n_left = _types.size();
for (auto&& wrapped : values) {
auto&& val = value_traits<Wrapped>::unwrap(wrapped);
assert(val.size() <= std::numeric_limits<uint16_t>::max());
if (--n_left || AllowPrefixes == allow_prefixes::yes) {
len += sizeof(uint16_t);
}
len += val.size();
len += sizeof(uint16_t) + val.size();
}
return len;
}
bytes serialize_single(bytes&& v) {
if (AllowPrefixes == allow_prefixes::no) {
assert(_types.size() == 1);
return std::move(v);
} else {
{
// FIXME: Optimize
std::vector<bytes> vec;
vec.reserve(1);
@@ -140,24 +119,15 @@ public:
return ::serialize_value(*this, vec);
}
}
bytes serialize_value(const std::vector<bytes>& values) {
return ::serialize_value(*this, values);
}
bytes serialize_value(std::vector<bytes>&& values) {
if (AllowPrefixes == allow_prefixes::no && _types.size() == 1 && values.size() == 1) {
return std::move(values[0]);
}
return ::serialize_value(*this, values);
template<typename RangeOfComponents>
static bytes serialize_value(const RangeOfComponents& values) {
bytes b(bytes::initialized_later(), serialized_size(values));
auto i = b.begin();
serialize_value(values, i);
return b;
}
bytes serialize_optionals(const std::vector<bytes_opt>& values) {
return ::serialize_value(*this, values);
}
bytes serialize_optionals(std::vector<bytes_opt>&& values) {
if (AllowPrefixes == allow_prefixes::no && _types.size() == 1 && values.size() == 1) {
assert(values[0]);
return std::move(*values[0]);
}
return ::serialize_value(*this, values);
return serialize_value(values);
}
bytes serialize_value_deep(const std::vector<data_value>& values) {
// TODO: Optimize
@@ -171,35 +141,19 @@ public:
return serialize_value(partial);
}
bytes decompose_value(const value_type& values) {
return ::serialize_value(*this, values);
return serialize_value(values);
}
class iterator : public std::iterator<std::input_iterator_tag, bytes_view> {
private:
ssize_t _types_left;
bytes_view _v;
value_type _current;
private:
void read_current() {
if (_types_left == 0) {
if (!_v.empty()) {
throw marshal_exception();
}
_v = bytes_view(nullptr, 0);
return;
}
--_types_left;
uint16_t len;
if (_types_left == 0 && AllowPrefixes == allow_prefixes::no) {
len = _v.size();
} else {
{
if (_v.empty()) {
if (AllowPrefixes == allow_prefixes::yes) {
_types_left = 0;
_v = bytes_view(nullptr, 0);
return;
} else {
throw marshal_exception();
}
_v = bytes_view(nullptr, 0);
return;
}
len = read_simple<uint16_t>(_v);
if (_v.size() < len) {
@@ -211,10 +165,10 @@ public:
}
public:
struct end_iterator_tag {};
iterator(const compound_type& t, const bytes_view& v) : _types_left(t._types.size()), _v(v) {
iterator(const bytes_view& v) : _v(v) {
read_current();
}
iterator(end_iterator_tag, const bytes_view& v) : _types_left(0), _v(nullptr, 0) {}
iterator(end_iterator_tag, const bytes_view& v) : _v(nullptr, 0) {}
iterator& operator++() {
read_current();
return *this;
@@ -226,16 +180,16 @@ public:
}
const value_type& operator*() const { return _current; }
const value_type* operator->() const { return &_current; }
bool operator!=(const iterator& i) const { return _v.begin() != i._v.begin() || _types_left != i._types_left; }
bool operator==(const iterator& i) const { return _v.begin() == i._v.begin() && _types_left == i._types_left; }
bool operator!=(const iterator& i) const { return _v.begin() != i._v.begin(); }
bool operator==(const iterator& i) const { return _v.begin() == i._v.begin(); }
};
iterator begin(const bytes_view& v) const {
return iterator(*this, v);
static iterator begin(const bytes_view& v) {
return iterator(v);
}
iterator end(const bytes_view& v) const {
static iterator end(const bytes_view& v) {
return iterator(typename iterator::end_iterator_tag(), v);
}
boost::iterator_range<iterator> components(const bytes_view& v) const {
static boost::iterator_range<iterator> components(const bytes_view& v) {
return { begin(v), end(v) };
}
value_type deserialize_value(bytes_view v) {

48
keys.hh
View File

@@ -107,15 +107,26 @@ public:
return get_compound_type(s)->equal(representation(), other.representation());
}
// begin() and end() return iterators over components of this compound. The iterator yields a bytes_view to the component.
// The iterators satisfy InputIterator concept.
auto begin() const {
return TopLevelView::compound::element_type::begin(representation());
}
// See begin()
auto end() const {
return TopLevelView::compound::element_type::end(representation());
}
// begin() and end() return iterators over components of this compound. The iterator yields a bytes_view to the component.
// The iterators satisfy InputIterator concept.
auto begin(const schema& s) const {
return get_compound_type(s)->begin(representation());
return begin();
}
// See begin()
auto end(const schema& s) const {
return get_compound_type(s)->end(representation());
return end();
}
bytes_view get_component(const schema& s, size_t idx) const {
@@ -124,9 +135,14 @@ public:
return *it;
}
// Returns a range of bytes_view
auto components() const {
return TopLevelView::compound::element_type::components(representation());
}
// Returns a range of bytes_view
auto components(const schema& s) const {
return boost::make_iterator_range(begin(s), end(s));
return components();
}
template<typename Hasher>
@@ -152,12 +168,12 @@ public:
return from_exploded(s, {});
}
static TopLevel from_exploded(const schema& s, const std::vector<bytes>& v) {
return TopLevel::from_bytes(get_compound_type(s)->serialize_value(v));
static TopLevel from_exploded(const std::vector<bytes>& v) {
return TopLevel::from_bytes(TopLevel::compound::element_type::serialize_value(v));
}
static TopLevel from_exploded(const schema& s, std::vector<bytes>&& v) {
return TopLevel::from_bytes(get_compound_type(s)->serialize_value(std::move(v)));
static TopLevel from_exploded(const schema& s, const std::vector<bytes>& v) {
return from_exploded(v);
}
// We don't allow optional values, but provide this method as an efficient adaptor
@@ -197,6 +213,14 @@ public:
return get_compound_type(s)->deserialize_value(_bytes);
}
std::vector<bytes> explode() const {
std::vector<bytes> result;
for (bytes_view c : components()) {
result.emplace_back(to_bytes(c));
}
return result;
}
struct less_compare {
typename TopLevel::compound _t;
less_compare(const schema& s) : _t(get_compound_type(s)) {}
@@ -260,6 +284,16 @@ public:
return get_compound_type(s)->end(_bytes);
}
// Returns a range of bytes_view
auto components() const {
return TopLevelView::compound::element_type::components(representation());
}
// Returns a range of bytes_view
auto components(const schema& s) const {
return components();
}
bytes_view get_component(const schema& s, size_t idx) const {
auto it = begin(s);
std::advance(it, idx);

2
tests/sstable_mutation_test.cc
View File

@@ -311,7 +311,7 @@ future<> test_range_reads(const dht::token& min, const dht::token& max, std::vec
return mutations.read().then([&expected, expected_size, count, stop] (mutation_opt mutation) mutable {
if (mutation) {
BOOST_REQUIRE(*count < expected_size);
BOOST_REQUIRE(bytes_view(expected.back()) == bytes_view(mutation->key()));
BOOST_REQUIRE(std::vector<bytes>({expected.back()}) == mutation->key().explode());
expected.pop_back();
(*count)++;
} else {