Avoid including cryptopp headers

cryptopp's config.h has the following pragma:

 #pragma GCC diagnostic ignored "-Wunused-function"

It is not wrapped in a push/pop. Because of that, including cryptopp
headers disables that warning on scylla code too.

The issue has been reported as
https://github.com/weidai11/cryptopp/issues/793

To work around it, this patch uses a pimpl to have a single .cc file
that has to include cryptopp headers.

While at it, it also reduces the differences and code duplication
between the md5 and sha1 hashers.

Signed-off-by: Rafael Ávila de Espíndola <espindola@scylladb.com>

configure.py

@@ -461,6 +461,7 @@ cassandra_interface = Thrift(source='interface/cassandra.thrift', service='Cassa
 scylla_core = (['database.cc',
                 'table.cc',
                 'atomic_cell.cc',
+                'hashers.cc',
                 'schema.cc',
                 'frozen_schema.cc',
                 'schema_registry.cc',
@@ -878,7 +879,7 @@ deps['tests/mutation_reader_test'] += ['tests/sstable_utils.cc']
 
 deps['tests/bytes_ostream_test'] = ['tests/bytes_ostream_test.cc', 'utils/managed_bytes.cc', 'utils/logalloc.cc', 'utils/dynamic_bitset.cc']
 deps['tests/input_stream_test'] = ['tests/input_stream_test.cc']
-deps['tests/UUID_test'] = ['utils/UUID_gen.cc', 'tests/UUID_test.cc', 'utils/uuid.cc', 'utils/managed_bytes.cc', 'utils/logalloc.cc', 'utils/dynamic_bitset.cc']
+deps['tests/UUID_test'] = ['utils/UUID_gen.cc', 'tests/UUID_test.cc', 'utils/uuid.cc', 'utils/managed_bytes.cc', 'utils/logalloc.cc', 'utils/dynamic_bitset.cc', 'hashers.cc']
 deps['tests/murmur_hash_test'] = ['bytes.cc', 'utils/murmur_hash.cc', 'tests/murmur_hash_test.cc']
 deps['tests/allocation_strategy_test'] = ['tests/allocation_strategy_test.cc', 'utils/logalloc.cc', 'utils/dynamic_bitset.cc']
 deps['tests/log_heap_test'] = ['tests/log_heap_test.cc']

cql3/query_processor.cc

@@ -39,11 +39,8 @@
  * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#define CRYPTOPP_ENABLE_NAMESPACE_WEAK 1
-
 #include "cql3/query_processor.hh"
 
-#include <cryptopp/md5.h>
 #include <seastar/core/metrics.hh>
 
 #include "cql3/CqlParser.hpp"
@@ -52,6 +49,7 @@
 #include "cql3/util.hh"
 #include "db/config.hh"
 #include "database.hh"
+#include "hashers.hh"
 
 namespace cql3 {
 
@@ -383,14 +381,6 @@ query_processor::get_stored_prepared_statement(
     }
 }
 
-static bytes md5_calculate(const std::string_view& s) {
-    constexpr size_t size = CryptoPP::Weak1::MD5::DIGESTSIZE;
-    CryptoPP::Weak::MD5 hash;
-    unsigned char digest[size];
-    hash.CalculateDigest(digest, reinterpret_cast<const unsigned char*>(s.data()), s.size());
-    return std::move(bytes{reinterpret_cast<const int8_t*>(digest), size});
-}
-
 static sstring hash_target(const std::string_view& query_string, const sstring& keyspace) {
     return keyspace + std::string(query_string);
 }
@@ -398,7 +388,7 @@ static sstring hash_target(const std::string_view& query_string, const sstring&
 prepared_cache_key_type query_processor::compute_id(
         const std::string_view& query_string,
         const sstring& keyspace) {
-    return prepared_cache_key_type(md5_calculate(hash_target(query_string, keyspace)));
+    return prepared_cache_key_type(md5_hasher::calculate(hash_target(query_string, keyspace)));
 }
 
 prepared_cache_key_type query_processor::compute_thrift_id(

db/schema_tables.cc

@@ -63,7 +63,7 @@
 #include "db/marshal/type_parser.hh"
 #include "db/config.hh"
 #include "db/extensions.hh"
-#include "md5_hasher.hh"
+#include "hashers.hh"
 
 #include <seastar/util/noncopyable_function.hh>
 

db/system_keyspace.cc

@@ -64,7 +64,7 @@
 #include "db/config.hh"
 #include "system_keyspace_view_types.hh"
 #include "schema_builder.hh"
-#include "md5_hasher.hh"
+#include "hashers.hh"
 #include "release.hh"
 #include "log.hh"
 #include "serializer.hh"

dht/random_partitioner.cc

@@ -19,7 +19,7 @@
  * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "md5_hasher.hh"
+#include "hashers.hh"
 #include "random_partitioner.hh"
 #include "utils/class_registrator.hh"
 #include "utils/div_ceil.hh"

digester.hh

@@ -22,7 +22,7 @@
 #pragma once
 
 #include "digest_algorithm.hh"
-#include "md5_hasher.hh"
+#include "hashers.hh"
 #include "xx_hasher.hh"
 
 #include <type_traits>

hashers.cc

@@ -0,0 +1,87 @@
+/*
+ * Copyright (C) 2019 ScyllaDB
+ */
+
+/*
+ * This file is part of Scylla.
+ *
+ * Scylla is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Scylla is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "hashers.hh"
+
+#define CRYPTOPP_ENABLE_NAMESPACE_WEAK 1
+#include <cryptopp/md5.h>
+#include <cryptopp/sha.h>
+
+template <typename T> struct hasher_traits;
+template <> struct hasher_traits<md5_hasher> { using impl_type = CryptoPP::Weak::MD5; };
+template <> struct hasher_traits<sha256_hasher> { using impl_type = CryptoPP::SHA256; };
+
+template <typename T, size_t size> struct hasher<T, size>::impl {
+    using impl_type = typename hasher_traits<T>::impl_type;
+
+    impl_type hash{};
+
+    void update(const char* ptr, size_t length) {
+        using namespace CryptoPP;
+        static_assert(sizeof(char) == sizeof(byte), "Assuming lengths will be the same");
+        hash.Update(reinterpret_cast<const byte*>(ptr), length * sizeof(byte));
+    }
+
+    bytes finalize() {
+        bytes digest{bytes::initialized_later(), size};
+        hash.Final(reinterpret_cast<unsigned char*>(digest.begin()));
+        return digest;
+    }
+
+    std::array<uint8_t, size> finalize_array() {
+        std::array<uint8_t, size> array;
+        hash.Final(reinterpret_cast<unsigned char*>(array.data()));
+        return array;
+    }
+};
+
+template <typename T, size_t size> hasher<T, size>::hasher() : _impl(std::make_unique<impl>()) {}
+
+template <typename T, size_t size> hasher<T, size>::~hasher() = default;
+
+template <typename T, size_t size> hasher<T, size>::hasher(hasher&& o) noexcept = default;
+
+template <typename T, size_t size> hasher<T, size>::hasher(const hasher& o) : _impl(std::make_unique<hasher<T, size>::impl>(*o._impl)) {}
+
+template <typename T, size_t size> hasher<T, size>& hasher<T, size>::operator=(hasher&& o) noexcept = default;
+
+template <typename T, size_t size> hasher<T, size>& hasher<T, size>::operator=(const hasher& o) {
+    _impl = std::make_unique<hasher<T, size>::impl>(*o._impl);
+    return *this;
+}
+
+template <typename T, size_t size> bytes hasher<T, size>::finalize() { return _impl->finalize(); }
+
+template <typename T, size_t size> std::array<uint8_t, size> hasher<T, size>::finalize_array() {
+    return _impl->finalize_array();
+}
+
+template <typename T, size_t size> void hasher<T, size>::update(const char* ptr, size_t length) { _impl->update(ptr, length); }
+
+template <typename T, size_t size> bytes hasher<T, size>::calculate(const std::string_view& s) {
+    typename hasher<T, size>::impl::impl_type hash;
+    unsigned char digest[size];
+    hash.CalculateDigest(digest, reinterpret_cast<const unsigned char*>(s.data()), s.size());
+    return std::move(bytes{reinterpret_cast<const int8_t*>(digest), size});
+}
+
+template class hasher<md5_hasher, 16>;
+template class hasher<sha256_hasher, 32>;

hashers.hh

@@ -0,0 +1,50 @@
+/*
+ * Copyright (C) 2019 ScyllaDB
+ */
+
+/*
+ * This file is part of Scylla.
+ *
+ * Scylla is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Scylla is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "bytes.hh"
+
+class md5_hasher;
+
+template <typename T, size_t size> class hasher {
+    struct impl;
+    std::unique_ptr<impl> _impl;
+
+public:
+    hasher();
+    ~hasher();
+    hasher(hasher&&) noexcept;
+    hasher(const hasher&);
+    hasher& operator=(hasher&&) noexcept;
+    hasher& operator=(const hasher&);
+
+    bytes finalize();
+    std::array<uint8_t, size> finalize_array();
+    void update(const char* ptr, size_t length);
+
+    // Use update and finalize to compute the hash over the full view.
+    static bytes calculate(const std::string_view& s);
+};
+
+class md5_hasher : public hasher<md5_hasher, 16> {};
+
+class sha256_hasher : public hasher<sha256_hasher, 32> {};

md5_hasher.hh

@@ -1,52 +0,0 @@
-/*
- * Copyright (C) 2015 ScyllaDB
- */
-
-/*
- * This file is part of Scylla.
- *
- * Scylla is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * Scylla is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#pragma once
-
-#define CRYPTOPP_ENABLE_NAMESPACE_WEAK 1
-
-#include <cryptopp/md5.h>
-#include "hashing.hh"
-#include "bytes.hh"
-
-class md5_hasher {
-    CryptoPP::Weak::MD5 hash{};
-public:
-    static constexpr size_t size = CryptoPP::Weak::MD5::DIGESTSIZE;
-
-    void update(const char* ptr, size_t length) {
-        using namespace CryptoPP;
-        static_assert(sizeof(char) == sizeof(byte), "Assuming lengths will be the same");
-        hash.Update(reinterpret_cast<const byte*>(ptr), length * sizeof(byte));
-    }
-
-    bytes finalize() {
-        bytes digest{bytes::initialized_later(), size};
-        hash.Final(reinterpret_cast<unsigned char*>(digest.begin()));
-        return digest;
-    }
-
-    std::array<uint8_t, size> finalize_array() {
-        std::array<uint8_t, size> array;
-        hash.Final(reinterpret_cast<unsigned char*>(array.data()));
-        return array;
-    }
-};

repair/repair.cc

@@ -33,6 +33,7 @@
 #include "message/messaging_service.hh"
 #include "sstables/sstables.hh"
 #include "database.hh"
+#include "hashers.hh"
 
 #include <boost/algorithm/string/predicate.hpp>
 #include <boost/algorithm/string/split.hpp>
@@ -42,7 +43,6 @@
 #include <boost/range/algorithm_ext.hpp>
 #include <boost/range/adaptor/map.hpp>
 
-#include <cryptopp/sha.h>
 #include <seastar/core/gate.hh>
 #include <seastar/util/defer.hh>
 
@@ -325,24 +325,6 @@ static void check_in_shutdown() {
     repair_tracker.check_in_shutdown();
 }
 
-class sha256_hasher {
-    CryptoPP::SHA256 hash{};
-public:
-    void update(const char* ptr, size_t length) {
-        // In Crypto++ v6, the `byte` typedef has been moved to CryptoPP:: namespace
-        // We bring the namespace in to make the same code work for both 5.x and 6.x versions
-        using namespace CryptoPP;
-        static_assert(sizeof(char) == sizeof(byte), "Assuming lengths will be the same");
-        hash.Update(reinterpret_cast<const byte*>(ptr), length * sizeof(byte));
-    }
-
-    void finalize(std::array<uint8_t, 32>& digest) {
-        static_assert(CryptoPP::SHA256::DIGESTSIZE == std::tuple_size<std::remove_reference_t<decltype(digest)>>::value * sizeof(digest[0]),
-                "digest size");
-        hash.Final(reinterpret_cast<unsigned char*>(digest.data()));
-    }
-};
-
 class partition_hasher {
     const schema& _schema;
     sha256_hasher _hasher;
@@ -451,8 +433,7 @@ public:
     stop_iteration consume_end_of_partition() {
         consume_range_tombstones_until_end();
 
-        std::array<uint8_t, 32> digest;
-        _hasher.finalize(digest);
+        std::array<uint8_t, 32> digest = _hasher.finalize_array();
         _hasher = { };
 
         _checksum.add(partition_checksum(digest));
@@ -474,10 +455,9 @@ future<partition_checksum> partition_checksum::compute_legacy(flat_mutation_read
                 if (!mopt) {
                     return stop_iteration::yes;
                 }
-                std::array<uint8_t, 32> digest;
                 sha256_hasher h;
                 feed_hash(h, *mopt);
-                h.finalize(digest);
+                std::array<uint8_t, 32> digest = h.finalize_array();
                 checksum.add(partition_checksum(digest));
                 return stop_iteration::no;
             });

schema_mutations.cc

@@ -22,7 +22,7 @@
 #include "schema_mutations.hh"
 #include "canonical_mutation.hh"
 #include "db/schema_tables.hh"
-#include "md5_hasher.hh"
+#include "hashers.hh"
 
 schema_mutations::schema_mutations(canonical_mutation columnfamilies,
                                    canonical_mutation columns,

tests/mutation_test.cc

@@ -25,7 +25,7 @@
 #include <boost/range/algorithm/copy.hpp>
 #include <boost/range/algorithm_ext/push_back.hpp>
 #include "mutation_query.hh"
-#include "md5_hasher.hh"
+#include "hashers.hh"
 #include "xx_hasher.hh"
 
 #include <seastar/core/sstring.hh>

utils/UUID_gen.cc

@@ -40,9 +40,7 @@
 
 #include <stdlib.h>
 #include <atomic>
-
-#define CRYPTOPP_ENABLE_NAMESPACE_WEAK 1
-#include <cryptopp/md5.h>
+#include "hashers.hh"
 
 namespace utils {
 
@@ -85,12 +83,7 @@ UUID UUID_gen::get_name_UUID(sstring_view s) {
 }
 
 UUID UUID_gen::get_name_UUID(const unsigned char *s, size_t len) {
-    static_assert(CryptoPP::Weak1::MD5::DIGESTSIZE == 16, "MD5 digests should be 16 bytes long");
-    int8_t digest[16];
-
-    CryptoPP::Weak::MD5 hash;
-    static_assert(sizeof(char) == sizeof(int8_t), "Assumed that chars are bytes");
-    hash.CalculateDigest(reinterpret_cast<unsigned char*>(digest), s, len);
+    bytes digest = md5_hasher::calculate(std::string_view(reinterpret_cast<const char*>(s), len));
 
     // set version to 3
     digest[6] &= 0x0f;