bahman/scylla
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

make timestamp string format cassandra compatible

Browse Source 
when we convert timestamp into string it must look like: '2017-12-27T11:57:42.500Z'
it concerns any conversion except JSON timestamp format
JSON string has space as time separator and must look like: '2017-12-27 11:57:42.500Z'
both formats always contain milliseconds and timezone specification

Fixes #14518
Fixes #7997

Closes #14726
This commit is contained in:
Alexey Novikov
2023-07-17 15:17:28 +03:00
committed by Botond Dénes
parent 1b7bde2e9e
commit ff721ec3e3
9 changed files with 47 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
concrete_types.hh
View File

@@ -118,6 +118,8 @@ struct date_type_impl final : public concrete_type<db_clock::time_point> {
using timestamp_date_base_class = concrete_type<db_clock::time_point>;
sstring timestamp_to_json_string(const timestamp_date_base_class& t, const bytes_view& bv);
struct timeuuid_type_impl final : public concrete_type<utils::UUID> {
timeuuid_type_impl();
static utils::UUID from_sstring(sstring_view s);

2
cql3/type_json.cc
View File

@@ -489,7 +489,7 @@ struct to_json_string_visitor {
sstring operator()(const string_type_impl& t) { return quote_json_string(t.to_string(bv)); }
sstring operator()(const bytes_type_impl& t) { return quote_json_string("0x" + t.to_string(bv)); }
sstring operator()(const boolean_type_impl& t) { return t.to_string(bv); }
sstring operator()(const timestamp_date_base_class& t) { return quote_json_string(t.to_string(bv)); }
sstring operator()(const timestamp_date_base_class& t) { return quote_json_string(timestamp_to_json_string(t, bv)); }
sstring operator()(const timeuuid_type_impl& t) { return quote_json_string(t.to_string(bv)); }
sstring operator()(const map_type_impl& t) { return to_json_string_aux(t, bv); }
sstring operator()(const set_type_impl& t) { return to_json_string_aux(t, bv); }

12
test/boost/castas_fcts_test.cc
View File

@@ -494,29 +494,29 @@ SEASTAR_TEST_CASE(test_time_casts_in_selection_clause) {
}
{
auto msg = e.execute_cql("SELECT CAST(CAST(a AS timestamp) AS text), CAST(CAST(a AS date) AS text), CAST(CAST(b as date) AS text), CAST(CAST(c AS timestamp) AS text) FROM test").get0();
assert_that(msg).is_rows().with_size(1).with_row({{utf8_type->from_string("2009-12-17T00:26:29.805000")},
assert_that(msg).is_rows().with_size(1).with_row({{utf8_type->from_string("2009-12-17T00:26:29.805Z")},
{utf8_type->from_string("2009-12-17")},
{utf8_type->from_string("2015-05-21")},
{utf8_type->from_string("2015-05-21T00:00:00")}});
{utf8_type->from_string("2015-05-21T00:00:00.000Z")}});
}
{
auto msg = e.execute_cql("SELECT CAST(a AS text), CAST(b as text), CAST(c AS text), CAST(d AS text) FROM test").get0();
assert_that(msg).is_rows().with_size(1).with_row({{utf8_type->from_string("d2177dd0-eaa2-11de-a572-001b779c76e3")},
{utf8_type->from_string("2015-05-21T11:03:02")},
{utf8_type->from_string("2015-05-21T11:03:02.000Z")},
{utf8_type->from_string("2015-05-21")},
{utf8_type->from_string("11:03:02.000000000")}});
}
{
auto msg = e.execute_cql("SELECT CAST(CAST(a AS timestamp) AS ascii), CAST(CAST(a AS date) AS ascii), CAST(CAST(b as date) AS ascii), CAST(CAST(c AS timestamp) AS ascii) FROM test").get0();
assert_that(msg).is_rows().with_size(1).with_row({{ascii_type->from_string("2009-12-17T00:26:29.805000")},
assert_that(msg).is_rows().with_size(1).with_row({{ascii_type->from_string("2009-12-17T00:26:29.805Z")},
{ascii_type->from_string("2009-12-17")},
{ascii_type->from_string("2015-05-21")},
{ascii_type->from_string("2015-05-21T00:00:00")}});
{ascii_type->from_string("2015-05-21T00:00:00.000Z")}});
}
{
auto msg = e.execute_cql("SELECT CAST(a AS ascii), CAST(b as ascii), CAST(c AS ascii), CAST(d AS ascii) FROM test").get0();
assert_that(msg).is_rows().with_size(1).with_row({{ascii_type->from_string("d2177dd0-eaa2-11de-a572-001b779c76e3")},
{ascii_type->from_string("2015-05-21T11:03:02")},
{ascii_type->from_string("2015-05-21T11:03:02.000Z")},
{ascii_type->from_string("2015-05-21")},
{ascii_type->from_string("11:03:02.000000000")}});
}

4
test/boost/expr_test.cc
View File

@@ -163,7 +163,7 @@ BOOST_AUTO_TEST_CASE(expr_printer_timestamp_test) {
raw_value::make_value(timestamp_type->from_string("2011-03-02T03:05:00+0000")),
timestamp_type
);
BOOST_REQUIRE_EQUAL(expr_print(timestamp_const), "'2011-03-02T03:05:00+0000'");
BOOST_REQUIRE_EQUAL(expr_print(timestamp_const), "'2011-03-02T03:05:00.000Z'");
}
BOOST_AUTO_TEST_CASE(expr_printer_time_test) {
@@ -179,7 +179,7 @@ BOOST_AUTO_TEST_CASE(expr_printer_date_test) {
raw_value::make_value(date_type->from_string("2011-02-03+0000")),
date_type
};
BOOST_REQUIRE_EQUAL(expr_print(date_const), "'2011-02-03T00:00:00+0000'");
BOOST_REQUIRE_EQUAL(expr_print(date_const), "'2011-02-03T00:00:00.000Z'");
}
BOOST_AUTO_TEST_CASE(expr_printer_duration_test) {

4
test/boost/json_cql_query_test.cc
View File

@@ -95,7 +95,7 @@ SEASTAR_TEST_CASE(test_select_json_types) {
"\"\\\"G\\\"\": \"127.0.0.1\", " // note the double quoting on case-sensitive column names
"\"\\\"H\\\"\": 3, "
"\"\\\"I\\\"\": \"zażółć gęślą jaźń\", "
"\"j\": \"2001-10-18T14:15:55.134000\", "
"\"j\": \"2001-10-18 14:15:55.134Z\", "
"\"k\": \"d2177dd0-eaa2-11de-a572-001b779c76e3\", "
"\"l\": \"d2177dd0-eaa2-11de-a572-001b779c76e3\", "
"\"m\": \"varchar\", "
@@ -127,7 +127,7 @@ SEASTAR_TEST_CASE(test_select_json_types) {
utf8_type->decompose("\"127.0.0.1\""),
utf8_type->decompose("3"),
utf8_type->decompose("\"zażółć gęślą jaźń\""),
utf8_type->decompose("\"2001-10-18T14:15:55.134000\""),
utf8_type->decompose("\"2001-10-18 14:15:55.134Z\""),
utf8_type->decompose("\"d2177dd0-eaa2-11de-a572-001b779c76e3\""),
utf8_type->decompose("\"d2177dd0-eaa2-11de-a572-001b779c76e3\""),
utf8_type->decompose("\"varchar\""),

16
test/boost/types_test.cc
View File

@@ -278,27 +278,27 @@ void test_timestamp_like_string_conversions(data_type timestamp_type) {
BOOST_REQUIRE(timestamp_type->equal(timestamp_type->from_string("2015-07-03T12:30:00+1230"), timestamp_type->decompose(tp)));
BOOST_REQUIRE(timestamp_type->equal(timestamp_type->from_string("2015-07-02T23:00-0100"), timestamp_type->decompose(tp)));
BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "2015-07-03T00:00:00");
BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "2015-07-03T00:00:00.000Z");
// test fractional milliseconds
tp = db_clock::time_point(db_clock::duration(1435881600123));
BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "2015-07-03T00:00:00.123000");
BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "2015-07-03T00:00:00.123Z");
// test time_stamps around the unix epoch time
tp = db_clock::time_point(db_clock::duration(0));
BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "1970-01-01T00:00:00");
BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "1970-01-01T00:00:00.000Z");
tp = db_clock::time_point(db_clock::duration(456));
BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "1970-01-01T00:00:00.456000");
BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "1970-01-01T00:00:00.456Z");
tp = db_clock::time_point(db_clock::duration(-456));
BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "1969-12-31T23:59:59.544000");
BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "1969-12-31T23:59:59.544Z");
// test time_stamps around year 0
tp = db_clock::time_point(db_clock::duration(-62167219200000));
BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "0000-01-01T00:00:00");
BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "0000-01-01T00:00:00.000Z");
tp = db_clock::time_point(db_clock::duration(-62167219199211));
BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "0000-01-01T00:00:00.789000");
BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "0000-01-01T00:00:00.789Z");
tp = db_clock::time_point(db_clock::duration(-62167219200789));
BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "-0001-12-31T23:59:59.211000");
BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "-0001-12-31T23:59:59.211Z");
auto now = time(nullptr);
::tm local_now;

1
test/cql-pytest/cassandra_tests/functions/cast_fcts_test.py
View File

@@ -169,7 +169,6 @@ def testNoLossOfPrecisionForCastToDecimal(cql, test_keyspace):
assertRows(execute(cql, table, "SELECT CAST(bigint_clmn AS decimal), CAST(varint_clmn AS decimal) FROM %s"),
row(Decimal("9223372036854775807"), Decimal("1234567890123456789")))
@pytest.mark.xfail(reason="issue #14518")
def testTimeCastsInSelectionClause(cql, test_keyspace):
with create_table(cql, test_keyspace, "(a timeuuid primary key, b timestamp, c date, d time)") as table:
yearMonthDay = "2015-05-21"

11
test/cql-pytest/test_json.py
View File

@@ -18,6 +18,7 @@ from cassandra.protocol import FunctionFailure, InvalidRequest
import pytest
import json
from decimal import Decimal
from datetime import datetime
@pytest.fixture(scope="module")
def type1(cql, test_keyspace):
@@ -29,7 +30,7 @@ def type1(cql, test_keyspace):
@pytest.fixture(scope="module")
def table1(cql, test_keyspace, type1):
table = test_keyspace + "." + unique_name()
cql.execute(f"CREATE TABLE {table} (p int PRIMARY KEY, v int, bigv bigint, a ascii, b boolean, vi varint, mai map<ascii, int>, tup frozen<tuple<text, int>>, l list<text>, d double, t time, dec decimal, tupmap map<frozen<tuple<text, int>>, int>, t1 frozen<{type1}>, \"CaseSensitive\" int)")
cql.execute(f"CREATE TABLE {table} (p int PRIMARY KEY, v int, bigv bigint, a ascii, b boolean, vi varint, mai map<ascii, int>, tup frozen<tuple<text, int>>, l list<text>, d double, t time, dec decimal, tupmap map<frozen<tuple<text, int>>, int>, t1 frozen<{type1}>, \"CaseSensitive\" int, ts timestamp)")
yield table
cql.execute("DROP TABLE " + table)
@@ -310,6 +311,14 @@ def test_tojson_time(cql, table1):
cql.execute(stmt, [p, 123])
assert list(cql.execute(f"SELECT toJson(t) from {table1} where p = {p}")) == [('"00:00:00.000000123"',)]
# Check that toJson() returns timestamp string in correct cassandra compatible format (issue #7997)
# with milliseconds and timezone specification
def test_tojson_timestamp(cql, table1):
p = unique_key_int()
stmt = cql.prepare(f"INSERT INTO {table1} (p, ts) VALUES (?, ?)")
cql.execute(stmt, [p, datetime(2014, 1, 1, 12, 15, 45)])
assert list(cql.execute(f"SELECT toJson(ts) from {table1} where p = {p}")) == [('"2014-01-01 12:15:45.000Z"',)]
# The EquivalentJson class wraps a JSON string, and compare equal to other
# strings if both are valid JSON strings which decode to the same object.
# EquivalentJson("....") can be used in assert_rows() checks below, to check

35
types/types.cc
View File

@@ -68,7 +68,7 @@ requires requires {
requires std::same_as<typename T::duration, std::chrono::milliseconds>;
}
sstring
time_point_to_string(const T& tp)
time_point_to_string(const T& tp, bool use_time_separator = true)
{
auto count = tp.time_since_epoch().count();
auto d = std::div(int64_t(count), int64_t(1000));
@@ -78,17 +78,7 @@ time_point_to_string(const T& tp)
return fmt::format("{} milliseconds (out of range)", count);
}
auto to_string = [] (const std::tm& tm) {
auto year_digits = tm.tm_year >= -1900 ? 4 : 5;
return fmt::format("{:-0{}d}-{:02d}-{:02d}T{:02d}:{:02d}:{:02d}",
tm.tm_year + 1900, year_digits, tm.tm_mon + 1, tm.tm_mday,
tm.tm_hour, tm.tm_min, tm.tm_sec);
};
auto millis = d.rem;
if (!millis) {
return fmt::format("{}", to_string(tm));
}
// adjust seconds for time points earlier than posix epoch
// to keep the fractional millis positive
if (millis < 0) {
@@ -96,8 +86,13 @@ time_point_to_string(const T& tp)
seconds--;
gmtime_r(&seconds, &tm);
}
auto micros = millis * 1000;
return fmt::format("{}.{:06d}", to_string(tm), micros);
const auto time_separator = (use_time_separator) ? "T" : " ";
auto year_digits = tm.tm_year >= -1900 ? 4 : 5;
return fmt::format("{:-0{}d}-{:02d}-{:02d}{}{:02d}:{:02d}:{:02d}.{:03d}Z",
tm.tm_year + 1900, year_digits, tm.tm_mon + 1, tm.tm_mday, time_separator,
tm.tm_hour, tm.tm_min, tm.tm_sec, millis);
}
sstring simple_date_to_string(const uint32_t days_count) {
@@ -2853,6 +2848,12 @@ static sstring format_if_not_empty(
return f(static_cast<const N&>(*b));
}
sstring timestamp_to_json_string(const timestamp_date_base_class& t, const bytes_view& bv)
{
auto tp = value_cast<const timestamp_date_base_class::native_type>(t.deserialize(bv));
return format_if_not_empty(t, &tp, [](const db_clock::time_point& v) { return time_point_to_string(v, false); });
}
static sstring to_string_impl(const abstract_type& t, const void* v);
namespace {
@@ -3629,16 +3630,12 @@ sstring data_value::to_parsable_string() const {
abstract_type::kind type_kind = _type->without_reversed().get_kind();
if (type_kind == abstract_type::kind::date || type_kind == abstract_type::kind::timestamp) {
// Put timezone information after a date or timestamp to specify that it's in UTC
// Otherwise it will be parsed as a date in the local timezone.
return fmt::format("'{}+0000'", *this);
}
if (type_kind == abstract_type::kind::utf8
|| type_kind == abstract_type::kind::ascii
|| type_kind == abstract_type::kind::inet
|| type_kind == abstract_type::kind::time
|| type_kind == abstract_type::kind::date
|| type_kind == abstract_type::kind::timestamp
) {
// Put quotes on types that require it
return fmt::format("'{}'", *this);