streaming: add pytest case to reproduce mutation loss issue

Introduce a test that demonstrates mutation loss caused by premature loop termination in tablet_sstable_streamer::stream. The code broke out of the SSTable iteration when encountering a non-overlapping range, which skipped subsequent SSTables that should have been partially contained. This test showcases the problem only. Example: Tablet range: [4, 5] SSTable ranges: [0,5] [0, 3] <--- is considered exhausted, and causes skip to next tablet [2, 5] <--- is missed for range [4, 5] (cherry picked from commit 656ce27e7f)
2025-11-18 09:32:07 +02:00
parent 612c4321c8
commit fa58e526a0
4 changed files with 265 additions and 2 deletions
--- a/test/cluster/object_store/test_backup.py
+++ b/test/cluster/object_store/test_backup.py
@@ -1,17 +1,21 @@
 #!/usr/bin/env python3
-
+import json
 import os
 import logging
 import asyncio
+import subprocess
+import tempfile
+
 import pytest
 import time
 import random

 from test.pylib.minio_server import MinioServer
+from test.cqlpy.util import local_process_id
 from test.pylib.manager_client import ManagerClient
 from test.cluster.object_store.conftest import get_s3_resource, format_tuples
 from test.cluster.conftest import skip_mode
-from test.cluster.util import wait_for_cql_and_get_hosts
+from test.cluster.util import wait_for_cql_and_get_hosts, new_test_keyspace
 from concurrent.futures import ThreadPoolExecutor
 from test.pylib.rest_client import read_barrier
 from test.pylib.util import unique_name, wait_for_first_completed
@@ -947,3 +951,94 @@ async def test_restore_primary_replica_different_dc_scope_all(manager: ManagerCl
        streamed_to = set([ r[1].group(1) for r in res ])
        logger.info(f'{s.ip_addr} {host_ids[s.server_id]} streamed to {streamed_to}, expected {r_servers}')
        assert len(streamed_to) == 2
+
+@pytest.mark.asyncio
+@pytest.mark.xfail(reason="issue #26979")
+async def test_backup_broken_streaming(manager: ManagerClient, s3_server):
+    # Define configuration for the servers.
+    objconf = MinioServer.create_conf(s3_server.address, s3_server.port, s3_server.region)
+    config = {
+        'enable_user_defined_functions': False,
+        'object_storage_endpoints': objconf,
+        'experimental_features': ['keyspace-storage-options'],
+        'task_ttl_in_seconds': 300,
+    }
+    cmd = ['--smp', '1', '--logger-log-level', 'sstables_loader=debug:sstable=debug']
+    server = await manager.server_add(config=config, cmdline=cmd)
+
+    # Obtain the CQL interface from the manager.
+    cql = manager.get_cql()
+
+    pid = local_process_id(cql)
+    if not pid:
+        pytest.skip("Can't find local Scylla process")
+    # Now that we know the process id, use /proc to find the executable.
+    try:
+        scylla_path = os.readlink(f'/proc/{pid}/exe')
+    except:
+        pytest.skip("Can't find local Scylla executable")
+    # Confirm that this executable is a real tool-providing Scylla by trying
+    # to run it with the "--list-tools" option
+    try:
+        subprocess.check_output([scylla_path, '--list-tools'])
+    except:
+        pytest.skip("Local server isn't Scylla")
+
+    async with new_test_keyspace(manager,
+                                 "WITH replication = {'class': 'NetworkTopologyStrategy', 'replication_factor': 1}") as keyspace:
+        table = 'test_cf'
+        create_table_query = (
+            f"CREATE TABLE {keyspace}.{table} (name text PRIMARY KEY, value text) "
+            f"WITH tablets = {{'min_tablet_count': '16'}};"
+        )
+        cql.execute(create_table_query)
+
+        expected_rows = 0
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            resource_dir = "test/resource/sstables/fully_partially_contained_ssts"
+            schema_file = os.path.join(tmp_dir, "schema.cql")
+            with open(schema_file, "w") as f:
+                f.write(f"CREATE TABLE {keyspace}.{table} (name text PRIMARY KEY, value text)")
+                f.flush()
+            for root, _, files in os.walk(resource_dir):
+                for file in files:
+                    local_path = os.path.join(root, file)
+                    print("Processing file:", local_path)
+                    sst = subprocess.check_output(
+                        [scylla_path, "sstable", "write", "--schema-file", schema_file,
+                         "--output-dir", tmp_dir, "--input-file", local_path])
+                    expected_rows += json.loads(subprocess.check_output(
+                        [scylla_path, "sstable", "query", "-q", f"SELECT COUNT(*) FROM scylla_sstable.{table}",
+                         "--output-format", "json", "--sstables",
+                         os.path.join(tmp_dir, f"me-{sst.decode().strip()}-big-TOC.txt")]).decode())[0]['count']
+
+            prefix = unique_name('/test/streaming_')
+            s3_resource = get_s3_resource(s3_server)
+            bucket = s3_resource.Bucket(s3_server.bucket_name)
+            sstables = []
+
+            print(f"Uploading files from '{tmp_dir}' to prefix '{prefix}':")
+
+            for root, _, files in os.walk(tmp_dir):
+                for file in files:
+                    if file.endswith("-TOC.txt"):
+                        sstables.append(file)
+                    local_path = os.path.join(root, file)
+                    s3_key = f"{prefix}/{file}"
+
+                    print(f" - Uploading {local_path} to {s3_key}")
+                    bucket.upload_file(local_path, s3_key)
+
+        restore_task_id = await manager.api.restore(
+            server.ip_addr, keyspace, table,
+            s3_server.address, s3_server.bucket_name,
+            prefix, sstables, "node"
+        )
+
+        status = await manager.api.wait_task(server.ip_addr, restore_task_id)
+        assert status and status.get(
+            'state') == 'done', f"Restore task failed on server {server.server_id}. Reason {status}"
+
+        res = cql.execute(f"SELECT COUNT(*) FROM {keyspace}.{table} BYPASS CACHE USING TIMEOUT 600s;")
+
+        assert res[0].count == expected_rows, f"number of rows after restore is incorrect: {res[0].count}"
--- a/test/resource/sstables/fully_partially_contained_ssts/sst1.json
+++ b/test/resource/sstables/fully_partially_contained_ssts/sst1.json
@@ -0,0 +1,56 @@
+[
+  {
+    "key": {
+      "token": "-6916531810059846059",
+      "raw": "00873030343233335f3734633861633966316331346330383037353663316362353766633662326535623936303839333761366130393663316532353937346339353430373163313066653632656264303531326530303337333835666335343632333535653538393533323530653631353635613038633062373039356366326564323233643038",
+      "value": "004233_74c8ac9f1c14c080756c1cb57fc6b2e5b9608937a6a096c1e25974c954071c10fe62ebd0512e0037385fc5462355e58953250e61565a08c0b7095cf2ed223d08"
+    },
+    "clustering_elements": [
+      {
+        "type": "clustering-row",
+        "key": {
+          "raw": "",
+          "value": ""
+        },
+        "marker": {
+          "timestamp": 1762936589432341
+        },
+        "columns": {
+          "value": {
+            "is_live": true,
+            "type": "regular",
+            "timestamp": 1762936589432341,
+            "value": "67dd90bc58a8e5e4129670a23ca9fc66047ce5cf7acb510dfb91db998d661d6656e5915fe2d843e5ea122b3fe41aed113e6b32a0f6bfa09c3ada4c6d384afee7dba7c86187e135114f527a67c3464672d779557e25a93200a226de5105df6c049d784eca6f8889e5181fb9c95899ef50e3744edf398573f6c3e1ac88e1e37e8978b357b32e1b04b90805effb719367b3dbc98b671662f2f25af9c2766d244d8209485de8b405c49633a5ed54d170348bc0922227ff32ebcaa4ea893b81c4a6743d950a029d32d1e5662aed2e429eac65fe3e7872a27f5390f0b89edd40badc46e05c4f9be0803d5ccd9863bf472b2bc6e0839ea3b727146b552b39297f8b5298c1c9e41ad9a3269d9d8640c7b7372323801a017361f72f849cf062e913652775c05e354e8dc283e78a2d0b2a0406b54fbb83bdbfcb9d6701920e67ce386b38be0c6ade6bf75cd118478bf60732fd6ed868cea6c848535d554cdc43758058ed8af9892f90c522247803b567660abe8bfbb79c8185cf8cdc23d4cc8df19129af8f4f014e9cab8fa43131033c5c21fb64cd480e7a9d97a1aad44cb9cbd3a49347eb4a17f47ce734920acf0d7024383db9f774c7bcc0a50a32ed7546afff15e3af0ebfc6bf3add56bb11e3caccde1e6162930bbc9125d62c30995eee4b4a3709d7e5986eff4d85e5f6a6c411c58f3a5a7a168724b7883ff35dedcdd13a94c069a15e21d0b866cf539e772217eb58e63e70a3f81ccdc4f8d04a5163baaa374be20ca5bbeafc4fea805ecbd5776671f9a648f654d564754fc24791f88d1c2fb4441cf1c90a5f5422a374c77395322440b21db84219482c09a8f967157ebe94cbb2bf7f68dc59f02b1699871c7f5fcde1db0216b39f50958db9e0ff6694250dc0130f2ae94da518475c946d5588b7c47ccb6426da91b86207399d8dd2a810182cd960f6d8312b6664a48340f189a4be758d98702a9ee7fb003b43c1859ea9a5a7fff48a4c7eacafa704faeb091784db2f817b0b5511b249f0df1de42f54ac3947163a85fdf70b4a2b7593a5631e2ba2166ff09c025104987dad60d1397f9fc27bdca1b09eda7ff56893eeb6a99a98fc116bf813981ee828ae3c357a03ace4e26807a7bec56403d66a1b13645578cb1b7a364e37b2ab90a0f836a52714439f033760ec27a03ddbbf8eaa0c3d8c8d8dddf9e657bd0f8f7952b9352129ab534ee1ef54677c528732cacecac2a26dd39e38100b56b3fdbf04e0d108aad02696f8de4f4bf1fd424fd730983756833d6f705e089a1d3dd128a377bd1abfba30e01e055c2923d84b2344fd65e27accbe22bcb30bad39a6d6ddbc1b81d03c44e04343da4e08efad03368a75c970b7ecf3be0108177a929394df92e9a48852eeb10f75c60a0bf38304dbf5fe7244a50d8186092e83c913b0156366ab38e84b790d74ad5933544283"
+          }
+        }
+      }
+    ]
+  },
+  {
+    "key": {
+      "token": "-4616664894560819421",
+      "raw": "00873030363733345f6233363535626463636539366663656238653534393637653465646236356436346534303539616533393834333662366366666630656536653931303332633636656261353936663837636330636232363034333435373534666238653630393761353130653735626532626636326536613933366265343466333930323837",
+      "value": "006734_b3655bdcce96fceb8e54967e4edb65d64e4059ae398436b6cfff0ee6e91032c66eba596f87cc0cb2604345754fb8e6097a510e75be2bf62e6a936be44f390287"
+    },
+    "clustering_elements": [
+      {
+        "type": "clustering-row",
+        "key": {
+          "raw": "",
+          "value": ""
+        },
+        "marker": {
+          "timestamp": 1762936589659261
+        },
+        "columns": {
+          "value": {
+            "is_live": true,
+            "type": "regular",
+            "timestamp": 1762936589659261,
+            "value": "6ccc5dce2064c390202f095fbb96b64aca8ed238e1830d3a5b310722a7b374e34f6501ea50ee88dfc18c133a0b5c37824bfb21a21af1a79c013bb070cc8fe160899498ada00d99fa498177cb5a7631df6261f466b26ea25055339af3aa2b2080f8c090bfe94e1b51914fc4b0730ae30b5f9f4021e2edb3a2b2208fe7cf77559520bfb63eb1da539d5b1a54b0130fd8816b75d91945593ceae6c5c3fdec249e524d7fcdadb3d67151611eed9d1ce2c35fe92b2c490152897f2fa3dc141f9c62fbdf09a967748627312762915a48b578815dba0563b61d1717f1c7396c9729b92c7f37e808f37fc0210845954ca37fd362cd3c77dc5980ee22bdc6fef371f25f524d04d8619ce6fdeef3383abdbf5a26f5581785f93a9fc776ff521b5add7a187a92e6116db8b66315faa312dd0c5b754ba29a645ce8fae9dfc33b9bab77601f62e4b0d35d8be05949744cfbefabd95da85723ab3f925f91b14a4ed30d76c98d67492c3e0bfd3f8e9279845f77064d7ce7f4d5b7b3eeb4664977f3d1c0cde5b7e672d04d2229852e452c55454ec6df6e88cc129c5e4e2ed043b3ca6b346b46c4d8eb90acf3d7d68130cb489a757f877d370c1f06c1f58ea9fb8c24230c592ab4e3b0ff7179d0e2edb14ff4566d2280e22623cc43971af73005d88d357a8b2dfd0d319452ba6df19022563a378ba8ca60d28def1dffa3641fa30c7f66b7ff09e2f93237d706766527395deed2be475667f8dcf474486d979e3697c0b1e04a6bef425c9d5068aa219a90d8d9c5ef335c4ec08b5f39822ad867cfd3e2767495ed0dcaebd6989134e9811578979ffeec9a4b2abe771deafdbf0b3d5f568109ea77c67464f9dd73ba75058d051910d3570b4758d91b80f027e30a91540cfa393c55e63185cb854fe0cfe6762789a0eec383931cbb6e41481de5d18040ebca0b7acb979cc53b3c5474777545c3b17ad96d0103c9edd91462e045a6999e6d91c79f37472b965adb59aa73681ac821737096a321dcbc3572a7f053711b6172fbafe178124bf3e78473847076d64a8de52790aff0071a0f893718979290c545a88b3a801ee2247dfe9fc7ed21a3ec53259ab878990849c1976be5bf2033ee43f76c11c2967b6f44566c4909698541a06bc47a2def666d434d42bcc953da1a033e1fbbf4eccccfac40698e714d35d243d1941fb34916f34afe0288eabb8afb181a2cda976f0ec1c30fdd1cba1efee536986a9647472ae740a949c358f21d450ccd2f9ca2a54c28d2dba8a90d71b4d86b89e9cd1607b63cdf44ff3974d3efc21e4a6d58a6c4c9520ae703dd8ee86c29097aed5ae7d71b4ff080de4f9489d1563627d8a4eb1a1a6cd02c700994a464235675f2700faa14bbad9581d96cecddd5f283bd0ef83e5f1ca417e82b4634694a427af1df524a11a729eba26d012b61a105a33e31a1ffa8"
+          }
+        }
+      }
+    ]
+  }
+]
--- a/test/resource/sstables/fully_partially_contained_ssts/sst2.json
+++ b/test/resource/sstables/fully_partially_contained_ssts/sst2.json
@@ -0,0 +1,56 @@
+[
+  {
+    "key": {
+      "token": "-6915275422531117591",
+      "raw": "00873031323237335f3730316135363365643661653465303963333435663765646265333066376530653130323431653962393335353162616361373230623831316365373139303332303663343139666666613434643230333538386561626236663337343636303231636433303838393335393061663866636266636462666638633662653562",
+      "value": "012273_701a563ed6ae4e09c345f7edbe30f7e0e10241e9b93551baca720b811ce71903206c419fffa44d203588eabb6f37466021cd308893590af8fcbfcdbff8c6be5b"
+    },
+    "clustering_elements": [
+      {
+        "type": "clustering-row",
+        "key": {
+          "raw": "",
+          "value": ""
+        },
+        "marker": {
+          "timestamp": 1762936709124086
+        },
+        "columns": {
+          "value": {
+            "is_live": true,
+            "type": "regular",
+            "timestamp": 1762936709124086,
+            "value": "8357025e3d3dd2b90ec4fc75c71eec49ea4e90cd988f88e007647eff285aee143366eb9fc9b31f9fc9d56d432c4518a3fc5c2d544312b3690c286bc4c911a7ce4d8bca9e86e41ad5e12cd4d8d0bc38971460dc91e51983d2afd23b815f5b62472f61ae4095d405b9c3e27954e9700fc67606ac25b6d9bdeae87d31cc8100a493ee1336d91c75e0ed6398f7b2bb7044bfa3b17517e9d70d8a6f6bf1a90ce21c4009d3e9a45f7d69271882462871810165b47cf63428a5aa39efcee49fcd5bc50d3183acbd9ef40c26dfc9e5aa07ea6c9c5ecc30452686802a7110bb9fa522b062b79f4772e2218c765708ae2d8b073cb29d088340c853d99401884827f2ec0a40846f5aebdc2ae53c7838e7c8cc40a6ec4566f829991d23ce4ac016c7a6e1e06cb40204f087f2fc1ef37c89d8aa69eb803eea6ec4831a8965f3c2eba1a8f68ee3bff4abce5a10f88d7efe227ff213907407421f146a85451edc0ee23ec738a49dc6acda201aa5f8ef5c73ebb0e18fe7f1f0ddb1c4d7d155f926067f00a809456426c688258787534f571c7477ffc4d13c5003d6de81d603edb50ee3c0935d5af8843da943cd77a0be46a30e5da2eac07b4c1d73ca27b295d87128c9ec8dec8e8eed7a5b48de9ed03343f3955947d0dd815592dc49b7ed14d7d6f94651e1f0a9c52c8de61a45569398d203890fa39ceebeef9e1e0340698b94c1cea4f2e451a36011f242e1951b4ea372635681f08accda3e92866d931166effff43ef02632f06c0b6c774dd69f55636b84d58089a16686337a4ef4e4faa01c01c546012c090be5719e2bff9a5895cacb3061c0691603b87af0b1ddb6fffaf37a057c4817e3dc8b5dd7e3d40066b67533403ad10e55264acfdb82c1d89b4669ca01600cc0cc62301183ffa34cf5a4e7533cb8dd323a139f4fad659fbc0b01edeb1b41cf6a53ccecf43a31ae459035c029adcc42d83f8b00bce6e961130906e9701d582ec2caabc50437cbaa67fc4c8fdd80806e8fbf8a3781312344b9bf7eeb558ab0008684eb1abd15e63352fb943554478abf102ba4f4cbb402f75d30c9cf0a976fa4622e164a435986c4cb7d49dd9020198eab33595a90dd9db88990f354f1a4ef04d57f3e57f5d3257a070246e8dfc2314250be53bb81bf39db9f7247ef6718a2c5e1710d70497d0a87ac6dcd6277df0f09b8300db7dbacb147da1659962f4b1450480956b0f01546faf071712c118d33b4fdd1f56127f4659214e3a358149dae2ca0bf0760a8614f2f6e80ba64d7eca8fff920aaa4fc294d638e5d6817e4aa178aa84605cd0860ef1f26abd559c2dbd2b178ba7ce1f08abfc14c7c52be379ffef34469a0d5eb579044ecab4d9c66c80138a778278dd4acb48b4333e2ac8b1b9f7e615e4b68246ff74f6b6e86100908d82491e15e8b140839df826910ec3d78ee0ab8c07af6"
+          }
+        }
+      }
+    ]
+  },
+  {
+    "key": {
+      "token": "-5765536745937792235",
+      "raw": "00873031303934385f6537623538623362636364666537303938393738356130386638646363616631643362343263373431616434656334316436376662353037353862383064613064633262653438353661313065376133323032323933613564383963613331613661363864616334303932373338376565313736353537316538313933626663",
+      "value": "010948_e7b58b3bccdfe70989785a08f8dccaf1d3b42c741ad4ec41d67fb50758b80da0dc2be4856a10e7a3202293a5d89ca31a6a68dac40927387ee1765571e8193bfc"
+    },
+    "clustering_elements": [
+      {
+        "type": "clustering-row",
+        "key": {
+          "raw": "",
+          "value": ""
+        },
+        "marker": {
+          "timestamp": 1762936709038335
+        },
+        "columns": {
+          "value": {
+            "is_live": true,
+            "type": "regular",
+            "timestamp": 1762936709038335,
+            "value": "1effd8a1b5405518fcaf04c5df3e935652798315c5e41cd7eb73252c9b4defb4e05f05fd28bfed86bde53ff5326dbf5c2a1754176bbc5e1043c226f199ab5fcb0f0d4b7eff08cc6edb5828e571e1426aa730428b614804afcbd27d7e9c90fb3efbf4258af29bb17a52e32dbe34a282b2a509ed8abcf0402485d21f81f05b2c27ce69a358e552ff397539893cfce5893f295a5894fa83470261c2333e8d7949dc25a99b5e6ba8b8911edd15b2ef703605dccbe705d07e5c1b4d39be658f1af2c6e391784f09b93dd63cf1eff8e6d581562ad7a133faedda561d3e0e8a034da1fd6b2b08d882b26fc7abc253f7e59b61f94576a23d2f91f55e49d265b963d85748ca61ea3cef3710312b2f89cee4a3fa0fcc5c949a9add328083d34120eee27b727ce41b9b7eb3fda9da3657eeff89df797c6c528c675573aea35c5aaef931d2d1f88ca7f3d2139f542e80d6d9c5a098758d9837be435d4ad97f98a54ddcdb34b864f779966f6d7e05a41c552c5ab83c416cb9edd1cc85297e9ec879df70f0d2fc5f7b54a55fdb74f9f58ed9a90a28edf12ff37940c557ad7862eb3aae1fcc2212dbc006d0bcb4e7923b811667ef01cbf524d99a161ce7682a4e3cdd17a1ee1a49f5eac4370f97a5b9e3630f0eba89bd9134bff7f83818c6ef1f85628095bc19d0f67ce2175154ef2b692bd6990d18cf95ea73f1e0b83c07ea74abfddaf982006bdd024d85c3b9dabfb0bd2175af927303efebf382a89583e30990d72712337744d61c7c8c36fd0dc4dff6fe326a6ffab924a0fff8a7edfbe8196a93a8dab274f0f8550ae7719b5f27699f27f0a4a52e68005219dd909317f3892f0c3102da82a61c71a1e6003d8bd1343b5f88b31e5b67711fc495a7a30f19042c2f0e78271679971723e205049dffb659c8e5f54c9311353ae9f86455ff04018db1275a02db2a1fdbddc66185d3df4d82776469e8de111048435d17153adf438cac990aa2ebaf602d8d8cfe698c2d0a82ebdfda73eb024a7e0e8b7de45593c9eb1404f4dfa32e723ccdb43bd77a517733706b6bae00713e77757c0e7effc7397b4724204f418c26c055d606e8d56ac2786ad4e7465c430126c4d7dc30aa87e21e27555e7385f345eee98961f9410002a22cd6f5b44e42af96d8337db1411cba2f8e2c77b5831e04a84707b528a5e9e0c004e2a95a41d64c7f44c6bb8091405e13615f713f2d02c73861ce902b8a207aa945287cb667a215e8cea5b1ac452b789be772f3e110dc8366d1f82ed687696f18f401da97d0f6bf9d03e1a2f660f02f5218303b00859cc634f7620f412a4b97ce6165a0be20bd9a6b48c7300cbb1e0a12e12b9eabaad2df24e01155911fd2ff996260d9ac7a97838669bcd77afbb4e70fef26471595ea5a43462b8d549a04fde233885edf8afea5cba53b0eae53275e49acd004f08855"
+          }
+        }
+      }
+    ]
+  }
+]
--- a/test/resource/sstables/fully_partially_contained_ssts/sst3.json
+++ b/test/resource/sstables/fully_partially_contained_ssts/sst3.json
@@ -0,0 +1,56 @@
+[
+  {
+    "key": {
+      "token": "-5761288770845929376",
+      "raw": "00873031373130395f3138646531303264643433333930653838656132363334383831343263336332373935393463343630663231353036643238333566303133356462383333326534386565336337343361316338363365656138623634386437343164346166333138373631643034633265313436643234636339306532393839313834316131",
+      "value": "017109_18de102dd43390e88ea263488142c3c279594c460f21506d2835f0135db8332e48ee3c743a1c863eea8b648d741d4af318761d04c2e146d24cc90e29891841a1"
+    },
+    "clustering_elements": [
+      {
+        "type": "clustering-row",
+        "key": {
+          "raw": "",
+          "value": ""
+        },
+        "marker": {
+          "timestamp": 1762936709444301
+        },
+        "columns": {
+          "value": {
+            "is_live": true,
+            "type": "regular",
+            "timestamp": 1762936709444301,
+            "value": "9591274f47c91f1c28dfe8d4983239c461d1318966b454cb6df703eaa64602f5b7cabb6d763e283006875786e67e0b6f7d8f8a96a93c62730861df7aa736d87b8ee53969b03e469a738195ff65edffe4ba06d26cd9e1808f1e41efdcc633c7b300ec38137b1020ffd3945b36890bb6557b1c817dcc3f0734883d06ac8df8001f5c946d3f69239f467eb66261dbb643d4185a3db4e98f839f03460cf7a6b9901a6277b2c1feee430291c6ea6c78b0fb67beeca2677b6226b5e68d43a4ce3fbc6309f63e6ddfe3f3b004c1e1f1b117211f17ed37f0669e03e74f3ca7cebad8002c118cfae31dbd2db1fd63e0c77c8a1051ad6c2a9ad4cb7c1451e66ca967aec64f5204a234fddb964733aa9c805aa298fee3da71b6e1c9fb229b48b06431ee80690be702ab57d2fedd86d942061af79ff7e789bf9b062f305018345b545a1fc298e827a6f7ebecd7e23e41e4a94aa81e083c782e88f2b1e78e4e36dc97346dbbb67ccb9220553f6f3dbabf010941b71a4909edd92fa955409f49b4bb2623db5dd501d337c447d43ae19bfc4435ab24d684db75c623ce62c7bf8cf767ca846e0a0b0853f19f987a7eb26083c7c7de55327572b71d85bc44435d38dcbe188df299f6f9292e34293c1fe300d63bd7cb786bca908d49f21012f95e882fb954eded5f5efb950cc2fa27c6525eb3be7def4a883373c5065604350107aa0621d5b003650968d364b8dc2f54a090e0eb9456e52ccfeb67ebf0e212f47ecdac9231009bb97351bb6c57a8a77dedd481594bdfa96329dd47e39f89332a85681f77c2f4f4666abf96a787ca5a56b05971127256e91efe70421418b3101d4fb84a8a5aa10bfc2e3f44da75279dfa4c180efaf44434226ab4036e318d1ae1304f1afe32d1111d29544236d774482470a0c2c2dd0b24f72150f18e7ac74d2c95db5b25be2fdd9d0c2a716c0ac229915edca19501fcca9f7593729d57448c37fbc80c1f18cf7186fd3b621769b58238dfe49bcad20bde956e650ca3a1b84f0ba1e381c2a39cbf0ea61270a80be5e2a3525ae9fb135e4b166e3457da32c89b292bbde180ea6f5cc1acfc9877d4ef9d102020a9e4d7ea08b6e757ecd79a78e2f282471ba9335329a5d439249e20d18fa78823b93a7e8038501387d11c3e227ace275ae69b14ce7cf2a79d0dca597d6ee4b79fa20b64f992749eb756e0ae716d100a3fd6c1a7a94185b1857f53a65836f638243e0732c1914205f372d072a81084d117202fbb9bbb6d36b198dd60a85ac76737b4811055481204d5743f686d0cbd86db47039be91ebd0ef169efc10ea73972590d1e87f246f4653813bcba15f140747c20586572d42b1cec66c183c8d863eb792655c0545c42784f854864efa0a3fd2ebb1d8fe5399036569621f9355346892e59e8a8fe3c7b8c8cde555dece0999f77faeeff7f6920f6"
+          }
+        }
+      }
+    ]
+  },
+  {
+    "key": {
+      "token": "-4613959642921636184",
+      "raw": "00873031303231375f6336633634393161356235326233646539313832653934613762383834333865656339373630633962623836633566386139393739366232376664623761623763376265323162363630363439303032653337666331666637626137626236663365376537353537366562316431303331653738333434376436323332633263",
+      "value": "010217_c6c6491a5b52b3de9182e94a7b88438eec9760c9bb86c5f8a99796b27fdb7ab7c7be21b660649002e37fc1ff7ba7bb6f3e7e75576eb1d1031e783447d6232c2c"
+    },
+    "clustering_elements": [
+      {
+        "type": "clustering-row",
+        "key": {
+          "raw": "",
+          "value": ""
+        },
+        "marker": {
+          "timestamp": 1762936708974816
+        },
+        "columns": {
+          "value": {
+            "is_live": true,
+            "type": "regular",
+            "timestamp": 1762936708974816,
+            "value": "9d6a19ca775ae707bbac625f92a83f47843bdcaed07864380de44b78b3a0172e30dfeeb544a60017ae0f5685eabf6b33c7f9b0671cea66e60ea7c68acc3b3a3f0e15ef1736b57b5bdbb33df3be57af5e23c1acf9e90cb896ee107abdb997f6bc29a472c0f2c314cd23c1960798f9fdf69cac6679e90922137f251397111043df91af87561530532e0a02445ad19b936d1f7e0075a74438c8e881156697f7b9f14f6d5e4ce904b968c947e7683486280efb84ef4526b9a1836294feb469f773929b17e987a0acf9f2ea13fd4946e1384534b837823904ac9c0aad6501ab666586eb667ab941677ffe2a8c675812273ab0311d111685dbf6eaeeff036cf2c93470f7411a149d71bca6a83360fc129ce06734381aaa5d5dd6dcce687dca059c513e23123ceb7b96581d2d08ee6a6bce2e079f1c20b470036fb62edc701d8e5637da08104e6a6fd1816ab739b973ea5400c4b4028ad16cbb75ceeed3fb1bac1bcc4d2a90c285db855be170bdf12392821a0ba5062a796e1a5554c02c04ddc5fddf0249f6737d5fc58eaadeba0457c7342e2a31bd546d70120980caf3854c3e9044acdf014d0453e434aac3b50c45b3facaeb37a1a8135c875a9bff452804d405d80b925f1d7a1303562aa8833a8729131bee627a36eb26a2df3f0fd2b85ffee4755add1126bcb56949d2f0a8e4e69cd5fc6b1862315a13bf296286f8dd3cf7ce5ffe882665feb8239a5b4752f95facb35e04b05360647d4f6bdcf548d0ae1957c16563769a89b0b921fbaceca19fbb92f2331ee1bb51395cf6f19f865faf1b2406a3ed0fa9a999ed41af493506edf00f5bbfd44bd61572314c2870b1d584c62ec396ab297a24609ae2b0ba4319350501d19a853071b32680a860ef611f0af770c9901e7d3d1d3a7d89c5f11729cc662b04624e38e97cf37ea2cfe8f0155a579cd64b737c46f633343dd3be7898e96769e3cd4f75a8a7d4f14eba40c899265fe82b3c3b539419ee992be61d876cdb23fdff896770a1b1c1787bbd457e8a3ec3ccc8ac903928aea19e96c3eb111e1e7b4d9f742a2faad575192208cdbce1404b8470b852dff28c1a8edf98799e67ecc518577bdcccae02ed21e9145617c5535b648970fed0563549510661309c5b39570e87406e4d40ee1338e0ddd64b4ec5b03cb15333b4f5f024c57d4987e4690e2b4a958f5617d186428dca02f34682fb204b5cc46202c32b7baa8ce0b6e210f1dd8ba94ce51d6bd62987a687db27a33cf1ec7cb449946a65982f5dbb81acca61a2d6fe16a550dadba31e6f0dca5d796660130c3969f2c8a5af3e55f2d8fdf0ac207662311b0369c5e5a754e9153c06393faa09912326c8e370c1ac0da25ce5e96fb243702fcd0260013ebba70df3556d61047755a1ebcfb80b49524e0f7bdcf17ee0d013c9486d4271091a4b78b0452fce881858"
+          }
+        }
+      }
+    ]
+  }
+]