test/alternator: switch some fixture scopes from "session" to "module"

In conftest.py we have several fixtures creating shared tables which many
test files can share, so they are marked with the "session" scope - all
the tests in the testing session may share the same instance. This is fine.

Some of test files have additional fixtures for creating special tables
needed only in those files. Those were also, unnecessarily, marked
"session" scope as well. This means that these temporary tables are
only deleted at the very end of test suite, event though they can be
deleted at the end of the test file which needed them. This is exactly
what the "module" fixture scope is, so this patch changes all the
fixtures private to one test file to be "module".

After this patch, the teardown of the last test in the suite goes down
from 4 seconds to just 1.5 seconds (it's still long because there are
still plenty of session-scoped fixtures in conftest.py).

Another small benefit is that the peak disk usage of the test suite is
lower, because some of the temporary tables are deleted sooner.

This patch does not change any test functionality, and also does not
make any test faster - it just changes the order of the fixture
teardowns.

Signed-off-by: Nadav Har'El <nyh@scylladb.com>
Message-Id: <20210317175036.1773774-1-nyh@scylladb.com>

test/alternator/test_filter_expression.py

@@ -73,7 +73,7 @@ def random_item(p, i):
     if t == 0:
         item['j'] = item['i']
     return item
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="module")
 def test_table_sn_with_data(test_table_sn):
     p = random_string()
     # TODO: because we use random items here, it may be difficult to reproduce

test/alternator/test_gsi.py

@@ -82,7 +82,7 @@ def test_gsi_identical(dynamodb):
 # One of the simplest forms of a non-trivial GSI: The base table has a hash
 # and sort key, and the index reverses those roles. Other attributes are just
 # copied.
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="module")
 def test_table_gsi_1(dynamodb):
     table = create_test_table(dynamodb,
         KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' },
@@ -203,7 +203,7 @@ def test_gsi_missing_attribute_definition(dynamodb):
 # hash key (which is the base's hash key). In the materialized-view-based
 # implementation, we need to remember the other part of the base key as a
 # clustering key.
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="module")
 def test_table_gsi_1_hash_only(dynamodb):
     table = create_test_table(dynamodb,
         KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' },
@@ -250,7 +250,7 @@ def test_gsi_key_not_in_index(test_table_gsi_1_hash_only):
 
 # A second scenario of GSI. Base table has just hash key, Index has a
 # different hash key - one of the non-key attributes from the base table.
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="module")
 def test_table_gsi_2(dynamodb):
     table = create_test_table(dynamodb,
         KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' } ],
@@ -382,7 +382,7 @@ def test_gsi_wrong_type_attribute_batch(test_table_gsi_2):
 # difficult to implement in Alternator because Scylla's materialized-views
 # implementation only allows one new key column in the view, and here
 # we need two (which, also, aren't actual columns, but map items).
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="module")
 def test_table_gsi_3(dynamodb):
     table = create_test_table(dynamodb,
         KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' } ],
@@ -453,7 +453,7 @@ def test_gsi_missing_attribute_3(test_table_gsi_3):
     assert not any([i['p'] == p for i in full_scan(test_table_gsi_3, ConsistentRead=False, IndexName='hello')])
 
 # A fourth scenario of GSI. Two GSIs on a single base table.
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="module")
 def test_table_gsi_4(dynamodb):
     table = create_test_table(dynamodb,
         KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' } ],
@@ -500,7 +500,7 @@ def test_gsi_4_describe(test_table_gsi_4):
     assert multiset([g['IndexName'] for g in gsis]) == multiset(['hello_a', 'hello_b'])
 
 # A scenario for GSI in which the table has both hash and sort key
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="module")
 def test_table_gsi_5(dynamodb):
     table = create_test_table(dynamodb,
         KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' }, { 'AttributeName': 'c', 'KeyType': 'RANGE' } ],
@@ -875,7 +875,7 @@ def test_gsi_very_long_name(dynamodb):
 # name. This assumes that materialized-view names are composed using the
 # index's name (which is currently what we do).
 
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="module")
 def test_table_gsi_random_name(dynamodb):
     index_name = random_string()
     table = create_test_table(dynamodb,

test/alternator/test_key_condition_expression.py

@@ -31,7 +31,7 @@ from util import random_string, full_query, multiset
 # queries. This fixture is useful for writing many small query tests which
 # read the same input data without needing to re-insert data for every test,
 # so overall the test suite is faster.
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="module")
 def test_table_sn_with_sorted_partition(test_table_sn):
     p = random_string()
     items = [{'p': p, 'c': i, 'a': random_string()} for i in range(12)]
@@ -43,7 +43,7 @@ def test_table_sn_with_sorted_partition(test_table_sn):
         batch.put_item({'p': random_string(), 'c': 123, 'a': random_string()})
     yield test_table_sn, p, items
 
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="module")
 def test_table_ss_with_sorted_partition(test_table):
     p = random_string()
     items = [{'p': p, 'c': str(i).zfill(3), 'a': random_string()} for i in range(12)]
@@ -53,7 +53,7 @@ def test_table_ss_with_sorted_partition(test_table):
         batch.put_item({'p': random_string(), 'c': '123', 'a': random_string()})
     yield test_table, p, items
 
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="module")
 def test_table_sb_with_sorted_partition(test_table_sb):
     p = random_string()
     items = [{'p': p, 'c': bytearray(str(i).zfill(3), 'ascii'), 'a': random_string()} for i in range(12)]

test/alternator/test_key_conditions.py

@@ -31,7 +31,7 @@ from util import random_string, random_bytes, full_query, multiset
 # queries. This fixture is useful for writing many small query tests which
 # read the same input data without needing to re-insert data for every test,
 # so overall the test suite is faster.
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="module")
 def test_table_sn_with_sorted_partition(test_table_sn):
     p = random_string()
     items = [{'p': p, 'c': i, 'a': random_string()} for i in range(12)]
@@ -43,7 +43,7 @@ def test_table_sn_with_sorted_partition(test_table_sn):
         batch.put_item({'p': random_string(), 'c': 123, 'a': random_string()})
     yield test_table_sn, p, items
 
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="module")
 def test_table_ss_with_sorted_partition(test_table):
     p = random_string()
     items = [{'p': p, 'c': str(i).zfill(3), 'a': random_string()} for i in range(12)]
@@ -53,7 +53,7 @@ def test_table_ss_with_sorted_partition(test_table):
         batch.put_item({'p': random_string(), 'c': '123', 'a': random_string()})
     yield test_table, p, items
 
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="module")
 def test_table_sb_with_sorted_partition(test_table_sb):
     p = random_string()
     items = [{'p': p, 'c': bytearray(str(i).zfill(3), 'ascii'), 'a': random_string()} for i in range(12)]

test/alternator/test_lsi.py

@@ -128,7 +128,7 @@ def test_lsi_wrong(dynamodb):
 # A simple scenario for LSI. Base table has a partition key and a sort key,
 # index has the same partition key key but a different sort key - one of
 # the non-key attributes from the base table.
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="module")
 def test_table_lsi_1(dynamodb):
     table = create_test_table(dynamodb,
         KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' }, { 'AttributeName': 'c', 'KeyType': 'RANGE' } ],
@@ -169,7 +169,7 @@ def test_lsi_1(test_table_lsi_1):
 
 # A second scenario of LSI. Base table has both hash and sort keys,
 # a local index is created on each non-key parameter
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="module")
 def test_table_lsi_4(dynamodb):
     table = create_test_table(dynamodb,
         KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' }, { 'AttributeName': 'c', 'KeyType': 'RANGE' } ],
@@ -229,7 +229,7 @@ def test_lsi_describe(test_table_lsi_4):
     # TODO: check also ProvisionedThroughput, IndexArn
 
 # A table with selective projection - only keys are projected into the index
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="module")
 def test_table_lsi_keys_only(dynamodb):
     table = create_test_table(dynamodb,
         KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' }, { 'AttributeName': 'c', 'KeyType': 'RANGE' } ],
@@ -313,7 +313,7 @@ def test_lsi_consistent_read(test_table_lsi_1):
                        'b': {'AttributeValueList': [b2], 'ComparisonOperator': 'EQ'}})
 
 # A table with both gsi and lsi present
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="module")
 def test_table_lsi_gsi(dynamodb):
     table = create_test_table(dynamodb,
         KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' }, { 'AttributeName': 'c', 'KeyType': 'RANGE' } ],

test/alternator/test_query_filter.py

@@ -74,7 +74,7 @@ def random_item(p, i):
     if t == 0:
         item['j'] = item['i']
     return item
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="module")
 def test_table_sn_with_data(test_table_sn):
     p = random_string()
     # TODO: because we use random items here, it may be difficult to reproduce

test/alternator/test_streams.py

@@ -431,7 +431,7 @@ def test_get_records_nonexistent_iterator(dynamodbstreams):
 # It is still worthwhile to use a fixture rather than to create a table
 # explicitly - it is convenient, safe (the table gets deleted automatically)
 # and if in the future we can work around the DynamoDB problem, we can return
-# these fixtures to session scope.
+# these fixtures to module scope.
 
 def create_table_ss(dynamodb, dynamodbstreams, type):
     table = create_test_table(dynamodb,
@@ -1211,10 +1211,10 @@ def test_streams_1_new_and_old_images(test_table_ss_new_and_old_images, dynamodb
 
 # A fixture which creates a test table with a stream enabled, and returns a
 # bunch of interesting information collected from the CreateTable response.
-# This fixture is session-scoped - it can be shared by multiple tests below,
+# This fixture is module-scoped - it can be shared by multiple tests below,
 # because we are not going to actually use or change this stream, we will
 # just do multiple tests on its setup.
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="module")
 def test_table_stream_with_result(dynamodb, dynamodbstreams):
     tablename = test_table_name()
     result = dynamodb.meta.client.create_table(TableName=tablename,

test/alternator/test_table.py

@@ -263,7 +263,7 @@ def test_table_streams_off(dynamodb):
 # named columns.
 special_column_name1 = 'attrs'
 special_column_name2 = ':attrs'
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="module")
 def test_table_special_column_name(dynamodb):
     table = create_test_table(dynamodb,
         KeySchema=[

test/alternator/test_tag.py

@@ -101,7 +101,7 @@ def test_tag_resource_overwrite(test_table):
     assert multiset(got['Tags']) == multiset(tags)
 
 PREDEFINED_TAGS = [{'Key': 'str1', 'Value': 'str2'}, {'Key': 'kkk', 'Value': 'vv'}, {'Key': 'keykey', 'Value': 'valvalvalval'}]
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="module")
 def test_table_tags(dynamodb):
     # The feature of creating a table already with tags was only added to
     # DynamoDB in April 2019, and to the botocore library in version 1.12.136

test/alternator/test_tracing.py

@@ -192,7 +192,7 @@ def expect_tracing_events(dynamodb, str, expected_events):
         assert event in events
 
 # A test table based on test_table_s, but with isolation level defined to 'always'
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="module")
 def test_table_s_isolation_always(dynamodb):
     table = create_test_table(dynamodb,
         KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' }, ],