From 6512af8b26b9661235f178776059014b9b423a61 Mon Sep 17 00:00:00 2001 From: Georgiy Lebedev Date: Sat, 1 Jul 2023 13:49:17 +0300 Subject: [PATCH 01/39] test: fix `box.tuple.new` variable argument syntax usage Variable argument syntax of `box.tuple.new` will be disabled in scope of tarantool/tarantool#4693: adapt the DML test suite to stop using it. --- test/suites/test_dml.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/suites/test_dml.py b/test/suites/test_dml.py index 0263d451..69293b55 100644 --- a/test/suites/test_dml.py +++ b/test/suites/test_dml.py @@ -210,7 +210,7 @@ def test_07_call_16(self): self.assertEqual(len(ans[0]), 1) self.assertIsInstance(ans[0][0], str) - self.assertSequenceEqual(con.call('box.tuple.new', [1, 2, 3, 'fld_1']), + self.assertSequenceEqual(con.call('box.tuple.new', [[1, 2, 3, 'fld_1']]), [[1, 2, 3, 'fld_1']]) self.assertSequenceEqual(con.call('box.tuple.new', 'fld_1'), [['fld_1']]) finally: @@ -236,7 +236,7 @@ def test_07_call_17(self): self.assertEqual(len(ans), 1) self.assertIsInstance(ans[0], str) - self.assertSequenceEqual(con.call('box.tuple.new', [1, 2, 3, 'fld_1']), + self.assertSequenceEqual(con.call('box.tuple.new', [[1, 2, 3, 'fld_1']]), [[1, 2, 3, 'fld_1']]) self.assertSequenceEqual(con.call('box.tuple.new', 'fld_1'), [['fld_1']]) From 97c1c37a5753d92605ae236651efd8c35ff34e5d Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Thu, 13 Jul 2023 19:31:29 +0300 Subject: [PATCH 02/39] doc: add week to interval example --- tarantool/msgpack_ext/types/interval.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tarantool/msgpack_ext/types/interval.py b/tarantool/msgpack_ext/types/interval.py index 10dc4847..c5bec39e 100644 --- a/tarantool/msgpack_ext/types/interval.py +++ b/tarantool/msgpack_ext/types/interval.py @@ -49,8 +49,8 @@ class Interval(): .. code-block:: python - di = tarantool.Interval(year=-1, month=2, day=3, - hour=4, minute=-5, sec=6, + di = tarantool.Interval(year=-1, month=2, week=-3, + day=4, hour=5, minute=-6, sec=7, nsec=308543321, adjust=tarantool.IntervalAdjust.NONE) From 778f2b77754bef4761eea83efbebdeefdaa770dd Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Thu, 13 Jul 2023 19:32:23 +0300 Subject: [PATCH 03/39] test: add week interval cases Add Interval with week encoding test cases and datetime arithmetic week test case. --- test/suites/test_datetime.py | 6 ++++++ test/suites/test_interval.py | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/test/suites/test_datetime.py b/test/suites/test_datetime.py index 40f84215..ae22dcfc 100644 --- a/test/suites/test_datetime.py +++ b/test/suites/test_datetime.py @@ -480,6 +480,12 @@ def test_tarantool_datetime_subtraction_different_timezones(self): 'res_add': tarantool.Datetime(year=2009, month=3, day=31), 'res_sub': tarantool.Datetime(year=2009, month=1, day=31), }, + 'week': { + 'arg_1': tarantool.Datetime(year=2008, month=2, day=3), + 'arg_2': tarantool.Interval(week=1), + 'res_add': tarantool.Datetime(year=2008, month=2, day=10), + 'res_sub': tarantool.Datetime(year=2008, month=1, day=27), + }, } def test_python_interval_addition(self): diff --git a/test/suites/test_interval.py b/test/suites/test_interval.py index 77f2cf52..1821e901 100644 --- a/test/suites/test_interval.py +++ b/test/suites/test_interval.py @@ -148,6 +148,24 @@ def test_interval_positional_init(self): 'msgpack': (b'\x00'), 'tarantool': r"datetime.interval.new({adjust='excess'})", }, + 'weeks': { + 'python': tarantool.Interval(week=3), + 'msgpack': (b'\x02\x02\x03\x08\x01'), + 'tarantool': r"datetime.interval.new({week=3})", + }, + 'date_with_week': { + 'python': tarantool.Interval(year=1, month=2, week=3, day=4), + 'msgpack': (b'\x05\x00\x01\x01\x02\x02\x03\x03\x04\x08\x01'), + 'tarantool': r"datetime.interval.new({year=1, month=2, week=3, day=4})", + }, + 'datetime_with_week': { + 'python': tarantool.Interval(year=1, month=2, week=3, day=4, hour=1, minute=2, + sec=3000, nsec=10000000), + 'msgpack': (b'\x09\x00\x01\x01\x02\x02\x03\x03\x04\x04\x01\x05\x02\x06\xcd\x0b\xb8' + b'\x07\xce\x00\x98\x96\x80\x08\x01'), + 'tarantool': r"datetime.interval.new({year=1, month=2, week=3, day=4, hour=1, " + r"min=2, sec=3000, nsec=10000000})", + }, } def test_msgpack_decode(self): From 1607a4603bf3c7058dd3aa9054d2147165b8ffcc Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Thu, 13 Jul 2023 19:31:17 +0300 Subject: [PATCH 04/39] interval: fix week arithmetic Before this patch, weeks were ignored in Interval addition and subtraction. This patch fixes the issue. --- CHANGELOG.md | 5 +++++ tarantool/msgpack_ext/types/interval.py | 2 ++ test/suites/test_interval.py | 22 ++++++++++++++++++++++ 3 files changed, 29 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ad980b7..84b8a35b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Fixed +- `tarantool.Interval` arithmetic with weeks + ## 1.1.0 - 2023-06-30 ### Added diff --git a/tarantool/msgpack_ext/types/interval.py b/tarantool/msgpack_ext/types/interval.py index c5bec39e..c1cf7189 100644 --- a/tarantool/msgpack_ext/types/interval.py +++ b/tarantool/msgpack_ext/types/interval.py @@ -145,6 +145,7 @@ def __add__(self, other): return Interval( year=self.year + other.year, month=self.month + other.month, + week=self.week + other.week, day=self.day + other.day, hour=self.hour + other.hour, minute=self.minute + other.minute, @@ -194,6 +195,7 @@ def __sub__(self, other): return Interval( year=self.year - other.year, month=self.month - other.month, + week=self.week - other.week, day=self.day - other.day, hour=self.hour - other.hour, minute=self.minute - other.minute, diff --git a/test/suites/test_interval.py b/test/suites/test_interval.py index 1821e901..8474620a 100644 --- a/test/suites/test_interval.py +++ b/test/suites/test_interval.py @@ -283,6 +283,28 @@ def test_unknown_adjust_decode(self): 'res_add': tarantool.Interval(year=3, adjust=tarantool.IntervalAdjust.LAST), 'res_sub': tarantool.Interval(year=1, adjust=tarantool.IntervalAdjust.LAST), }, + 'weeks': { + 'arg_1': tarantool.Interval(week=2), + 'arg_2': tarantool.Interval(week=1), + 'res_add': tarantool.Interval(week=3), + 'res_sub': tarantool.Interval(week=1), + }, + 'date_with_week': { + 'arg_1': tarantool.Interval(year=1, month=2, week=3, day=4), + 'arg_2': tarantool.Interval(year=4, month=3, week=2, day=1), + 'res_add': tarantool.Interval(year=5, month=5, week=5, day=5), + 'res_sub': tarantool.Interval(year=-3, month=-1, week=1, day=3), + }, + 'datetime_with_week': { + 'arg_1': tarantool.Interval(year=1, month=2, week=3, day=4, hour=1, minute=2, + sec=3000, nsec=10000000), + 'arg_2': tarantool.Interval(year=2, month=1, week=-1, day=31, hour=-3, minute=0, + sec=1000, nsec=9876543), + 'res_add': tarantool.Interval(year=3, month=3, week=2, day=35, hour=-2, minute=2, + sec=4000, nsec=19876543), + 'res_sub': tarantool.Interval(year=-1, month=1, week=4, day=-27, hour=4, minute=2, + sec=2000, nsec=123457), + }, } def test_python_interval_addition(self): From 6eec0e773012cb03099fad6f56532fa60bd7b641 Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Thu, 13 Jul 2023 19:44:57 +0300 Subject: [PATCH 05/39] interval: display weeks in str and repr --- CHANGELOG.md | 1 + tarantool/msgpack_ext/types/interval.py | 4 +-- test/suites/test_interval.py | 38 +++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 84b8a35b..35b5dcdc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - `tarantool.Interval` arithmetic with weeks +- `tarantool.Interval` weeks display in `str()` and `repr()` ## 1.1.0 - 2023-06-30 diff --git a/tarantool/msgpack_ext/types/interval.py b/tarantool/msgpack_ext/types/interval.py index c1cf7189..54551a18 100644 --- a/tarantool/msgpack_ext/types/interval.py +++ b/tarantool/msgpack_ext/types/interval.py @@ -233,8 +233,8 @@ def __eq__(self, other): return True def __repr__(self): - return f'tarantool.Interval(year={self.year}, month={self.month}, day={self.day}, ' + \ - f'hour={self.hour}, minute={self.minute}, sec={self.sec}, ' + \ + return f'tarantool.Interval(year={self.year}, month={self.month}, week={self.week}, ' + \ + f'day={self.day}, hour={self.hour}, minute={self.minute}, sec={self.sec}, ' + \ f'nsec={self.nsec}, adjust={self.adjust})' __str__ = __repr__ diff --git a/test/suites/test_interval.py b/test/suites/test_interval.py index 8474620a..6917b49c 100644 --- a/test/suites/test_interval.py +++ b/test/suites/test_interval.py @@ -71,42 +71,58 @@ def test_interval_positional_init(self): 'python': tarantool.Interval(year=1), 'msgpack': (b'\x02\x00\x01\x08\x01'), 'tarantool': r"datetime.interval.new({year=1})", + 'str': 'tarantool.Interval(year=1, month=0, week=0, day=0, hour=0, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', }, 'big_year': { 'python': tarantool.Interval(year=1000), 'msgpack': (b'\x02\x00\xcd\x03\xe8\x08\x01'), 'tarantool': r"datetime.interval.new({year=1000})", + 'str': 'tarantool.Interval(year=1000, month=0, week=0, day=0, hour=0, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', }, 'date': { 'python': tarantool.Interval(year=1, month=2, day=3), 'msgpack': (b'\x04\x00\x01\x01\x02\x03\x03\x08\x01'), 'tarantool': r"datetime.interval.new({year=1, month=2, day=3})", + 'str': 'tarantool.Interval(year=1, month=2, week=0, day=3, hour=0, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', }, 'big_month_date': { 'python': tarantool.Interval(year=1, month=100000, day=3), 'msgpack': (b'\x04\x00\x01\x01\xce\x00\x01\x86\xa0\x03\x03\x08\x01'), 'tarantool': r"datetime.interval.new({year=1, month=100000, day=3})", + 'str': 'tarantool.Interval(year=1, month=100000, week=0, day=3, hour=0, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', }, 'time': { 'python': tarantool.Interval(hour=1, minute=2, sec=3), 'msgpack': (b'\x04\x04\x01\x05\x02\x06\x03\x08\x01'), 'tarantool': r"datetime.interval.new({hour=1, min=2, sec=3})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=1, ' + 'minute=2, sec=3, nsec=0, adjust=Adjust.NONE)', }, 'big_seconds_time': { 'python': tarantool.Interval(hour=1, minute=2, sec=3000), 'msgpack': (b'\x04\x04\x01\x05\x02\x06\xcd\x0b\xb8\x08\x01'), 'tarantool': r"datetime.interval.new({hour=1, min=2, sec=3000})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=1, ' + 'minute=2, sec=3000, nsec=0, adjust=Adjust.NONE)', }, 'datetime': { 'python': tarantool.Interval(year=1, month=2, day=3, hour=1, minute=2, sec=3000), 'msgpack': (b'\x07\x00\x01\x01\x02\x03\x03\x04\x01\x05\x02\x06\xcd\x0b\xb8\x08\x01'), 'tarantool': r"datetime.interval.new({year=1, month=2, day=3, hour=1, " r"min=2, sec=3000})", + 'str': 'tarantool.Interval(year=1, month=2, week=0, day=3, hour=1, ' + 'minute=2, sec=3000, nsec=0, adjust=Adjust.NONE)', }, 'nanoseconds': { 'python': tarantool.Interval(nsec=10000000), 'msgpack': (b'\x02\x07\xce\x00\x98\x96\x80\x08\x01'), 'tarantool': r"datetime.interval.new({nsec=10000000})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + 'minute=0, sec=0, nsec=10000000, adjust=Adjust.NONE)', }, 'datetime_with_nanoseconds': { 'python': tarantool.Interval(year=1, month=2, day=3, hour=1, minute=2, @@ -115,6 +131,8 @@ def test_interval_positional_init(self): b'\x00\x98\x96\x80\x08\x01'), 'tarantool': r"datetime.interval.new({year=1, month=2, day=3, hour=1, " r"min=2, sec=3000, nsec=10000000})", + 'str': 'tarantool.Interval(year=1, month=2, week=0, day=3, hour=1, ' + 'minute=2, sec=3000, nsec=10000000, adjust=Adjust.NONE)', }, 'datetime_none_adjust': { 'python': tarantool.Interval(year=1, month=2, day=3, hour=1, minute=2, @@ -124,6 +142,8 @@ def test_interval_positional_init(self): b'\x00\x98\x96\x80\x08\x01'), 'tarantool': r"datetime.interval.new({year=1, month=2, day=3, hour=1, " r"min=2, sec=3000, nsec=10000000, adjust='none'})", + 'str': 'tarantool.Interval(year=1, month=2, week=0, day=3, hour=1, ' + 'minute=2, sec=3000, nsec=10000000, adjust=Adjust.NONE)', }, 'datetime_excess_adjust': { 'python': tarantool.Interval(year=1, month=2, day=3, hour=1, minute=2, @@ -133,6 +153,8 @@ def test_interval_positional_init(self): b'\x00\x98\x96\x80'), 'tarantool': r"datetime.interval.new({year=1, month=2, day=3, hour=1, " r"min=2, sec=3000, nsec=10000000, adjust='excess'})", + 'str': 'tarantool.Interval(year=1, month=2, week=0, day=3, hour=1, ' + 'minute=2, sec=3000, nsec=10000000, adjust=Adjust.EXCESS)', }, 'datetime_last_adjust': { 'python': tarantool.Interval(year=1, month=2, day=3, hour=1, minute=2, @@ -142,21 +164,29 @@ def test_interval_positional_init(self): b'\x00\x98\x96\x80\x08\x02'), 'tarantool': r"datetime.interval.new({year=1, month=2, day=3, hour=1, " r"min=2, sec=3000, nsec=10000000, adjust='last'})", + 'str': 'tarantool.Interval(year=1, month=2, week=0, day=3, hour=1, ' + 'minute=2, sec=3000, nsec=10000000, adjust=Adjust.LAST)', }, 'all_zeroes': { 'python': tarantool.Interval(adjust=tarantool.IntervalAdjust.EXCESS), 'msgpack': (b'\x00'), 'tarantool': r"datetime.interval.new({adjust='excess'})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.EXCESS)', }, 'weeks': { 'python': tarantool.Interval(week=3), 'msgpack': (b'\x02\x02\x03\x08\x01'), 'tarantool': r"datetime.interval.new({week=3})", + 'str': 'tarantool.Interval(year=0, month=0, week=3, day=0, hour=0, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', }, 'date_with_week': { 'python': tarantool.Interval(year=1, month=2, week=3, day=4), 'msgpack': (b'\x05\x00\x01\x01\x02\x02\x03\x03\x04\x08\x01'), 'tarantool': r"datetime.interval.new({year=1, month=2, week=3, day=4})", + 'str': 'tarantool.Interval(year=1, month=2, week=3, day=4, hour=0, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', }, 'datetime_with_week': { 'python': tarantool.Interval(year=1, month=2, week=3, day=4, hour=1, minute=2, @@ -165,6 +195,8 @@ def test_interval_positional_init(self): b'\x07\xce\x00\x98\x96\x80\x08\x01'), 'tarantool': r"datetime.interval.new({year=1, month=2, week=3, day=4, hour=1, " r"min=2, sec=3000, nsec=10000000})", + 'str': 'tarantool.Interval(year=1, month=2, week=3, day=4, hour=1, ' + 'minute=2, sec=3000, nsec=10000000, adjust=Adjust.NONE)', }, } @@ -216,6 +248,12 @@ def test_tarantool_encode(self): self.assertSequenceEqual(self.adm(lua_eval), [True]) + def test_class_string(self): + for name, case in self.cases.items(): + with self.subTest(msg=name): + self.assertEqual(str(case['python']), case['str']) + self.assertEqual(repr(case['python']), case['str']) + def test_unknown_field_decode(self): case = b'\x01\x09\xce\x00\x98\x96\x80' self.assertRaisesRegex( From 5a50fe4c60580f1273c5d23f2ac8287443533585 Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Mon, 17 Jul 2023 17:35:14 +0300 Subject: [PATCH 06/39] interval: validate limits Before this patch, any value was allowed for interval attributes. Now we use the same rules as in Tarantool. A couple of issues were met while developing this patch, follow [1, 2] for core updates. 1. https://github.com/tarantool/tarantool/issues/8878 2. https://github.com/tarantool/tarantool/issues/8887 --- CHANGELOG.md | 3 + tarantool/msgpack_ext/types/interval.py | 58 ++++ test/suites/test_interval.py | 369 ++++++++++++++++++++++++ 3 files changed, 430 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 35b5dcdc..1e9aff0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Changed +- Validate `tarantool.Interval` limits with the same rules as in Tarantool. + ### Fixed - `tarantool.Interval` arithmetic with weeks - `tarantool.Interval` weeks display in `str()` and `repr()` diff --git a/tarantool/msgpack_ext/types/interval.py b/tarantool/msgpack_ext/types/interval.py index 54551a18..e44dcce0 100644 --- a/tarantool/msgpack_ext/types/interval.py +++ b/tarantool/msgpack_ext/types/interval.py @@ -16,6 +16,60 @@ 8: 'adjust', } +# https://github.com/tarantool/tarantool/blob/ff57f990f359f6d7866c1947174d8ba0e97b1ea6/src/lua/datetime.lua#L112-L146 +SECS_PER_DAY = 86400 + +MIN_DATE_YEAR = -5879610 +MIN_DATE_MONTH = 6 +MIN_DATE_DAY = 22 +MAX_DATE_YEAR = 5879611 +MAX_DATE_MONTH = 7 +MAX_DATE_DAY = 11 + +AVERAGE_DAYS_YEAR = 365.25 +AVERAGE_WEEK_YEAR = AVERAGE_DAYS_YEAR / 7 +INT_MAX = 2147483647 +MAX_YEAR_RANGE = MAX_DATE_YEAR - MIN_DATE_YEAR +MAX_MONTH_RANGE = MAX_YEAR_RANGE * 12 +MAX_WEEK_RANGE = MAX_YEAR_RANGE * AVERAGE_WEEK_YEAR +MAX_DAY_RANGE = MAX_YEAR_RANGE * AVERAGE_DAYS_YEAR +MAX_HOUR_RANGE = MAX_DAY_RANGE * 24 +MAX_MIN_RANGE = MAX_HOUR_RANGE * 60 +MAX_SEC_RANGE = MAX_DAY_RANGE * SECS_PER_DAY +MAX_NSEC_RANGE = INT_MAX + +max_val = { + 'year': MAX_YEAR_RANGE, + 'month': MAX_MONTH_RANGE, + 'week': MAX_WEEK_RANGE, + 'day': MAX_DAY_RANGE, + 'hour': MAX_HOUR_RANGE, + 'minute': MAX_MIN_RANGE, + 'sec': MAX_SEC_RANGE, + 'nsec': MAX_NSEC_RANGE, +} + + +def verify_range(intv): + """ + Check allowed values. Approach is the same as in tarantool/tarantool. + + :param intv: Raw interval to verify. + :type intv: :class:`~tarantool.Interval` + + :raise: :exc:`ValueError` + + :meta private: + """ + + for field_name, range_max in max_val.items(): + val = getattr(intv, field_name) + # Tarantool implementation has a bug + # https://github.com/tarantool/tarantool/issues/8878 + if (val > range_max) or (val < -range_max): + raise ValueError(f"value {val} of {field_name} is out of " + f"allowed range [{-range_max}, {range_max}]") + # https://github.com/tarantool/c-dt/blob/cec6acebb54d9e73ea0b99c63898732abd7683a6/dt_arithmetic.h#L34 class Adjust(Enum): @@ -92,6 +146,8 @@ def __init__(self, *, year=0, month=0, week=0, :param adjust: Interval adjustment rule. Refer to :meth:`~tarantool.Datetime.__add__`. :type adjust: :class:`~tarantool.IntervalAdjust`, optional + + :raise: :exc:`ValueError` """ self.year = year @@ -104,6 +160,8 @@ def __init__(self, *, year=0, month=0, week=0, self.nsec = nsec self.adjust = adjust + verify_range(self) + def __add__(self, other): """ Valid operations: diff --git a/test/suites/test_interval.py b/test/suites/test_interval.py index 6917b49c..b2726776 100644 --- a/test/suites/test_interval.py +++ b/test/suites/test_interval.py @@ -13,6 +13,16 @@ from tarantool.error import MsgpackError from tarantool.msgpack_ext.packer import default as packer_default from tarantool.msgpack_ext.unpacker import ext_hook as unpacker_ext_hook +from tarantool.msgpack_ext.types.interval import ( + MAX_YEAR_RANGE, + MAX_MONTH_RANGE, + MAX_WEEK_RANGE, + MAX_DAY_RANGE, + MAX_HOUR_RANGE, + MAX_MIN_RANGE, + MAX_SEC_RANGE, + MAX_NSEC_RANGE, +) from .lib.tarantool_server import TarantoolServer from .lib.skip import skip_or_run_datetime_test @@ -198,6 +208,262 @@ def test_interval_positional_init(self): 'str': 'tarantool.Interval(year=1, month=2, week=3, day=4, hour=1, ' 'minute=2, sec=3000, nsec=10000000, adjust=Adjust.NONE)', }, + 'min_year_interval': { + 'python': tarantool.Interval(year=-int(MAX_YEAR_RANGE)), + 'msgpack': (b'\x02\x00\xd2\xff\x4c\x91\x8b\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{year=-{int(MAX_YEAR_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{year=-{int(MAX_YEAR_RANGE)} + 1}}) - " + r"datetime.interval.new({year=1})", + 'str': f'tarantool.Interval(year=-{int(MAX_YEAR_RANGE)}, month=0, week=0, day=0, ' + 'hour=0, minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'max_year_interval': { + 'python': tarantool.Interval(year=int(MAX_YEAR_RANGE)), + 'msgpack': (b'\x02\x00\xce\x00\xb3\x6e\x75\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{year={int(MAX_YEAR_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{year={int(MAX_YEAR_RANGE)} - 1}}) + " + r"datetime.interval.new({year=1})", + 'str': f'tarantool.Interval(year={int(MAX_YEAR_RANGE)}, month=0, week=0, day=0, ' + 'hour=0, minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'min_month_interval': { + 'python': tarantool.Interval(month=-int(MAX_MONTH_RANGE)), + 'msgpack': (b'\x02\x01\xd2\xf7\x96\xd2\x84\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{month=-{int(MAX_MONTH_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{month=-{int(MAX_MONTH_RANGE)} + 1}}) - " + r"datetime.interval.new({month=1})", + 'str': f'tarantool.Interval(year=0, month=-{int(MAX_MONTH_RANGE)}, week=0, day=0, ' + 'hour=0, minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'max_month_interval': { + 'python': tarantool.Interval(month=int(MAX_MONTH_RANGE)), + 'msgpack': (b'\x02\x01\xce\x08\x69\x2d\x7c\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{month={int(MAX_MONTH_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{month={int(MAX_MONTH_RANGE)} - 1}}) + " + r"datetime.interval.new({month=1})", + 'str': f'tarantool.Interval(year=0, month={int(MAX_MONTH_RANGE)}, week=0, day=0, ' + 'hour=0, minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'min_week_interval': { + 'python': tarantool.Interval(week=-int(MAX_WEEK_RANGE)), + 'msgpack': (b'\x02\x02\xd2\xdb\x6d\x85\xa8\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{week=-{int(MAX_WEEK_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{week=-{int(MAX_WEEK_RANGE)} + 1}}) - " + r"datetime.interval.new({week=1})", + 'str': f'tarantool.Interval(year=0, month=0, week=-{int(MAX_WEEK_RANGE)}, day=0, ' + 'hour=0, minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'max_week_interval': { + 'python': tarantool.Interval(week=int(MAX_WEEK_RANGE)), + 'msgpack': (b'\x02\x02\xce\x24\x92\x7a\x58\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{week={int(MAX_WEEK_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{week={int(MAX_WEEK_RANGE)} - 1}}) + " + r"datetime.interval.new({week=1})", + 'str': f'tarantool.Interval(year=0, month=0, week={int(MAX_WEEK_RANGE)}, day=0, ' + 'hour=0, minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'min_day_interval': { + 'python': tarantool.Interval(day=-int(MAX_DAY_RANGE)), + 'msgpack': (b'\x02\x03\xd3\xff\xff\xff\xfe\xff\xfe\xa7\x92\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{day=-{int(MAX_DAY_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{day=-{int(MAX_DAY_RANGE)} + 1}}) - " + r"datetime.interval.new({day=1})", + 'str': f'tarantool.Interval(year=0, month=0, week=0, day=-{int(MAX_DAY_RANGE)}, ' + 'hour=0, minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + 'tarantool_8887_issue': True, + }, + 'max_day_interval': { + 'python': tarantool.Interval(day=int(MAX_DAY_RANGE)), + 'msgpack': (b'\x02\x03\xcf\x00\x00\x00\x01\x00\x01\x58\x6e\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{day={int(MAX_DAY_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{day={int(MAX_DAY_RANGE)} - 1}}) + " + r"datetime.interval.new({day=1})", + 'str': f'tarantool.Interval(year=0, month=0, week=0, day={int(MAX_DAY_RANGE)}, hour=0, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + 'tarantool_8887_issue': True, + }, + 'min_int32_day_interval': { + 'python': tarantool.Interval(day=-2147483648), + 'msgpack': (b'\x02\x03\xd2\x80\x00\x00\x00\x08\x01'), + 'tarantool': r"datetime.interval.new({day=-2147483648})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=-2147483648, hour=0, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'max_int32_day_interval': { + 'python': tarantool.Interval(day=2147483647), + 'msgpack': (b'\x02\x03\xce\x7f\xff\xff\xff\x08\x01'), + 'tarantool': r"datetime.interval.new({day=2147483647})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=2147483647, hour=0, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'min_hour_interval': { + 'python': tarantool.Interval(hour=-int(MAX_HOUR_RANGE)), + 'msgpack': (b'\x02\x04\xd3\xff\xff\xff\xe7\xff\xdf\xb5\xaa\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{hour=-{int(MAX_HOUR_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{hour=-{int(MAX_HOUR_RANGE)} + 1}}) - " + r"datetime.interval.new({hour=1})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, ' + f'hour=-{int(MAX_HOUR_RANGE)}, minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + 'tarantool_8887_issue': True, + }, + 'max_hour_interval': { + 'python': tarantool.Interval(hour=int(MAX_HOUR_RANGE)), + 'msgpack': (b'\x02\x04\xcf\x00\x00\x00\x18\x00\x20\x4a\x56\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{hour={int(MAX_HOUR_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{hour={int(MAX_HOUR_RANGE)} - 1}}) + " + r"datetime.interval.new({hour=1})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, ' + f'hour={int(MAX_HOUR_RANGE)}, minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + 'tarantool_8887_issue': True, + }, + 'min_int32_hour_interval': { + 'python': tarantool.Interval(hour=-2147483648), + 'msgpack': (b'\x02\x04\xd2\x80\x00\x00\x00\x08\x01'), + 'tarantool': r"datetime.interval.new({hour=-2147483648})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=-2147483648, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'max_int32_hour_interval': { + 'python': tarantool.Interval(hour=2147483647), + 'msgpack': (b'\x02\x04\xce\x7f\xff\xff\xff\x08\x01'), + 'tarantool': r"datetime.interval.new({hour=2147483647})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=2147483647, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'min_minute_interval': { + 'python': tarantool.Interval(minute=-int(MAX_MIN_RANGE)), + 'msgpack': (b'\x02\x05\xd3\xff\xff\xfa\x5f\xf8\x6e\x93\xd8\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{min=-{int(MAX_MIN_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{min=-{int(MAX_MIN_RANGE)} + 1}}) - " + r"datetime.interval.new({min=1})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + f'minute=-{int(MAX_MIN_RANGE)}, sec=0, nsec=0, adjust=Adjust.NONE)', + 'tarantool_8887_issue': True, + }, + 'max_minute_interval': { + 'python': tarantool.Interval(minute=int(MAX_MIN_RANGE)), + 'msgpack': (b'\x02\x05\xcf\x00\x00\x05\xa0\x07\x91\x6c\x28\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{min={int(MAX_MIN_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{min={int(MAX_MIN_RANGE)} - 1}}) + " + r"datetime.interval.new({min=1})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + f'minute={int(MAX_MIN_RANGE)}, sec=0, nsec=0, adjust=Adjust.NONE)', + 'tarantool_8887_issue': True, + }, + 'min_int32_minute_interval': { + 'python': tarantool.Interval(minute=-2147483648), + 'msgpack': (b'\x02\x05\xd2\x80\x00\x00\x00\x08\x01'), + 'tarantool': r"datetime.interval.new({min=-2147483648})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + 'minute=-2147483648, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'max_int32_minute_interval': { + 'python': tarantool.Interval(minute=2147483647), + 'msgpack': (b'\x02\x05\xce\x7f\xff\xff\xff\x08\x01'), + 'tarantool': r"datetime.interval.new({min=2147483647})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + 'minute=2147483647, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'min_sec_interval': { + 'python': tarantool.Interval(sec=-int(MAX_SEC_RANGE)), + 'msgpack': (b'\x02\x06\xd3\xff\xfe\xae\x7e\x39\xea\xa6\xa0\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{sec=-{int(MAX_SEC_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{sec=-{int(MAX_SEC_RANGE)} + 1}}) - " + r"datetime.interval.new({sec=1})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + f'minute=0, sec=-{int(MAX_SEC_RANGE)}, nsec=0, adjust=Adjust.NONE)', + 'tarantool_8887_issue': True, + }, + 'max_sec_interval': { + 'python': tarantool.Interval(sec=int(MAX_SEC_RANGE)), + 'msgpack': (b'\x02\x06\xcf\x00\x01\x51\x81\xc6\x15\x59\x60\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{sec={int(MAX_SEC_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{sec={int(MAX_SEC_RANGE)} - 1}}) + " + r"datetime.interval.new({sec=1})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + f'minute=0, sec={int(MAX_SEC_RANGE)}, nsec=0, adjust=Adjust.NONE)', + 'tarantool_8887_issue': True, + }, + 'min_int32_sec_interval': { + 'python': tarantool.Interval(sec=-2147483648), + 'msgpack': (b'\x02\x06\xd2\x80\x00\x00\x00\x08\x01'), + 'tarantool': r"datetime.interval.new({sec=-2147483648})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + 'minute=0, sec=-2147483648, nsec=0, adjust=Adjust.NONE)', + }, + 'max_int32_sec_interval': { + 'python': tarantool.Interval(sec=2147483647), + 'msgpack': (b'\x02\x06\xce\x7f\xff\xff\xff\x08\x01'), + 'tarantool': r"datetime.interval.new({sec=2147483647})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + 'minute=0, sec=2147483647, nsec=0, adjust=Adjust.NONE)', + }, + 'min_nsec_interval': { + 'python': tarantool.Interval(nsec=-int(MAX_NSEC_RANGE)), + 'msgpack': (b'\x02\x07\xd2\x80\x00\x00\x01\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{nsec=-{int(MAX_NSEC_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{nsec=-{int(MAX_NSEC_RANGE)} + 1}}) - " + r"datetime.interval.new({nsec=1})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + f'minute=0, sec=0, nsec=-{int(MAX_NSEC_RANGE)}, adjust=Adjust.NONE)', + }, + 'max_nsec_interval': { + 'python': tarantool.Interval(nsec=int(MAX_NSEC_RANGE)), + 'msgpack': (b'\x02\x07\xce\x7f\xff\xff\xff\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{nsec={int(MAX_NSEC_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{nsec={int(MAX_NSEC_RANGE)} - 1}}) + " + r"datetime.interval.new({nsec=1})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + f'minute=0, sec=0, nsec={int(MAX_NSEC_RANGE)}, adjust=Adjust.NONE)', + }, } def test_msgpack_decode(self): @@ -215,6 +481,9 @@ def test_msgpack_decode(self): def test_tarantool_decode(self): for name, case in self.cases.items(): with self.subTest(msg=name): + if ('tarantool_8887_issue' in case) and (case['tarantool_8887_issue'] is True): + self.skipTest('See https://github.com/tarantool/tarantool/issues/8887') + self.adm(f"box.space['test']:replace{{'{name}', {case['tarantool']}, 'field'}}") self.assertSequenceEqual(self.con.select('test', name), @@ -230,6 +499,9 @@ def test_msgpack_encode(self): def test_tarantool_encode(self): for name, case in self.cases.items(): with self.subTest(msg=name): + if ('tarantool_8887_issue' in case) and (case['tarantool_8887_issue'] is True): + self.skipTest('See https://github.com/tarantool/tarantool/issues/8887') + self.con.insert('test', [name, case['python'], 'field']) lua_eval = f""" @@ -266,6 +538,87 @@ def test_unknown_adjust_decode(self): MsgpackError, '3 is not a valid Adjust', lambda: unpacker_ext_hook(6, case, self.con._unpacker_factory())) + out_of_range_cases = { + 'year_too_small': { + 'kwargs': {'year': -int(MAX_YEAR_RANGE + 1)}, + 'range': MAX_YEAR_RANGE, + }, + 'year_too_large': { + 'kwargs': {'year': int(MAX_YEAR_RANGE + 1)}, + 'range': MAX_YEAR_RANGE, + }, + 'month_too_small': { + 'kwargs': {'month': -int(MAX_MONTH_RANGE + 1)}, + 'range': MAX_MONTH_RANGE, + }, + 'month_too_big': { + 'kwargs': {'month': int(MAX_MONTH_RANGE + 1)}, + 'range': MAX_MONTH_RANGE, + }, + 'week_too_small': { + 'kwargs': {'week': -int(MAX_WEEK_RANGE + 1)}, + 'range': MAX_WEEK_RANGE, + }, + 'week_too_big': { + 'kwargs': {'week': int(MAX_WEEK_RANGE + 1)}, + 'range': MAX_WEEK_RANGE, + }, + 'day_too_small': { + 'kwargs': {'day': -int(MAX_DAY_RANGE + 1)}, + 'range': MAX_DAY_RANGE, + }, + 'day_too_big': { + 'kwargs': {'day': int(MAX_DAY_RANGE + 1)}, + 'range': MAX_DAY_RANGE, + }, + 'hour_too_small': { + 'kwargs': {'hour': -int(MAX_HOUR_RANGE + 1)}, + 'range': MAX_HOUR_RANGE, + }, + 'hour_too_big': { + 'kwargs': {'hour': int(MAX_HOUR_RANGE + 1)}, + 'range': MAX_HOUR_RANGE, + }, + 'minute_too_small': { + 'kwargs': {'minute': -int(MAX_MIN_RANGE + 1)}, + 'range': MAX_MIN_RANGE, + }, + 'minute_too_big': { + 'kwargs': {'minute': int(MAX_MIN_RANGE + 1)}, + 'range': MAX_MIN_RANGE, + }, + 'sec_too_small': { + 'kwargs': {'sec': -int(MAX_SEC_RANGE + 1)}, + 'range': MAX_SEC_RANGE, + }, + 'sec_too_big': { + 'kwargs': {'sec': int(MAX_SEC_RANGE + 1)}, + 'range': MAX_SEC_RANGE, + }, + 'nsec_too_small': { + 'kwargs': {'nsec': -int(MAX_NSEC_RANGE + 1)}, + 'range': MAX_NSEC_RANGE, + }, + 'nsec_too_big': { + 'kwargs': {'nsec': int(MAX_NSEC_RANGE + 1)}, + 'range': MAX_NSEC_RANGE, + }, + } + + def test_out_of_range(self): + # pylint: disable=cell-var-from-loop + + for name, case in self.out_of_range_cases.items(): + with self.subTest(msg=name): + name = next(iter(case['kwargs'])) + val = case['kwargs'][name] + self.assertRaisesRegex( + ValueError, re.escape( + f"value {val} of {name} is out of " + f"allowed range [{-case['range']}, {case['range']}]" + ), + lambda: tarantool.Interval(**case['kwargs'])) + arithmetic_cases = { 'year': { 'arg_1': tarantool.Interval(year=2), @@ -369,6 +722,22 @@ def test_tarantool_interval_subtraction(self): self.assertSequenceEqual(self.con.call('sub', case['arg_1'], case['arg_2']), [case['res_sub']]) + def test_addition_overflow(self): + self.assertRaisesRegex( + ValueError, re.escape( + f"value {int(MAX_YEAR_RANGE) + 1} of year is out of " + f"allowed range [{-MAX_YEAR_RANGE}, {MAX_YEAR_RANGE}]" + ), + lambda: tarantool.Interval(year=int(MAX_YEAR_RANGE)) + tarantool.Interval(year=1)) + + def test_subtraction_overflow(self): + self.assertRaisesRegex( + ValueError, re.escape( + f"value {-int(MAX_YEAR_RANGE) - 1} of year is out of " + f"allowed range [{-MAX_YEAR_RANGE}, {MAX_YEAR_RANGE}]" + ), + lambda: tarantool.Interval(year=-int(MAX_YEAR_RANGE)) - tarantool.Interval(year=1)) + @classmethod def tearDownClass(cls): cls.con.close() From 3a8e39fd826c47fb6245ce1054fddd50ee3d0ea9 Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Tue, 18 Jul 2023 10:20:47 +0300 Subject: [PATCH 07/39] changelog: fix spelling --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e9aff0e..19ea2b26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -207,7 +207,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Support pandas way to build datetime from timestamp (PR #252). `timestamp_since_utc_epoch` is a parameter to set timestamp - convertion behavior for timezone-aware datetimes. + conversion behavior for timezone-aware datetimes. If ``False`` (default), behaves similar to Tarantool `datetime.new()`: @@ -227,7 +227,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Thus, if ``False``, datetime is computed from timestamp since epoch and then timezone is applied without any - convertion. In that case, `dt.timestamp` won't be equal to + conversion. In that case, `dt.timestamp` won't be equal to initialization `timestamp` for all timezones with non-zero offset. If ``True``, behaves similar to `pandas.Timestamp`: From f2104eef46ea9277e8a1032daaae59f21eab28b7 Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Tue, 18 Jul 2023 10:18:51 +0300 Subject: [PATCH 08/39] lint: validate changelog with codespell --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 25809cd8..d3919795 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ install: PYTHON_FILES=tarantool test setup.py docs/source/conf.py -TEXT_FILES=README.rst docs/source/*.rst +TEXT_FILES=README.rst CHANGELOG.md docs/source/*.rst .PHONY: lint lint: python3 -m pylint --recursive=y $(PYTHON_FILES) From 21f77a6bd23741bdb90dea8d0aed315af45b5e0a Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Wed, 19 Jul 2023 17:21:22 +0300 Subject: [PATCH 09/39] release 1.1.1 Overview This release introduces various datetime interval fixes and quality of life improvements. Breaking changes - Forbid to create datetime intervals out of Tarantool limits (PR #302). Changed - Validate `tarantool.Interval` limits with the same rules as in Tarantool (PR #302). Fixed - `tarantool.Interval` arithmetic with weeks (PR #302). - `tarantool.Interval` weeks display in `str()` and `repr()` (PR #302). --- CHANGELOG.md | 8 ++++---- debian/changelog | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19ea2b26..13e58a2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,14 +4,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Unreleased +## 1.1.1 - 2023-07-19 ### Changed -- Validate `tarantool.Interval` limits with the same rules as in Tarantool. +- Validate `tarantool.Interval` limits with the same rules as in Tarantool (PR #302). ### Fixed -- `tarantool.Interval` arithmetic with weeks -- `tarantool.Interval` weeks display in `str()` and `repr()` +- `tarantool.Interval` arithmetic with weeks (PR #302). +- `tarantool.Interval` weeks display in `str()` and `repr()` (PR #302). ## 1.1.0 - 2023-06-30 diff --git a/debian/changelog b/debian/changelog index 3d653acf..429c65a1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,22 @@ +python3-tarantool (1.1.1-0) unstable; urgency=medium + + ## Overview + + This release introduces various datetime interval fixes and quality of life + improvements. + + ## Breaking changes + - Forbid to create datetime intervals out of Tarantool limits (PR #302). + + ## Changed + - Validate `tarantool.Interval` limits with the same rules as in Tarantool (PR #302). + + ## Fixed + - `tarantool.Interval` arithmetic with weeks (PR #302). + - `tarantool.Interval` weeks display in `str()` and `repr()` (PR #302). + + -- Georgy Moiseev Wed, 19 Jul 2023 18:00:00 +0300 + python3-tarantool (1.1.0-0) unstable; urgency=medium ## Overview From a64c6e0b5e7b803f30d5b074c30edeba75814913 Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Mon, 7 Aug 2023 17:41:00 +0300 Subject: [PATCH 10/39] test: wait until server is ready Current `wait_until_started` method wait till `box.info.status` is "running" and start sending requests. Such approach is wrong: status "running" [1] does not guarantees that instance is available for writing (like creating new users) [2]. It results in flaky test results (for example, see 4 restarts in [3]). This patch fixes the issue. After this patch, it is guaranteed that after `wait_until_ready` call instance had finished to run its initialization script, including box.cfg, and available for writing data if it is expected to be able to. Windows instances use a separate lock mechanism, so this patch is not related to Windows instances. 1. https://github.com/tarantool/tarantool/blob/983a7ec215d46b6d02935d1baa8bbe07fc371795/src/box/box.cc#L5425-L5426 2. https://github.com/tarantool/tarantool/blob/983a7ec215d46b6d02935d1baa8bbe07fc371795/src/box/box.cc#L5454 3. https://github.com/tarantool/tarantool/actions/runs/5759802620 --- test/suites/box.lua | 2 ++ test/suites/crud_server.lua | 2 ++ test/suites/lib/tarantool_server.py | 21 ++++++++------------- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/test/suites/box.lua b/test/suites/box.lua index eb4330dd..c8110d18 100644 --- a/test/suites/box.lua +++ b/test/suites/box.lua @@ -19,3 +19,5 @@ box.cfg{ pid_file = "box.pid", auth_type = (auth_type:len() > 0) and auth_type or nil, } + +rawset(_G, 'ready', true) diff --git a/test/suites/crud_server.lua b/test/suites/crud_server.lua index 8bb58fcb..f939818a 100644 --- a/test/suites/crud_server.lua +++ b/test/suites/crud_server.lua @@ -91,3 +91,5 @@ if crud_imported == false or vshard_imported == false then else configure_crud_instance(primary_listen, crud, vshard) end + +rawset(_G, 'ready', true) diff --git a/test/suites/lib/tarantool_server.py b/test/suites/lib/tarantool_server.py index 1dd5e053..c8f34b37 100644 --- a/test/suites/lib/tarantool_server.py +++ b/test/suites/lib/tarantool_server.py @@ -302,27 +302,22 @@ def prepare_args(self): return shlex.split(self.binary if not self.script else self.script_dst) - def wait_until_started(self): + def wait_until_ready(self): """ - Wait until server is started. + Wait until server is configured and ready to work. Server consists of two parts: 1) wait until server is listening on sockets - 2) wait until server tells us his status + 2) wait until server finishes executing its script """ while True: try: temp = TarantoolAdmin('0.0.0.0', self.args['admin']) - while True: - ans = temp('box.info.status')[0] - if ans in ('running', 'hot_standby', 'orphan') or ans.startswith('replica'): - temp.disconnect() - return True - if ans in ('loading',): - continue - - raise ValueError(f"Strange output for `box.info.status`: {ans}") + ans = temp('ready')[0] + temp.disconnect() + if isinstance(ans, bool) and ans: + return True except socket.error as exc: if exc.errno == errno.ECONNREFUSED: time.sleep(0.1) @@ -352,7 +347,7 @@ def start(self): cwd=self.vardir, stdout=self.log_des, stderr=self.log_des) - self.wait_until_started() + self.wait_until_ready() def stop(self): """ From ed8d3d2673cf2cdbba2add458cb1d7827a956085 Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Mon, 7 Aug 2023 17:42:34 +0300 Subject: [PATCH 11/39] ci: use WSL with Ubuntu 22.04 Some installation certificates are outdated for Ubuntu 22.04. --- .github/workflows/packing.yml | 8 ++++---- .github/workflows/testing.yml | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/packing.yml b/.github/workflows/packing.yml index eabf5096..b5b84f32 100644 --- a/.github/workflows/packing.yml +++ b/.github/workflows/packing.yml @@ -148,16 +148,16 @@ jobs: - name: Setup WSL for tarantool uses: Vampire/setup-wsl@v1 with: - distribution: Ubuntu-20.04 + distribution: Ubuntu-22.04 - name: Install tarantool - shell: wsl-bash_Ubuntu-20.04 {0} + shell: wsl-bash_Ubuntu-22.04 {0} run: | curl -L https://tarantool.io/release/2/installer.sh | bash -s sudo apt install -y tarantool tarantool-dev - name: Setup test tarantool instance - shell: wsl-bash_Ubuntu-20.04 {0} + shell: wsl-bash_Ubuntu-22.04 {0} run: | rm -f ./tarantool.pid ./tarantool.log TNT_PID=$(tarantool ./test/suites/lib/tarantool_python_ci.lua > tarantool.log 2>&1 & echo $!) @@ -172,7 +172,7 @@ jobs: - name: Stop test tarantool instance if: ${{ always() }} - shell: wsl-bash_Ubuntu-20.04 {0} + shell: wsl-bash_Ubuntu-22.04 {0} run: | cat tarantool.log || true kill $(cat tarantool.pid) || true diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 199744f2..88fa08d7 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -262,16 +262,16 @@ jobs: - name: Setup WSL for tarantool uses: Vampire/setup-wsl@v2 with: - distribution: Ubuntu-20.04 + distribution: Ubuntu-22.04 - name: Install tarantool ${{ matrix.tarantool }} for WSL (2.10 and newer) - shell: wsl-bash_Ubuntu-20.04 {0} + shell: wsl-bash_Ubuntu-22.04 {0} run: | curl -L https://tarantool.io/release/2/installer.sh | bash -s sudo apt install -y tarantool=${{ matrix.tarantool }} tarantool-dev=${{ matrix.tarantool }} - name: Setup test tarantool instance - shell: wsl-bash_Ubuntu-20.04 {0} + shell: wsl-bash_Ubuntu-22.04 {0} run: | rm -f ./tarantool.pid ./tarantool.log TNT_PID=$(tarantool ./test/suites/lib/tarantool_python_ci.lua > tarantool.log 2>&1 & echo $!) @@ -286,7 +286,7 @@ jobs: - name: Stop test tarantool instance if: ${{ always() }} - shell: wsl-bash_Ubuntu-20.04 {0} + shell: wsl-bash_Ubuntu-22.04 {0} run: | cat tarantool.log || true kill $(cat tarantool.pid) || true @@ -334,16 +334,16 @@ jobs: - name: Setup WSL for tarantool uses: Vampire/setup-wsl@v2 with: - distribution: Ubuntu-20.04 + distribution: Ubuntu-22.04 - name: Install tarantool ${{ matrix.tarantool }} for WSL - shell: wsl-bash_Ubuntu-20.04 {0} + shell: wsl-bash_Ubuntu-22.04 {0} run: | curl -L https://tarantool.io/release/2/installer.sh | bash -s sudo apt install -y tarantool=${{ matrix.tarantool }} tarantool-dev=${{ matrix.tarantool }} - name: Setup test tarantool instance - shell: wsl-bash_Ubuntu-20.04 {0} + shell: wsl-bash_Ubuntu-22.04 {0} run: | rm -f ./tarantool.pid ./tarantool.log TNT_PID=$(tarantool ./test/suites/lib/tarantool_python_ci.lua > tarantool.log 2>&1 & echo $!) @@ -358,7 +358,7 @@ jobs: - name: Stop test tarantool instance if: ${{ always() }} - shell: wsl-bash_Ubuntu-20.04 {0} + shell: wsl-bash_Ubuntu-22.04 {0} run: | cat tarantool.log || true kill $(cat tarantool.pid) || true From 42bbe146e8cf3e7e4162f87324ec350a5d33b526 Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Mon, 7 Aug 2023 19:24:25 +0300 Subject: [PATCH 12/39] test: assert admin commands success Admin command are sent through text protocol. Responses are rarely processed, especially those which sets up schema and users. But such requests can contain errors, like "already exists" or syntax ones. Having asserts on schema set up makes it easier to debug some cases. For example, it was helpful in discovering the reason behind "user not exists" fails fixed in "wait until server is ready" patchset commit [1]. 1. https://github.com/tarantool/tarantool-python/actions/runs/5784619729/job/15675655683?pr=306 --- test/suites/test_connection.py | 71 +++++++++++++-------- test/suites/test_datetime.py | 23 +++++-- test/suites/test_dbapi.py | 13 ++-- test/suites/test_decimal.py | 29 ++++++--- test/suites/test_dml.py | 62 ++++++++++-------- test/suites/test_encoding.py | 84 +++++++++++++----------- test/suites/test_error_ext.py | 20 ++++-- test/suites/test_execute.py | 13 ++-- test/suites/test_interval.py | 20 ++++-- test/suites/test_mesh.py | 29 ++++++--- test/suites/test_pool.py | 60 ++++++++++------- test/suites/test_push.py | 94 ++++++++++++--------------- test/suites/test_schema.py | 113 ++++++++++++++++++--------------- test/suites/test_uuid.py | 39 ++++++++---- test/suites/utils.py | 16 +++++ 15 files changed, 419 insertions(+), 267 deletions(-) create mode 100644 test/suites/utils.py diff --git a/test/suites/test_connection.py b/test/suites/test_connection.py index 52234608..4402f0b0 100644 --- a/test/suites/test_connection.py +++ b/test/suites/test_connection.py @@ -5,8 +5,9 @@ import sys import unittest - import decimal + +import pkg_resources import msgpack import tarantool @@ -14,6 +15,7 @@ from .lib.skip import skip_or_run_decimal_test, skip_or_run_varbinary_test from .lib.tarantool_server import TarantoolServer +from .utils import assert_admin_success class TestSuiteConnection(unittest.TestCase): @@ -26,35 +28,48 @@ def setUpClass(cls): cls.srv.start() cls.adm = cls.srv.admin - cls.adm(r""" + resp = cls.adm(""" box.schema.user.create('test', {password = 'test', if_not_exists = true}) - box.schema.user.grant('test', 'read,write,execute', 'universe') - - box.schema.create_space('space_varbin') - - box.space['space_varbin']:format({ - { - 'id', - type = 'number', - is_nullable = false - }, - { - 'varbin', - type = 'varbinary', - is_nullable = false, - } - }) - - box.space['space_varbin']:create_index('id', { - type = 'tree', - parts = {1, 'number'}, - unique = true}) - - box.space['space_varbin']:create_index('varbin', { - type = 'tree', - parts = {2, 'varbinary'}, - unique = true}) + box.schema.user.grant('test', 'read,write,execute', 'universe', + nil, {if_not_exists = true}) + + return true """) + assert_admin_success(resp) + + if cls.srv.admin.tnt_version >= pkg_resources.parse_version('2.2.1'): + resp = cls.adm(""" + box.schema.create_space('space_varbin', {if_not_exists = true}) + + box.space['space_varbin']:format({ + { + 'id', + type = 'number', + is_nullable = false + }, + { + 'varbin', + type = 'varbinary', + is_nullable = false, + } + }) + + box.space['space_varbin']:create_index('id', { + type = 'tree', + parts = {1, 'number'}, + unique = true, + if_not_exists = true}) + + box.space['space_varbin']:create_index('varbin', { + type = 'tree', + parts = {2, 'varbinary'}, + unique = true, + if_not_exists = true}) + + return true + """) + assert_admin_success(resp) + cls.con = None def setUp(self): diff --git a/test/suites/test_datetime.py b/test/suites/test_datetime.py index ae22dcfc..9f250804 100644 --- a/test/suites/test_datetime.py +++ b/test/suites/test_datetime.py @@ -16,6 +16,7 @@ from .lib.tarantool_server import TarantoolServer from .lib.skip import skip_or_run_datetime_test, skip_or_run_datetime_2_11_test +from .utils import assert_admin_success class TestSuiteDatetime(unittest.TestCase): @@ -28,25 +29,28 @@ def setUpClass(cls): cls.srv.start() cls.adm = cls.srv.admin - cls.adm(r""" + resp = cls.adm(""" _, datetime = pcall(require, 'datetime') - box.schema.space.create('test') + box.schema.space.create('test', {if_not_exists = true}) box.space['test']:create_index('primary', { type = 'tree', parts = {1, 'string'}, - unique = true}) + unique = true, + if_not_exists = true}) pcall(function() box.schema.space.create('test_pk') box.space['test_pk']:create_index('primary', { type = 'tree', parts = {1, 'datetime'}, - unique = true}) + unique = true, + if_not_exists = true}) end) box.schema.user.create('test', {password = 'test', if_not_exists = true}) - box.schema.user.grant('test', 'read,write,execute', 'universe') + box.schema.user.grant('test', 'read,write,execute', 'universe', + nil, {if_not_exists = true}) local function add(arg1, arg2) return arg1 + arg2 @@ -57,7 +61,10 @@ def setUpClass(cls): return arg1 - arg2 end rawset(_G, 'sub', sub) + + return true """) + assert_admin_success(resp) cls.con = tarantool.Connection(cls.srv.host, cls.srv.args['primary'], user='test', password='test') @@ -67,7 +74,11 @@ def setUp(self): if self.srv.is_started(): self.srv.touch_lock() - self.adm("box.space['test']:truncate()") + resp = self.adm(""" + box.space['test']:truncate() + return true + """) + assert_admin_success(resp) def test_datetime_class_api(self): datetime = tarantool.Datetime(year=2022, month=8, day=31, hour=18, minute=7, sec=54, diff --git a/test/suites/test_dbapi.py b/test/suites/test_dbapi.py index 83156b39..bd8c7487 100644 --- a/test/suites/test_dbapi.py +++ b/test/suites/test_dbapi.py @@ -12,6 +12,7 @@ from tarantool import dbapi from .lib.tarantool_server import TarantoolServer from .lib.skip import skip_or_run_sql_test +from .utils import assert_admin_success class TestSuiteDBAPI(dbapi20.DatabaseAPI20Test): @@ -39,6 +40,14 @@ def setUpClass(cls): "port": cls.srv.args['primary'] } + # grant full access to guest + resp = cls.srv.admin(""" + box.schema.user.grant('guest', 'create,read,write,execute', 'universe', + nil, {if_not_exists = true}) + return true + """) + assert_admin_success(resp) + @skip_or_run_sql_test def setUp(self): # prevent a remote tarantool from clean our session @@ -46,10 +55,6 @@ def setUp(self): self.srv.touch_lock() self.con.flush_schema() - # grant full access to guest - self.srv.admin("box.schema.user.grant('guest', 'create,read,write," - "execute', 'universe')") - @classmethod def tearDownClass(cls): cls.con.close() diff --git a/test/suites/test_decimal.py b/test/suites/test_decimal.py index b880eaa3..2875a7da 100644 --- a/test/suites/test_decimal.py +++ b/test/suites/test_decimal.py @@ -16,6 +16,7 @@ from .lib.tarantool_server import TarantoolServer from .lib.skip import skip_or_run_decimal_test +from .utils import assert_admin_success class TestSuiteDecimal(unittest.TestCase): @@ -28,26 +29,32 @@ def setUpClass(cls): cls.srv.start() cls.adm = cls.srv.admin - cls.adm(r""" - _, decimal = pcall(require, 'decimal') + resp = cls.adm(""" + decimal_supported, decimal = pcall(require, 'decimal') - box.schema.space.create('test') + box.schema.space.create('test', {if_not_exists = true}) box.space['test']:create_index('primary', { type = 'tree', parts = {1, 'string'}, - unique = true}) + unique = true, + if_not_exists = true}) - pcall(function() + if decimal_supported then box.schema.space.create('test_pk') box.space['test_pk']:create_index('primary', { type = 'tree', parts = {1, 'decimal'}, - unique = true}) - end) + unique = true, + if_not_exists = true}) + end box.schema.user.create('test', {password = 'test', if_not_exists = true}) - box.schema.user.grant('test', 'read,write,execute', 'universe') + box.schema.user.grant('test', 'read,write,execute', 'universe', + nil, {if_not_exists = true}) + + return true """) + assert_admin_success(resp) cls.con = tarantool.Connection(cls.srv.host, cls.srv.args['primary'], user='test', password='test') @@ -57,7 +64,11 @@ def setUp(self): if self.srv.is_started(): self.srv.touch_lock() - self.adm("box.space['test']:truncate()") + resp = self.adm(""" + box.space['test']:truncate() + return true + """) + assert_admin_success(resp) valid_cases = { 'simple_decimal_1': { diff --git a/test/suites/test_dml.py b/test/suites/test_dml.py index 69293b55..f557451e 100644 --- a/test/suites/test_dml.py +++ b/test/suites/test_dml.py @@ -10,6 +10,7 @@ from .lib.tarantool_server import TarantoolServer from .lib.skip import skip_or_run_error_extra_info_test +from .utils import assert_admin_success class TestSuiteRequest(unittest.TestCase): @@ -22,29 +23,35 @@ def setUpClass(cls): cls.srv.start() cls.con = tarantool.Connection(cls.srv.host, cls.srv.args['primary']) cls.adm = cls.srv.admin - cls.space_created = cls.adm("box.schema.create_space('space_1')") - cls.adm(""" - box.space['space_1']:create_index('primary', { - type = 'tree', - parts = {1, 'num'}, - unique = true}) - """.replace('\n', ' ')) - cls.adm(""" - box.space['space_1']:create_index('secondary', { - type = 'tree', - parts = {2, 'num', 3, 'str'}, - unique = false}) - """.replace('\n', ' ')) - cls.space_created = cls.adm("box.schema.create_space('space_2')") - cls.adm(""" - box.space['space_2']:create_index('primary', { - type = 'hash', - parts = {1, 'num'}, - unique = true}) - """.replace('\n', ' ')) - cls.adm("json = require('json')") - cls.adm("fiber = require('fiber')") - cls.adm("uuid = require('uuid')") + cls.space_created = cls.adm("box.schema.create_space('space_1', {if_not_exists = true})") + resp = cls.adm(""" + box.space['space_1']:create_index('primary', { + type = 'tree', + parts = {1, 'num'}, + unique = true, + if_not_exists = true}) + + box.space['space_1']:create_index('secondary', { + type = 'tree', + parts = {2, 'num', 3, 'str'}, + unique = false, + if_not_exists = true}) + + box.schema.create_space('space_2', {if_not_exists = true}) + + box.space['space_2']:create_index('primary', { + type = 'hash', + parts = {1, 'num'}, + unique = true, + if_not_exists = true}) + + json = require('json') + fiber = require('fiber') + uuid = require('uuid') + + return true + """) + assert_admin_success(resp) if not sys.platform.startswith("win"): cls.sock_srv = TarantoolServer(create_unix_socket=True) @@ -60,10 +67,11 @@ def setUp(self): def test_00_00_authenticate(self): self.assertIsNone(self.srv.admin(""" - box.schema.user.create('test', { password = 'test' }) + box.schema.user.create('test', { password = 'test', if_not_exists = true }) """)) self.assertIsNone(self.srv.admin(""" - box.schema.user.grant('test', 'execute,read,write', 'universe') + box.schema.user.grant('test', 'execute,read,write', 'universe', + nil, {if_not_exists = true}) """)) self.assertEqual(self.con.authenticate('test', 'test')._data, None) @@ -311,7 +319,7 @@ def test_11_select_all_hash(self): space.select((), iterator=tarantool.const.ITERATOR_EQ) def test_12_update_fields(self): - self.srv.admin( + resp = self.srv.admin( """ do local sp = box.schema.create_space('sp', { @@ -325,7 +333,9 @@ def test_12_update_fields(self): parts = {1, 'unsigned'} }) end + return true """) + assert_admin_success(resp) self.con.insert('sp', [2, 'help', 4]) self.assertSequenceEqual( self.con.update('sp', (2,), [('+', 'thi', 3)]), diff --git a/test/suites/test_encoding.py b/test/suites/test_encoding.py index 14a08f54..dcda5983 100644 --- a/test/suites/test_encoding.py +++ b/test/suites/test_encoding.py @@ -1,16 +1,18 @@ """ This module tests various type encoding cases. """ -# pylint: disable=missing-class-docstring,missing-function-docstring +# pylint: disable=missing-class-docstring,missing-function-docstring,duplicate-code import sys import unittest +import pkg_resources import tarantool from tarantool.error import DatabaseError from .lib.skip import skip_or_run_varbinary_test, skip_or_run_error_extra_info_test from .lib.tarantool_server import TarantoolServer +from .utils import assert_admin_success class TestSuiteEncoding(unittest.TestCase): @@ -24,10 +26,14 @@ def setUpClass(cls): cls.srv.script = 'test/suites/box.lua' cls.srv.start() - cls.srv.admin(""" - box.schema.user.create('test', { password = 'test' }) - box.schema.user.grant('test', 'execute,read,write', 'universe') + resp = cls.srv.admin(""" + box.schema.user.create('test', { password = 'test', if_not_exists = true }) + box.schema.user.grant('test', 'execute,read,write', 'universe', + nil, {if_not_exists = true}) + + return true """) + assert_admin_success(resp) args = [cls.srv.host, cls.srv.args['primary']] kwargs = {'user': 'test', 'password': 'test'} @@ -35,41 +41,47 @@ def setUpClass(cls): cls.con_encoding_none = tarantool.Connection(*args, encoding=None, **kwargs) cls.conns = [cls.con_encoding_utf8, cls.con_encoding_none] - cls.srv.admin("box.schema.create_space('space_str')") - cls.srv.admin(""" + resp = cls.srv.admin(""" + box.schema.create_space('space_str', {if_not_exists = true}) box.space['space_str']:create_index('primary', { type = 'tree', parts = {1, 'str'}, - unique = true}) - """.replace('\n', ' ')) - - cls.srv.admin("box.schema.create_space('space_varbin')") - cls.srv.admin(r""" - box.space['space_varbin']:format({ - { - 'id', - type = 'number', - is_nullable = false - }, - { - 'varbin', - type = 'varbinary', - is_nullable = false, - } - }) - """.replace('\n', ' ')) - cls.srv.admin(""" - box.space['space_varbin']:create_index('id', { - type = 'tree', - parts = {1, 'number'}, - unique = true}) - """.replace('\n', ' ')) - cls.srv.admin(""" - box.space['space_varbin']:create_index('varbin', { - type = 'tree', - parts = {2, 'varbinary'}, - unique = true}) - """.replace('\n', ' ')) + unique = true, + if_not_exists = true}) + + return true + """) + assert_admin_success(resp) + + if cls.srv.admin.tnt_version >= pkg_resources.parse_version('2.2.1'): + resp = cls.srv.admin(""" + box.schema.create_space('space_varbin', {if_not_exists = true}) + box.space['space_varbin']:format({ + { + 'id', + type = 'number', + is_nullable = false + }, + { + 'varbin', + type = 'varbinary', + is_nullable = false, + } + }) + box.space['space_varbin']:create_index('id', { + type = 'tree', + parts = {1, 'number'}, + unique = true, + if_not_exists = true}) + box.space['space_varbin']:create_index('varbin', { + type = 'tree', + parts = {2, 'varbinary'}, + unique = true, + if_not_exists = true}) + + return true + """) + assert_admin_success(resp) def assertNotRaises(self, func, *args, **kwargs): try: diff --git a/test/suites/test_error_ext.py b/test/suites/test_error_ext.py index 8d1a63cf..52622794 100644 --- a/test/suites/test_error_ext.py +++ b/test/suites/test_error_ext.py @@ -15,6 +15,7 @@ from .lib.tarantool_server import TarantoolServer from .lib.skip import skip_or_run_error_ext_type_test +from .utils import assert_admin_success class TestSuiteErrorExt(unittest.TestCase): @@ -27,18 +28,23 @@ def setUpClass(cls): cls.srv.start() cls.adm = cls.srv.admin - cls.adm(r""" - box.schema.space.create('test') + resp = cls.adm(""" + box.schema.space.create('test', {if_not_exists = true}) box.space['test']:create_index('primary', { type = 'tree', parts = {1, 'string'}, - unique = true}) + unique = true, + if_not_exists = true}) box.schema.user.create('test', {password = 'test', if_not_exists = true}) - box.schema.user.grant('test', 'read,write,execute,create', 'universe') + box.schema.user.grant('test', 'read,write,execute,create', 'universe', + nil, {if_not_exists = true}) box.schema.user.create('no_grants', {if_not_exists = true}) + + return true """) + assert_admin_success(resp) cls.conn_encoding_utf8 = tarantool.Connection( cls.srv.host, cls.srv.args['primary'], @@ -78,7 +84,11 @@ def setUp(self): if self.srv.is_started(): self.srv.touch_lock() - self.adm("box.space['test']:truncate()") + resp = self.adm(""" + box.space['test']:truncate() + return true + """) + assert_admin_success(resp) # msgpack data for different encodings are actually the same, # but sometimes python msgpack module use different string diff --git a/test/suites/test_execute.py b/test/suites/test_execute.py index 66fe0982..46e85ce7 100644 --- a/test/suites/test_execute.py +++ b/test/suites/test_execute.py @@ -9,6 +9,7 @@ import tarantool from .lib.tarantool_server import TarantoolServer from .lib.skip import skip_or_run_sql_test +from .utils import assert_admin_success class TestSuiteExecute(unittest.TestCase): @@ -32,6 +33,14 @@ def setUpClass(cls): cls.srv.start() cls.con = tarantool.Connection(cls.srv.host, cls.srv.args['primary']) + # grant full access to guest + resp = cls.srv.admin(""" + box.schema.user.grant('guest', 'create,read,write,execute', 'universe', + nil, {if_not_exists = true}) + return true + """) + assert_admin_success(resp) + @skip_or_run_sql_test def setUp(self): # prevent a remote tarantool from clean our session @@ -39,10 +48,6 @@ def setUp(self): self.srv.touch_lock() self.con.flush_schema() - # grant full access to guest - self.srv.admin("box.schema.user.grant('guest', 'create,read,write," - "execute', 'universe')") - @classmethod def tearDownClass(cls): cls.con.close() diff --git a/test/suites/test_interval.py b/test/suites/test_interval.py index b2726776..c241b312 100644 --- a/test/suites/test_interval.py +++ b/test/suites/test_interval.py @@ -26,6 +26,7 @@ from .lib.tarantool_server import TarantoolServer from .lib.skip import skip_or_run_datetime_test +from .utils import assert_admin_success class TestSuiteInterval(unittest.TestCase): @@ -38,17 +39,19 @@ def setUpClass(cls): cls.srv.start() cls.adm = cls.srv.admin - cls.adm(r""" + resp = cls.adm(""" _, datetime = pcall(require, 'datetime') - box.schema.space.create('test') + box.schema.space.create('test', {if_not_exists = true}) box.space['test']:create_index('primary', { type = 'tree', parts = {1, 'string'}, - unique = true}) + unique = true, + if_not_exists = true}) box.schema.user.create('test', {password = 'test', if_not_exists = true}) - box.schema.user.grant('test', 'read,write,execute', 'universe') + box.schema.user.grant('test', 'read,write,execute', 'universe', + nil, {if_not_exists = true}) local function add(arg1, arg2) return arg1 + arg2 @@ -59,7 +62,10 @@ def setUpClass(cls): return arg1 - arg2 end rawset(_G, 'sub', sub) + + return true """) + assert_admin_success(resp) cls.con = tarantool.Connection(cls.srv.host, cls.srv.args['primary'], user='test', password='test') @@ -69,7 +75,11 @@ def setUp(self): if self.srv.is_started(): self.srv.touch_lock() - self.adm("box.space['test']:truncate()") + resp = self.adm(""" + box.space['test']:truncate() + return true + """) + assert_admin_success(resp) def test_interval_positional_init(self): self.assertRaisesRegex( diff --git a/test/suites/test_mesh.py b/test/suites/test_mesh.py index b82ccc0e..606cc7d1 100644 --- a/test/suites/test_mesh.py +++ b/test/suites/test_mesh.py @@ -16,18 +16,23 @@ ClusterDiscoveryWarning, ) from .lib.tarantool_server import TarantoolServer +from .utils import assert_admin_success def create_server(_id): srv = TarantoolServer() srv.script = 'test/suites/box.lua' srv.start() - srv.admin("box.schema.user.create('test', {password = 'test', " - "if_not_exists = true})") - srv.admin("box.schema.user.grant('test', 'execute', 'universe')") + resp = srv.admin(f""" + box.schema.user.create('test', {{password = 'test', if_not_exists = true}}) + box.schema.user.grant('test', 'execute', 'universe', + nil, {{if_not_exists = true}}) - # Create srv_id function (for testing purposes). - srv.admin(f"function srv_id() return {_id} end") + function srv_id() return {_id} end + + return true + """) + assert_admin_success(resp) return srv @@ -43,18 +48,22 @@ def define_cluster_function(self, func_name, servers): function {func_name}() return {{{addresses_lua}}} end + return true """ for srv in self.servers: - srv.admin(func_body) + resp = srv.admin(func_body) + assert_admin_success(resp) def define_custom_cluster_function(self, func_name, retval): func_body = f""" function {func_name}() return {retval} end + return true """ for srv in self.servers: - srv.admin(func_body) + resp = srv.admin(func_body) + assert_admin_success(resp) @classmethod def setUpClass(cls): @@ -99,7 +108,11 @@ def assert_srv_id(con, srv_id): # Start instance#1, stop instance#2 -- response from # instance#1 again. self.srv.start() - self.srv.admin('function srv_id() return 1 end') + resp = self.srv.admin(""" + function srv_id() return 1 end + return true + """) + assert_admin_success(resp) self.srv2.stop() assert_srv_id(con, 1) diff --git a/test/suites/test_pool.py b/test/suites/test_pool.py index 7134f220..e15bcbbf 100644 --- a/test/suites/test_pool.py +++ b/test/suites/test_pool.py @@ -20,31 +20,46 @@ from .lib.skip import skip_or_run_sql_test from .lib.tarantool_server import TarantoolServer +from .utils import assert_admin_success def create_server(_id): srv = TarantoolServer() srv.script = 'test/suites/box.lua' srv.start() - srv.admin("box.schema.user.create('test', {password = 'test', " - "if_not_exists = true})") - srv.admin("box.schema.user.grant('test', 'execute', 'universe')") - srv.admin("box.schema.space.create('test')") - srv.admin(r"box.space.test:format({" - r" { name = 'pk', type = 'string' }," - r" { name = 'id', type = 'number', is_nullable = true }" - r"})") - srv.admin(r"box.space.test:create_index('pk'," - r"{ unique = true," - r" parts = {{field = 1, type = 'string'}}})") - srv.admin(r"box.space.test:create_index('id'," - r"{ unique = true," - r" parts = {{field = 2, type = 'number', is_nullable=true}}})") - srv.admin("box.schema.user.grant('test', 'read,write', 'space', 'test')") - srv.admin("json = require('json')") - - # Create srv_id function (for testing purposes). - srv.admin(f"function srv_id() return {_id} end") + resp = srv.admin(f""" + box.schema.user.create('test', {{password = 'test', if_not_exists = true}}) + box.schema.user.grant('test', 'execute', 'universe', + nil, {{if_not_exists = true}}) + box.schema.space.create('test', {{if_not_exists = true}}) + box.space.test:format({{ + {{ name = 'pk', type = 'string' }}, + {{ name = 'id', type = 'number', is_nullable = true }} + }}) + box.space.test:create_index('pk', + {{ + unique = true, + parts = {{ + {{field = 1, type = 'string'}} + }}, + if_not_exists = true, + }}) + box.space.test:create_index('id', + {{ + unique = true, + parts = {{ + {{field = 2, type = 'number', is_nullable=true}} + }}, + if_not_exists = true, + }}) + box.schema.user.grant('test', 'read,write', 'space', 'test', {{if_not_exists = true}}) + json = require('json') + + function srv_id() return {_id} end + + return true + """) + assert_admin_success(resp) return srv @@ -53,11 +68,12 @@ def create_server(_id): class TestSuitePool(unittest.TestCase): def set_ro(self, srv, read_only): if read_only: - req = r'box.cfg{read_only = true}' + req = r'box.cfg{read_only = true}; return true' else: - req = r'box.cfg{read_only = false}' + req = r'box.cfg{read_only = false}; return true' - srv.admin(req) + resp = srv.admin(req) + assert_admin_success(resp) def set_cluster_ro(self, read_only_list): assert len(self.servers) == len(read_only_list) diff --git a/test/suites/test_push.py b/test/suites/test_push.py index bd14455b..24a1ae7d 100644 --- a/test/suites/test_push.py +++ b/test/suites/test_push.py @@ -7,68 +7,58 @@ import unittest import tarantool from .lib.tarantool_server import TarantoolServer +from .utils import assert_admin_success def create_server(): srv = TarantoolServer() srv.script = 'test/suites/box.lua' srv.start() - srv.admin("box.schema.user.create('test', {password = 'test', " - "if_not_exists = true})") - srv.admin("box.schema.user.grant('test', 'read,write,execute', 'universe')") - - # Create server_function (for testing purposes). - srv.admin(""" - function server_function() - x = {0,0} - while x[1] < 3 do - x[1] = x[1] + 1 - box.session.push(x) + resp = srv.admin(""" + box.schema.user.create('test', {password = 'test', if_not_exists = true}) + box.schema.user.grant('test', 'read,write,execute', 'universe', + nil, {if_not_exists = true}) + + function server_function() + x = {0,0} + while x[1] < 3 do + x[1] = x[1] + 1 + box.session.push(x) + end + return x end - return x - end - """) - # Create tester space and on_replace trigger (for testing purposes). - srv.admin(""" - box.schema.create_space( - 'tester', { - format = { - {name = 'id', type = 'unsigned'}, - {name = 'name', type = 'string'}, - } - }) - """) - srv.admin(""" - box.space.tester:create_index( - 'primary_index', { - parts = { - {field = 1, type = 'unsigned'}, - } - }) - """) - srv.admin(""" - box.space.tester:create_index( - 'primary_index', { - parts = { - {field = 1, type = 'unsigned'}, - } - }) - """) - srv.admin(""" - function on_replace_callback() - x = {0,0} - while x[1] < 300 do - x[1] = x[1] + 100 - box.session.push(x) + box.schema.create_space( + 'tester', { + format = { + {name = 'id', type = 'unsigned'}, + {name = 'name', type = 'string'}, + } + }) + + box.space.tester:create_index( + 'primary_index', { + parts = { + {field = 1, type = 'unsigned'}, + }, + if_not_exists = true, + }) + + function on_replace_callback() + x = {0,0} + while x[1] < 300 do + x[1] = x[1] + 100 + box.session.push(x) + end end - end - """) - srv.admin(""" - box.space.tester:on_replace( - on_replace_callback - ) + + box.space.tester:on_replace( + on_replace_callback + ) + + return true """) + assert_admin_success(resp) return srv diff --git a/test/suites/test_schema.py b/test/suites/test_schema.py index 1402616d..6175c958 100644 --- a/test/suites/test_schema.py +++ b/test/suites/test_schema.py @@ -1,7 +1,7 @@ """ This module tests space and index schema fetch. """ -# pylint: disable=missing-class-docstring,missing-function-docstring,fixme,too-many-public-methods,too-many-branches,too-many-statements +# pylint: disable=missing-class-docstring,missing-function-docstring,fixme,too-many-public-methods,too-many-branches,too-many-statements,duplicate-code import sys import unittest @@ -12,6 +12,7 @@ from .lib.tarantool_server import TarantoolServer from .lib.skip import skip_or_run_constraints_test +from .utils import assert_admin_success # FIXME: I'm quite sure that there is a simpler way to count @@ -54,29 +55,35 @@ def setUpClass(cls): cls.srv = TarantoolServer() cls.srv.script = 'test/suites/box.lua' cls.srv.start() - cls.srv.admin("box.schema.user.create('test', {password = 'test', if_not_exists = true})") - cls.srv.admin("box.schema.user.grant('test', 'read,write,execute', 'universe')") - - # Create server_function and tester space (for fetch_schema opt testing purposes). - cls.srv.admin("function server_function() return 2+2 end") - cls.srv.admin(""" - box.schema.create_space( - 'tester', { - format = { - {name = 'id', type = 'unsigned'}, - {name = 'name', type = 'string', is_nullable = true}, - } - }) - """) - cls.srv.admin(""" - box.space.tester:create_index( - 'primary_index', { - parts = { - {field = 1, type = 'unsigned'}, - } - }) + resp = cls.srv.admin(""" + box.schema.user.create('test', {password = 'test', if_not_exists = true}) + box.schema.user.grant('test', 'read,write,execute', 'universe', + nil, {if_not_exists = true}) + + function server_function() return 2+2 end + + box.schema.create_space( + 'tester', { + format = { + {name = 'id', type = 'unsigned'}, + {name = 'name', type = 'string', is_nullable = true}, + }, + if_not_exists = true, + }) + + box.space.tester:create_index( + 'primary_index', { + parts = { + {field = 1, type = 'unsigned'}, + }, + if_not_exists = true, + }) + + box.space.tester:insert({1, null}) + + return true """) - cls.srv.admin("box.space.tester:insert({1, null})") + assert_admin_success(resp) cls.con = tarantool.Connection(cls.srv.host, cls.srv.args['primary'], encoding=cls.encoding, user='test', password='test') @@ -112,31 +119,34 @@ def setUpClass(cls): """) if cls.srv.admin.tnt_version >= pkg_resources.parse_version('2.10.0'): - cls.srv.admin(""" - box.schema.create_space( - 'constr_tester_1', { - format = { - { name = 'id', type = 'unsigned' }, - { name = 'payload', type = 'number' }, - } - }) - box.space.constr_tester_1:create_index('I1', { parts = {'id'} }) - - box.space.constr_tester_1:replace({1, 999}) - - box.schema.create_space( - 'constr_tester_2', { - format = { - { name = 'id', type = 'unsigned' }, - { name = 'table1_id', type = 'unsigned', - foreign_key = { fk_video = { space = 'constr_tester_1', field = 'id' } }, - }, - { name = 'payload', type = 'number' }, - } - }) - box.space.constr_tester_2:create_index('I1', { parts = {'id'} }) - box.space.constr_tester_2:create_index('I2', { parts = {'table1_id'} }) + resp = cls.srv.admin(""" + box.schema.create_space( + 'constr_tester_1', { + format = { + { name = 'id', type = 'unsigned' }, + { name = 'payload', type = 'number' }, + } + }) + box.space.constr_tester_1:create_index('I1', { parts = {'id'} }) + + box.space.constr_tester_1:replace({1, 999}) + + box.schema.create_space( + 'constr_tester_2', { + format = { + { name = 'id', type = 'unsigned' }, + { name = 'table1_id', type = 'unsigned', + foreign_key = { fk_video = { space = 'constr_tester_1', field = 'id' } }, + }, + { name = 'payload', type = 'number' }, + } + }) + box.space.constr_tester_2:create_index('I1', { parts = {'id'} }) + box.space.constr_tester_2:create_index('I2', { parts = {'table1_id'} }) + + return true """) + assert_admin_success(resp) def setUp(self): # prevent a remote tarantool from clean our session @@ -603,10 +613,13 @@ def tearDownClass(cls): # We need to drop spaces with foreign keys with predetermined order, # otherwise remote server clean() will fail to clean up resources. if cls.srv.admin.tnt_version >= pkg_resources.parse_version('2.10.0'): - cls.srv.admin(""" - box.space.constr_tester_2:drop() - box.space.constr_tester_1:drop() + resp = cls.srv.admin(""" + box.space.constr_tester_2:drop() + box.space.constr_tester_1:drop() + + return true """) + assert_admin_success(resp) cls.con.close() cls.con_schema_disable.close() diff --git a/test/suites/test_uuid.py b/test/suites/test_uuid.py index 0a9ef06b..90943fcb 100644 --- a/test/suites/test_uuid.py +++ b/test/suites/test_uuid.py @@ -7,6 +7,7 @@ import unittest import uuid +import pkg_resources import msgpack import tarantool @@ -15,6 +16,7 @@ from .lib.tarantool_server import TarantoolServer from .lib.skip import skip_or_run_uuid_test +from .utils import assert_admin_success class TestSuiteUUID(unittest.TestCase): @@ -27,26 +29,35 @@ def setUpClass(cls): cls.srv.start() cls.adm = cls.srv.admin - cls.adm(r""" + resp = cls.adm(""" _, uuid = pcall(require, 'uuid') - box.schema.space.create('test') + box.schema.space.create('test', {if_not_exists = true}) box.space['test']:create_index('primary', { type = 'tree', parts = {1, 'string'}, - unique = true}) + unique = true, + if_not_exists = true}) - pcall(function() - box.schema.space.create('test_pk') + box.schema.user.create('test', {password = 'test', if_not_exists = true}) + box.schema.user.grant('test', 'read,write,execute', 'universe', + nil, {if_not_exists = true}) + + return true + """) + assert_admin_success(resp) + + if cls.srv.admin.tnt_version >= pkg_resources.parse_version('2.4.1'): + resp = cls.adm(""" + box.schema.space.create('test_pk', {if_not_exists = true}) box.space['test_pk']:create_index('primary', { type = 'tree', parts = {1, 'uuid'}, - unique = true}) - end) - - box.schema.user.create('test', {password = 'test', if_not_exists = true}) - box.schema.user.grant('test', 'read,write,execute', 'universe') - """) + unique = true, + if_not_exists = true}) + return true + """) + assert_admin_success(resp) cls.con = tarantool.Connection(cls.srv.host, cls.srv.args['primary'], user='test', password='test') @@ -56,7 +67,11 @@ def setUp(self): if self.srv.is_started(): self.srv.touch_lock() - self.adm("box.space['test']:truncate()") + resp = self.adm(""" + box.space['test']:truncate() + return true + """) + assert_admin_success(resp) cases = { 'uuid_1': { diff --git a/test/suites/utils.py b/test/suites/utils.py new file mode 100644 index 00000000..65e68b4c --- /dev/null +++ b/test/suites/utils.py @@ -0,0 +1,16 @@ +""" +Various test utilities. +""" + + +def assert_admin_success(resp): + """ + Util to assert admin text request response. + It is expected that request last line is `return true`. + If something went wrong on executing, Tarantool throws an error + which would be a part of return values. + """ + + assert isinstance(resp, list), f'got unexpected resp type: {type(resp)}' + assert len(resp) > 0, 'got unexpected empty resp' + assert resp[0] is True, f'got unexpected resp: {resp}' From 66e53bca472fec0efacd690ebe6c54c33a7df3f9 Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Tue, 8 Aug 2023 11:14:47 +0300 Subject: [PATCH 13/39] test: fix confusing tuple value `null` is a common variable in Lua. Since the variable is not declared and strict mode is off, it is essentially `nil`. --- test/suites/test_schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/suites/test_schema.py b/test/suites/test_schema.py index 6175c958..c90ddbff 100644 --- a/test/suites/test_schema.py +++ b/test/suites/test_schema.py @@ -79,7 +79,7 @@ def setUpClass(cls): if_not_exists = true, }) - box.space.tester:insert({1, null}) + box.space.tester:insert({1, nil}) return true """) From a4b734aa46facc89fd967de1e74f9768341e56fc Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Tue, 19 Sep 2023 17:36:56 +0300 Subject: [PATCH 14/39] internal: more careful crud response process In most cases, crud API response process assumes that there are always two response values, which is rather uncommon. The reason of that is internal crud wrappers [1]. This patch reworks response process a bit to more carefully process this unusual case and to not confuse developers. 1. https://github.com/tarantool/crud/blob/53457477974fed42351cbd87f566d11e9f7e39bb/crud/common/schema.lua#L88 --- tarantool/connection.py | 51 +++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/tarantool/connection.py b/tarantool/connection.py index ea089c08..2fab182b 100644 --- a/tarantool/connection.py +++ b/tarantool/connection.py @@ -2158,7 +2158,7 @@ def crud_insert(self, space_name: str, values: Union[tuple, list], crud_resp = call_crud(self, "crud.insert", space_name, values, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) @@ -2192,7 +2192,7 @@ def crud_insert_object(self, space_name: str, values: dict, crud_resp = call_crud(self, "crud.insert_object", space_name, values, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) @@ -2230,7 +2230,7 @@ def crud_insert_many(self, space_name: str, values: Union[tuple, list], if crud_resp[0] is not None: res = CrudResult(crud_resp[0]) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: errs = [] for err in crud_resp[1]: errs.append(CrudError(err)) @@ -2271,7 +2271,7 @@ def crud_insert_object_many(self, space_name: str, values: Union[tuple, list], if crud_resp[0] is not None: res = CrudResult(crud_resp[0]) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: errs = [] for err in crud_resp[1]: errs.append(CrudError(err)) @@ -2306,7 +2306,7 @@ def crud_get(self, space_name: str, key: int, opts: Optional[dict] = None) -> Cr crud_resp = call_crud(self, "crud.get", space_name, key, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) @@ -2345,7 +2345,7 @@ def crud_update(self, space_name: str, key: int, operations: Optional[list] = No crud_resp = call_crud(self, "crud.update", space_name, key, operations, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) @@ -2377,7 +2377,7 @@ def crud_delete(self, space_name: str, key: int, opts: Optional[dict] = None) -> crud_resp = call_crud(self, "crud.delete", space_name, key, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) @@ -2411,7 +2411,7 @@ def crud_replace(self, space_name: str, values: Union[tuple, list], crud_resp = call_crud(self, "crud.replace", space_name, values, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) @@ -2445,7 +2445,7 @@ def crud_replace_object(self, space_name: str, values: dict, crud_resp = call_crud(self, "crud.replace_object", space_name, values, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) @@ -2483,7 +2483,7 @@ def crud_replace_many(self, space_name: str, values: Union[tuple, list], if crud_resp[0] is not None: res = CrudResult(crud_resp[0]) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: errs = [] for err in crud_resp[1]: errs.append(CrudError(err)) @@ -2524,7 +2524,7 @@ def crud_replace_object_many(self, space_name: str, values: Union[tuple, list], if crud_resp[0] is not None: res = CrudResult(crud_resp[0]) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: errs = [] for err in crud_resp[1]: errs.append(CrudError(err)) @@ -2567,7 +2567,7 @@ def crud_upsert(self, space_name: str, values: Union[tuple, list], crud_resp = call_crud(self, "crud.upsert", space_name, values, operations, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) @@ -2608,7 +2608,7 @@ def crud_upsert_object(self, space_name: str, values: dict, crud_resp = call_crud(self, "crud.upsert_object", space_name, values, operations, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) @@ -2646,7 +2646,7 @@ def crud_upsert_many(self, space_name: str, values_operation: Union[tuple, list] if crud_resp[0] is not None: res = CrudResult(crud_resp[0]) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: errs = [] for err in crud_resp[1]: errs.append(CrudError(err)) @@ -2687,7 +2687,7 @@ def crud_upsert_object_many(self, space_name: str, values_operation: Union[tuple if crud_resp[0] is not None: res = CrudResult(crud_resp[0]) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: errs = [] for err in crud_resp[1]: errs.append(CrudError(err)) @@ -2726,7 +2726,7 @@ def crud_select(self, space_name: str, conditions: Optional[list] = None, crud_resp = call_crud(self, "crud.select", space_name, conditions, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) @@ -2758,7 +2758,7 @@ def crud_min(self, space_name: str, index_name: str, opts: Optional[dict] = None crud_resp = call_crud(self, "crud.min", space_name, index_name, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) @@ -2790,7 +2790,7 @@ def crud_max(self, space_name: str, index_name: str, opts: Optional[dict] = None crud_resp = call_crud(self, "crud.max", space_name, index_name, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) @@ -2819,9 +2819,7 @@ def crud_truncate(self, space_name: str, opts: Optional[dict] = None) -> bool: crud_resp = call_crud(self, "crud.truncate", space_name, opts) - # In absence of an error, crud does not give - # variable err as nil (as in most cases). - if len(crud_resp) != 1: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return crud_resp[0] @@ -2850,9 +2848,7 @@ def crud_len(self, space_name: str, opts: Optional[dict] = None) -> int: crud_resp = call_crud(self, "crud.len", space_name, opts) - # In absence of an error, crud does not give - # variable err as nil (as in most cases). - if len(crud_resp) != 1: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return crud_resp[0] @@ -2877,9 +2873,7 @@ def crud_storage_info(self, opts: Optional[dict] = None) -> dict: crud_resp = call_crud(self, "crud.storage_info", opts) - # In absence of an error, crud does not give - # variable err as nil (as in most cases). - if len(crud_resp) != 1: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return crud_resp[0] @@ -2915,7 +2909,7 @@ def crud_count(self, space_name: str, conditions: Optional[list] = None, crud_resp = call_crud(self, "crud.count", space_name, conditions, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return crud_resp[0] @@ -2938,6 +2932,7 @@ def crud_stats(self, space_name: str = None) -> CrudResult: crud_resp = call_crud(self, "crud.stats", space_name) + # There are no errors in `crud.stats`. res = None if len(crud_resp.data[0]) > 0: res = CrudResult(crud_resp.data[0]) From fea0d5f751c0800b915624ad5db3d12ffe898bf6 Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Tue, 19 Sep 2023 17:41:31 +0300 Subject: [PATCH 15/39] api: exception rethrow in crud Before this patch, all DatabaseError exception, except for ER_NO_SUCH_PROC and ER_ACCESS_DENIED ones, were silently ignored. It resulted in "UnboundLocalError: local variable 'crud_resp' referenced before assignment" instead of a proper error. --- CHANGELOG.md | 5 +++++ tarantool/crud.py | 1 + test/suites/crud_mock_server.lua | 21 +++++++++++++++++++++ test/suites/test_crud.py | 21 +++++++++++++++++++++ 4 files changed, 48 insertions(+) create mode 100644 test/suites/crud_mock_server.lua diff --git a/CHANGELOG.md b/CHANGELOG.md index 13e58a2b..8555c6ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Fixed +- Exception rethrow in crud API (PR #310). + ## 1.1.1 - 2023-07-19 ### Changed diff --git a/tarantool/crud.py b/tarantool/crud.py index d10726c5..3a642564 100644 --- a/tarantool/crud.py +++ b/tarantool/crud.py @@ -69,5 +69,6 @@ def call_crud(conn, *args): if exc.code in (ER_NO_SUCH_PROC, ER_ACCESS_DENIED): exc_msg = ". Ensure that you're calling crud.router and user has sufficient grants" raise DatabaseError(exc.code, exc.message + exc_msg, extra_info=exc.extra_info) from exc + raise exc return crud_resp diff --git a/test/suites/crud_mock_server.lua b/test/suites/crud_mock_server.lua new file mode 100644 index 00000000..252059df --- /dev/null +++ b/test/suites/crud_mock_server.lua @@ -0,0 +1,21 @@ +#!/usr/bin/env tarantool + +local admin_listen = os.getenv("ADMIN") +local primary_listen = os.getenv("LISTEN") + +require('console').listen(admin_listen) +box.cfg{ + listen = primary_listen, + memtx_memory = 0.1 * 1024^3, -- 0.1 GiB + pid_file = "box.pid", +} + +box.schema.user.grant('guest', 'execute', 'universe', nil, {if_not_exists = true}) + +local function mock_replace() + error('Unexpected connection error') +end + +rawset(_G, 'crud', {replace = mock_replace}) + +rawset(_G, 'ready', true) diff --git a/test/suites/test_crud.py b/test/suites/test_crud.py index d1d7e186..2bbc5513 100644 --- a/test/suites/test_crud.py +++ b/test/suites/test_crud.py @@ -22,6 +22,14 @@ def create_server(): return srv +def create_mock_server(): + srv = TarantoolServer() + srv.script = 'test/suites/crud_mock_server.lua' + srv.start() + + return srv + + @unittest.skipIf(sys.platform.startswith("win"), "Crud tests on windows platform are not supported: " "complexity of the vshard replicaset configuration") @@ -33,14 +41,19 @@ def setUpClass(cls): print('-' * 70, file=sys.stderr) # Create server and extract helpful fields for tests. cls.srv = create_server() + cls.mock_srv = create_mock_server() cls.host = cls.srv.host cls.port = cls.srv.args['primary'] + cls.mock_host = cls.mock_srv.host + cls.mock_port = cls.mock_srv.args['primary'] def setUp(self): time.sleep(1) # Open connections to instance. self.conn = tarantool.Connection(host=self.host, port=self.port, user='guest', password='', fetch_schema=False) + self.mock_conn = tarantool.Connection(host=self.mock_host, port=self.mock_port, + user='guest', password='', fetch_schema=False) self.conn_mesh = tarantool.MeshConnection(host=self.host, port=self.port, user='guest', password='', fetch_schema=False) self.conn_pool = tarantool.ConnectionPool([{'host': self.host, 'port': self.port}], @@ -736,9 +749,15 @@ def test_crud_module_via_pool_connection(self): # Exception try testing. self._exception_operation_with_crud(testing_function, case, mode=tarantool.Mode.RW) + def test_error_rethrow(self): + self.assertRaisesRegex( + DatabaseError, "Unexpected connection error", + lambda: self.mock_conn.crud_replace('tester', [2, 100, 'Alice'], {'timeout': 10})) + def tearDown(self): # Close connections to instance. self.conn.close() + self.mock_conn.close() self.conn_mesh.close() self.conn_pool.close() @@ -747,3 +766,5 @@ def tearDownClass(cls): # Stop instance. cls.srv.stop() cls.srv.clean() + cls.mock_srv.stop() + cls.mock_srv.clean() From 90402e8926cdb07ba35835cc9fd6591d51fe3510 Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Tue, 19 Sep 2023 18:58:46 +0300 Subject: [PATCH 16/39] datetime: support big values for some platforms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before this patch, tarantool.Datetime constructor used datetime.fromtimestamp function to build a new datetime [1], except for negative timestamps for Windows platform. This constructor branch is used on each Tarantool datetime encoding or while building a tarantool.Datetime object from timestamp. datetime.fromtimestamp have some drawbacks: it "may raise OverflowError, if the timestamp is out of the range of values supported by the platform C localtime() or gmtime() functions, and OSError on localtime() or gmtime() failure. It’s common for this to be restricted to years in 1970 through 2038.". It had never happened on supported Unix platforms, but seem to be an issue for Windows ones. We already workaround this issue for years smaller than 1970 on Windows. After this patch, this workaround will be used for all platforms and timestamp values, allowing to provide similar behavior for platforms both restricted to years in 1970 through 2038 with localtime() or gmtime() or not. 1. https://docs.python.org/3/library/datetime.html#datetime.datetime.fromtimestamp --- CHANGELOG.md | 4 +++ tarantool/msgpack_ext/types/datetime.py | 14 ++++----- test/suites/test_datetime.py | 40 +++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8555c6ac..a4b400fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Exception rethrow in crud API (PR #310). +- Work with timestamps larger than year 2038 for some platforms (like Windows) (PR #311). + It covers + - building new `tarantool.Datetime` objects from timestamp, + - parsing datetime objects received from Tarantool. ## 1.1.1 - 2023-07-19 diff --git a/tarantool/msgpack_ext/types/datetime.py b/tarantool/msgpack_ext/types/datetime.py index 1a546cfc..993df404 100644 --- a/tarantool/msgpack_ext/types/datetime.py +++ b/tarantool/msgpack_ext/types/datetime.py @@ -6,7 +6,6 @@ from calendar import monthrange from copy import deepcopy from datetime import datetime, timedelta -import sys import pytz @@ -314,13 +313,12 @@ def __init__(self, *, timestamp=None, year=None, month=None, timestamp += nsec // NSEC_IN_SEC nsec = nsec % NSEC_IN_SEC - if (sys.platform.startswith("win")) and (timestamp < 0): - # Fails to create a datetime from negative timestamp on Windows. - _datetime = _EPOCH + timedelta(seconds=timestamp) - else: - # Timezone-naive datetime objects are treated by many datetime methods - # as local times, so we represent time in UTC explicitly if not provided. - _datetime = datetime.fromtimestamp(timestamp, pytz.UTC) + # datetime.fromtimestamp may raise OverflowError, if the timestamp + # is out of the range of values supported by the platform C localtime() + # function, and OSError on localtime() failure. It’s common for this + # to be restricted to years from 1970 through 2038, yet we want + # to support a wider range. + _datetime = _EPOCH + timedelta(seconds=timestamp) if nsec is not None: _datetime = _datetime.replace(microsecond=nsec // NSEC_IN_MKSEC) diff --git a/test/suites/test_datetime.py b/test/suites/test_datetime.py index 9f250804..80958fdc 100644 --- a/test/suites/test_datetime.py +++ b/test/suites/test_datetime.py @@ -153,6 +153,24 @@ def test_datetime_class_api_wth_tz(self): 'type': ValueError, 'msg': 'Failed to create datetime with ambiguous timezone "AET"' }, + 'under_min_timestamp_1': { + 'args': [], + 'kwargs': {'timestamp': -62135596801}, + 'type': OverflowError, + 'msg': 'date value out of range' + }, + 'under_min_timestamp_2': { + 'args': [], + 'kwargs': {'timestamp': -62135596800, 'nsec': -1}, + 'type': OverflowError, + 'msg': 'date value out of range' + }, + 'over_max_timestamp': { + 'args': [], + 'kwargs': {'timestamp': 253402300800}, + 'type': OverflowError, + 'msg': 'date value out of range' + }, } def test_datetime_class_invalid_init(self): @@ -293,6 +311,28 @@ def test_datetime_class_invalid_init(self): 'tarantool': r"datetime.new({timestamp=1661969274, nsec=308543321, " r"tz='Europe/Moscow'})", }, + 'min_datetime': { # Python datetime.MINYEAR is 1. + 'python': tarantool.Datetime(year=1, month=1, day=1, hour=0, minute=0, sec=0), + 'msgpack': (b'\x00\x09\x6e\x88\xf1\xff\xff\xff'), + 'tarantool': r"datetime.new({year=1, month=1, day=1, hour=0, min=0, sec=0})", + }, + 'max_datetime': { # Python datetime.MAXYEAR is 9999. + 'python': tarantool.Datetime(year=9999, month=12, day=31, hour=23, minute=59, sec=59, + nsec=999999999), + 'msgpack': (b'\x7f\x41\xf4\xff\x3a\x00\x00\x00\xff\xc9\x9a\x3b\x00\x00\x00\x00'), + 'tarantool': r"datetime.new({year=9999, month=12, day=31, hour=23, min=59, sec=59," + r"nsec=999999999})", + }, + 'min_datetime_timestamp': { # Python datetime.MINYEAR is 1. + 'python': tarantool.Datetime(timestamp=-62135596800), + 'msgpack': (b'\x00\x09\x6e\x88\xf1\xff\xff\xff'), + 'tarantool': r"datetime.new({timestamp=-62135596800})", + }, + 'max_datetime_timestamp': { # Python datetime.MAXYEAR is 9999. + 'python': tarantool.Datetime(timestamp=253402300799, nsec=999999999), + 'msgpack': (b'\x7f\x41\xf4\xff\x3a\x00\x00\x00\xff\xc9\x9a\x3b\x00\x00\x00\x00'), + 'tarantool': r"datetime.new({timestamp=253402300799, nsec=999999999})", + }, } def test_msgpack_decode(self): From e104037337090e679830a2411ab822ed34bb2a0d Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Wed, 20 Sep 2023 09:27:36 +0300 Subject: [PATCH 17/39] release 1.1.2 Overview This release introduce several bugfixes and behavior improvements. Fixed - Exception rethrow in crud API (PR #310). - Work with timestamps larger than year 2038 for some platforms (like Windows) (PR #311). It covers - building new `tarantool.Datetime` objects from timestamp, - parsing datetime objects received from Tarantool. --- CHANGELOG.md | 2 +- debian/changelog | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a4b400fc..cf674622 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Unreleased +## 1.1.2 - 2023-09-20 ### Fixed - Exception rethrow in crud API (PR #310). diff --git a/debian/changelog b/debian/changelog index 429c65a1..c8d5153d 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,19 @@ +python3-tarantool (1.1.2-0) unstable; urgency=medium + + ## Overview + + This release introduce several bugfixes and behavior improvements. + + ## Fixed + + - Exception rethrow in crud API (PR #310). + - Work with timestamps larger than year 2038 for some platforms (like Windows) (PR #311). + It covers + - building new `tarantool.Datetime` objects from timestamp, + - parsing datetime objects received from Tarantool. + + -- Georgy Moiseev Wed, 20 Sep 2023 10:00:00 +0300 + python3-tarantool (1.1.1-0) unstable; urgency=medium ## Overview From fc2323fe9de4af57c44e02d2058db6d1dddf53ae Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Tue, 3 Oct 2023 13:38:31 +0300 Subject: [PATCH 18/39] pylint: disable new too-many-arguments cases Disable newly uncovered cases of R0913 too-many-arguments. We don't plan to change existing public API now. Since pylint 3.0.0, issue [1] has been fixed, which uncovered several new too-many-arguments cases. 1. https://github.com/pylint-dev/pylint/issues/8667 --- tarantool/connection.py | 8 ++++++++ tarantool/connection_pool.py | 9 +++++++++ tarantool/msgpack_ext/types/datetime.py | 2 +- tarantool/msgpack_ext/types/interval.py | 1 + 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/tarantool/connection.py b/tarantool/connection.py index 2fab182b..17b43b64 100644 --- a/tarantool/connection.py +++ b/tarantool/connection.py @@ -215,6 +215,7 @@ def delete(self, space_name, key, *, index=None, on_push=None, on_push_ctx=None) """ Reference implementation: :meth:`~tarantool.Connection.delete`. """ + # pylint: disable=too-many-arguments raise NotImplementedError @@ -224,6 +225,7 @@ def upsert(self, space_name, tuple_value, op_list, *, index=None, """ Reference implementation: :meth:`~tarantool.Connection.upsert`. """ + # pylint: disable=too-many-arguments raise NotImplementedError @@ -232,6 +234,7 @@ def update(self, space_name, key, op_list, *, index=None, on_push=None, on_push_ """ Reference implementation: :meth:`~tarantool.Connection.update`. """ + # pylint: disable=too-many-arguments raise NotImplementedError @@ -249,6 +252,7 @@ def select(self, space_name, key, *, offset=None, limit=None, """ Reference implementation: :meth:`~tarantool.Connection.select`. """ + # pylint: disable=too-many-arguments raise NotImplementedError @@ -1618,6 +1622,7 @@ def delete(self, space_name, key, *, index=0, on_push=None, on_push_ctx=None): .. _delete: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/delete/ """ + # pylint: disable=too-many-arguments self._schemaful_connection_check() @@ -1680,6 +1685,7 @@ def upsert(self, space_name, tuple_value, op_list, *, index=0, on_push=None, on_ .. _upsert: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/upsert/ """ + # pylint: disable=too-many-arguments self._schemaful_connection_check() @@ -1771,6 +1777,7 @@ def update(self, space_name, key, op_list, *, index=0, on_push=None, on_push_ctx .. _update: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/update/ """ + # pylint: disable=too-many-arguments self._schemaful_connection_check() @@ -1961,6 +1968,7 @@ def select(self, space_name, key=None, *, offset=0, limit=0xffffffff, index=0, i .. _select: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/select/ """ + # pylint: disable=too-many-arguments self._schemaful_connection_check() diff --git a/tarantool/connection_pool.py b/tarantool/connection_pool.py index 1008991f..3c1e72ac 100644 --- a/tarantool/connection_pool.py +++ b/tarantool/connection_pool.py @@ -820,6 +820,7 @@ def replace(self, space_name, values, *, mode=Mode.RW, on_push=None, on_push_ctx .. _replace: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/replace/ """ + # pylint: disable=too-many-arguments return self._send(mode, 'replace', space_name, values, on_push=on_push, on_push_ctx=on_push_ctx) @@ -850,6 +851,7 @@ def insert(self, space_name, values, *, mode=Mode.RW, on_push=None, on_push_ctx= .. _insert: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/insert/ """ + # pylint: disable=too-many-arguments return self._send(mode, 'insert', space_name, values, on_push=on_push, on_push_ctx=on_push_ctx) @@ -883,6 +885,7 @@ def delete(self, space_name, key, *, index=0, mode=Mode.RW, on_push=None, on_pus .. _delete: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/delete/ """ + # pylint: disable=too-many-arguments return self._send(mode, 'delete', space_name, key, index=index, on_push=on_push, on_push_ctx=on_push_ctx) @@ -920,6 +923,7 @@ def upsert(self, space_name, tuple_value, op_list, *, index=0, mode=Mode.RW, .. _upsert: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/upsert/ """ + # pylint: disable=too-many-arguments return self._send(mode, 'upsert', space_name, tuple_value, op_list, index=index, on_push=on_push, on_push_ctx=on_push_ctx) @@ -957,6 +961,7 @@ def update(self, space_name, key, op_list, *, index=0, mode=Mode.RW, .. _update: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/update/ """ + # pylint: disable=too-many-arguments return self._send(mode, 'update', space_name, key, op_list, index=index, on_push=on_push, on_push_ctx=on_push_ctx) @@ -1023,6 +1028,7 @@ def select(self, space_name, key, *, offset=0, limit=0xffffffff, .. _select: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/select/ """ + # pylint: disable=too-many-arguments return self._send(mode, 'select', space_name, key, offset=offset, limit=limit, index=index, iterator=iterator, on_push=on_push, on_push_ctx=on_push_ctx) @@ -1214,6 +1220,7 @@ def crud_update(self, space_name, key, operations=None, opts=None, *, mode=Mode. :raise: :exc:`~tarantool.error.CrudModuleError`, :exc:`~tarantool.error.DatabaseError` """ + # pylint: disable=too-many-arguments return self._send(mode, 'crud_update', space_name, key, operations, opts) @@ -1379,6 +1386,7 @@ def crud_upsert(self, space_name, values, operations=None, opts=None, *, mode=Mo :raise: :exc:`~tarantool.error.CrudModuleError`, :exc:`~tarantool.error.DatabaseError` """ + # pylint: disable=too-many-arguments return self._send(mode, 'crud_upsert', space_name, values, operations, opts) @@ -1409,6 +1417,7 @@ def crud_upsert_object(self, space_name, values, operations=None, opts=None, *, :raise: :exc:`~tarantool.error.CrudModuleError`, :exc:`~tarantool.error.DatabaseError` """ + # pylint: disable=too-many-arguments return self._send(mode, 'crud_upsert_object', space_name, values, operations, opts) diff --git a/tarantool/msgpack_ext/types/datetime.py b/tarantool/msgpack_ext/types/datetime.py index 993df404..d7068f57 100644 --- a/tarantool/msgpack_ext/types/datetime.py +++ b/tarantool/msgpack_ext/types/datetime.py @@ -281,7 +281,7 @@ def __init__(self, *, timestamp=None, year=None, month=None, .. _datetime.new(): https://www.tarantool.io/en/doc/latest/reference/reference_lua/datetime/new/ """ - # pylint: disable=too-many-branches,too-many-locals,too-many-statements + # pylint: disable=too-many-branches,too-many-locals,too-many-statements,too-many-arguments tzinfo = None if tz != '': diff --git a/tarantool/msgpack_ext/types/interval.py b/tarantool/msgpack_ext/types/interval.py index e44dcce0..734a9088 100644 --- a/tarantool/msgpack_ext/types/interval.py +++ b/tarantool/msgpack_ext/types/interval.py @@ -149,6 +149,7 @@ def __init__(self, *, year=0, month=0, week=0, :raise: :exc:`ValueError` """ + # pylint: disable=too-many-arguments self.year = year self.month = month From 8549c1c8e7ad480cef8fdedb10326ef1cdfb2fea Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Tue, 3 Oct 2023 14:07:58 +0300 Subject: [PATCH 19/39] pylint: disable unnecessary-lambda in test Disable unnecessary-lambda lint warning in test. The warning had been uncovered since pylint 3.0.0, but it's not clear what exactly is the reason since the only CHANGELOG entry related to unnecessary-lambda declares that false positive cases were fixed [1]. 1. https://github.com/pylint-dev/pylint/issues/8496 --- test/suites/test_interval.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/suites/test_interval.py b/test/suites/test_interval.py index c241b312..2b6f6abe 100644 --- a/test/suites/test_interval.py +++ b/test/suites/test_interval.py @@ -616,7 +616,7 @@ def test_unknown_adjust_decode(self): } def test_out_of_range(self): - # pylint: disable=cell-var-from-loop + # pylint: disable=cell-var-from-loop,unnecessary-lambda for name, case in self.out_of_range_cases.items(): with self.subTest(msg=name): From e5fe54efafac49e6dfd0783ff5f522b4e4b714d3 Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Tue, 3 Oct 2023 14:18:59 +0300 Subject: [PATCH 20/39] doc: fix spelling --- tarantool/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tarantool/utils.py b/tarantool/utils.py index c7d9e246..b7ab436d 100644 --- a/tarantool/utils.py +++ b/tarantool/utils.py @@ -1,5 +1,5 @@ """ -This module provides untility functions for the package. +This module provides utility functions for the package. """ from base64 import decodebytes as base64_decode From 3c8eb9da94c5e1665318d994219efa104e5fb78e Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Tue, 3 Oct 2023 14:10:32 +0300 Subject: [PATCH 21/39] deps: fixate linters version Version lock is based on a latest version available for specific Python. --- requirements-test.txt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/requirements-test.txt b/requirements-test.txt index 94d16351..56f4b595 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,6 +1,8 @@ git+https://github.com/baztian/dbapi-compliance.git@ea7cb1b4#egg=dbapi-compliance pyyaml==6.0 importlib-metadata >= 1.0 ; python_version < '3.8' -pylint -flake8 -codespell +pylint ~= 3.0 ; python_version >= '3.8' +pylint ~= 2.13 ; python_version < '3.8' +flake8 ~= 6.1 ; python_version >= '3.8' +flake8 ~= 5.0 ; python_version < '3.8' +codespell ~= 2.2 From 52165d34f9e63331e16e13c6e19683791d82d756 Mon Sep 17 00:00:00 2001 From: Albert Skalt Date: Wed, 1 Nov 2023 12:27:03 +0300 Subject: [PATCH 22/39] doc: fix typo in docstring --- tarantool/mesh_connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tarantool/mesh_connection.py b/tarantool/mesh_connection.py index ebbc1a40..74b8af43 100644 --- a/tarantool/mesh_connection.py +++ b/tarantool/mesh_connection.py @@ -298,7 +298,7 @@ def __init__(self, host=None, port=None, :paramref:`~tarantool.MeshConnection.params.addrs` list. :param port: Refer to - :paramref:`~tarantool.Connection.params.host`. + :paramref:`~tarantool.Connection.params.port`. Value would be used to add one more server in :paramref:`~tarantool.MeshConnection.params.addrs` list. From b0833e705cbae30b94b0ef0b82e2657ec89d5b93 Mon Sep 17 00:00:00 2001 From: Albert Skalt Date: Thu, 2 Nov 2023 11:50:14 +0300 Subject: [PATCH 23/39] ci: run tests with tarantool CE master --- .github/workflows/testing.yml | 37 +++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 88fa08d7..82f299b4 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -27,6 +27,7 @@ jobs: - '2.8' - '2.10' - '2.11' + - 'master' python: - '3.6' - '3.7' @@ -57,11 +58,45 @@ jobs: - name: Clone the connector uses: actions/checkout@v3 + - name: Setup tt + run: | + curl -L https://tarantool.io/release/2/installer.sh | sudo bash + sudo apt install -y tt + tt version + tt init + - name: Install tarantool ${{ matrix.tarantool }} + if: matrix.tarantool != 'master' uses: tarantool/setup-tarantool@v2 with: tarantool-version: ${{ matrix.tarantool }} + - name: Get Tarantool master latest commit + if: matrix.tarantool == 'master' + run: | + commit_hash=$(git ls-remote https://github.com/tarantool/tarantool.git --branch master | head -c 8) + echo "LATEST_COMMIT=${commit_hash}" >> $GITHUB_ENV + shell: bash + + - name: Cache Tarantool master + if: matrix.tarantool == 'master' + id: cache-latest + uses: actions/cache@v3 + with: + path: | + ${{ github.workspace }}/bin + ${{ github.workspace }}/include + key: cache-latest-${{ env.LATEST_COMMIT }} + + - name: Setup Tarantool master + if: matrix.tarantool == 'master' && steps.cache-latest.outputs.cache-hit != 'true' + run: | + tt install tarantool master + + - name: Add Tarantool master to PATH + if: matrix.tarantool == 'master' + run: echo "${GITHUB_WORKSPACE}/bin" >> $GITHUB_PATH + - name: Setup Python for tests uses: actions/setup-python@v4 with: @@ -86,8 +121,6 @@ jobs: - name: Install the crud module for testing purposes run: | - curl -L https://tarantool.io/release/2/installer.sh | bash - sudo apt install -y tt tt rocks install crud - name: Run tests From 2983de854aa958b3890fccdd2eae8033480120de Mon Sep 17 00:00:00 2001 From: Albert Skalt Date: Tue, 31 Oct 2023 23:15:54 +0300 Subject: [PATCH 24/39] conn: create from socket fd This patch adds the ability to create Tarantool connection using an existing socket fd. To achieve this, several changes have been made to work with non-blocking sockets, as `socket.socketpair` creates such [1]. The authentication [2] might have already occured when we establish such a connection. If that's the case, there is no need to pass 'user' argument. On success, connect takes ownership of the `socket_fd`. 1. https://github.com/tarantool/tarantool/pull/8944 2. https://www.tarantool.io/en/doc/latest/dev_guide/internals/iproto/authentication/ Closes #304 --- CHANGELOG.md | 5 + tarantool/__init__.py | 5 +- tarantool/connection.py | 104 ++++++++++++++++-- tarantool/connection_pool.py | 29 +++-- tarantool/const.py | 6 + tarantool/mesh_connection.py | 39 ++++++- test/suites/__init__.py | 3 +- test/suites/lib/skip.py | 11 ++ test/suites/sidecar.py | 16 +++ test/suites/test_mesh.py | 1 - test/suites/test_socket_fd.py | 200 ++++++++++++++++++++++++++++++++++ 11 files changed, 394 insertions(+), 25 deletions(-) create mode 100644 test/suites/sidecar.py create mode 100644 test/suites/test_socket_fd.py diff --git a/CHANGELOG.md b/CHANGELOG.md index cf674622..f3bd49f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Added +- The ability to connect to the Tarantool using an existing socket fd (#304). + ## 1.1.2 - 2023-09-20 ### Fixed diff --git a/tarantool/__init__.py b/tarantool/__init__.py index 91f80e10..d7e99358 100644 --- a/tarantool/__init__.py +++ b/tarantool/__init__.py @@ -51,7 +51,7 @@ __version__ = '0.0.0-dev' -def connect(host="localhost", port=33013, user=None, password=None, +def connect(host="localhost", port=33013, socket_fd=None, user=None, password=None, encoding=ENCODING_DEFAULT, transport=DEFAULT_TRANSPORT, ssl_key_file=DEFAULT_SSL_KEY_FILE, ssl_cert_file=DEFAULT_SSL_CERT_FILE, @@ -64,6 +64,8 @@ def connect(host="localhost", port=33013, user=None, password=None, :param port: Refer to :paramref:`~tarantool.Connection.params.port`. + :param socket_fd: Refer to :paramref:`~tarantool.Connection.params.socket_fd`. + :param user: Refer to :paramref:`~tarantool.Connection.params.user`. :param password: Refer to @@ -93,6 +95,7 @@ def connect(host="localhost", port=33013, user=None, password=None, """ return Connection(host, port, + socket_fd=socket_fd, user=user, password=password, socket_timeout=SOCKET_TIMEOUT, diff --git a/tarantool/connection.py b/tarantool/connection.py index 17b43b64..1284fa25 100644 --- a/tarantool/connection.py +++ b/tarantool/connection.py @@ -4,6 +4,7 @@ # pylint: disable=too-many-lines,duplicate-code import os +import select import time import errno from enum import Enum @@ -51,6 +52,9 @@ RECONNECT_DELAY, DEFAULT_TRANSPORT, SSL_TRANSPORT, + DEFAULT_HOST, + DEFAULT_PORT, + DEFAULT_SOCKET_FD, DEFAULT_SSL_KEY_FILE, DEFAULT_SSL_CERT_FILE, DEFAULT_SSL_CA_FILE, @@ -594,7 +598,10 @@ class Connection(ConnectionInterface): :value: :exc:`~tarantool.error.CrudModuleError` """ - def __init__(self, host, port, + def __init__(self, + host=DEFAULT_HOST, + port=DEFAULT_PORT, + socket_fd=DEFAULT_SOCKET_FD, user=None, password=None, socket_timeout=SOCKET_TIMEOUT, @@ -623,8 +630,11 @@ def __init__(self, host, port, Unix sockets. :type host: :obj:`str` or :obj:`None` - :param port: Server port or Unix socket path. - :type port: :obj:`int` or :obj:`str` + :param port: Server port, or Unix socket path. + :type port: :obj:`int` or :obj:`str` or :obj:`None` + + :param socket_fd: socket fd number. + :type socket_fd: :obj:`int` or :obj:`None` :param user: User name for authentication on the Tarantool server. @@ -804,6 +814,18 @@ def __init__(self, host, port, """ # pylint: disable=too-many-arguments,too-many-locals,too-many-statements + if host is None and port is None and socket_fd is None: + raise ConfigurationError("need to specify host/port, " + "port (in case of Unix sockets) " + "or socket_fd") + + if socket_fd is not None and (host is not None or port is not None): + raise ConfigurationError("specifying both socket_fd and host/port is not allowed") + + if host is not None and port is None: + raise ConfigurationError("when specifying host, " + "it is also necessary to specify port") + if msgpack.version >= (1, 0, 0) and encoding not in (None, 'utf-8'): raise ConfigurationError("msgpack>=1.0.0 only supports None and " + "'utf-8' encoding option values") @@ -820,6 +842,7 @@ def __init__(self, host, port, recv.restype = ctypes.c_int self.host = host self.port = port + self.socket_fd = socket_fd self.user = user self.password = password self.socket_timeout = socket_timeout @@ -897,10 +920,37 @@ def connect_basic(self): :meta private: """ - if self.host is None: - self.connect_unix() - else: + if self.socket_fd is not None: + self.connect_socket_fd() + elif self.host is not None: self.connect_tcp() + else: + self.connect_unix() + + def connect_socket_fd(self): + """ + Establish a connection using an existing socket fd. + + +---------------------+--------------------------+-------------------------+ + | socket_fd / timeout | >= 0 | `None` | + +=====================+==========================+=========================+ + | blocking | Set non-blocking socket | Don't change, `select` | + | | lib call `select` | isn't needed | + +---------------------+--------------------------+-------------------------+ + | non-blocking | Don't change, socket lib | Don't change, call | + | | call `select` | `select` ourselves | + +---------------------+--------------------------+-------------------------+ + + :meta private: + """ + + self.connected = True + if self._socket: + self._socket.close() + + self._socket = socket.socket(fileno=self.socket_fd) + if self.socket_timeout is not None: + self._socket.settimeout(self.socket_timeout) def connect_tcp(self): """ @@ -1124,6 +1174,11 @@ def _recv(self, to_read): while to_read > 0: try: tmp = self._socket.recv(to_read) + except BlockingIOError: + ready, _, _ = select.select([self._socket.fileno()], [], [], self.socket_timeout) + if not ready: + raise NetworkError(TimeoutError()) # pylint: disable=raise-missing-from + continue except OverflowError as exc: self._socket.close() err = socket.error( @@ -1163,6 +1218,41 @@ def _read_response(self): # Read the packet return self._recv(length) + def _sendall(self, bytes_to_send): + """ + Sends bytes to the transport (socket). + + :param bytes_to_send: Message to send. + :type bytes_to_send: :obj:`bytes` + + :raise: :exc:`~tarantool.error.NetworkError` + + :meta private: + """ + + total_sent = 0 + while total_sent < len(bytes_to_send): + try: + sent = self._socket.send(bytes_to_send[total_sent:]) + if sent == 0: + err = socket.error( + errno.ECONNRESET, + "Lost connection to server during query" + ) + raise NetworkError(err) + total_sent += sent + except BlockingIOError as exc: + total_sent += exc.characters_written + _, ready, _ = select.select([], [self._socket.fileno()], [], self.socket_timeout) + if not ready: + raise NetworkError(TimeoutError()) # pylint: disable=raise-missing-from + except socket.error as exc: + err = socket.error( + errno.ECONNRESET, + "Lost connection to server during query" + ) + raise NetworkError(err) from exc + def _send_request_wo_reconnect(self, request, on_push=None, on_push_ctx=None): """ Send request without trying to reconnect. @@ -1191,7 +1281,7 @@ def _send_request_wo_reconnect(self, request, on_push=None, on_push_ctx=None): response = None while True: try: - self._socket.sendall(bytes(request)) + self._sendall(bytes(request)) response = request.response_class(self, self._read_response()) break except SchemaReloadException as exc: diff --git a/tarantool/connection_pool.py b/tarantool/connection_pool.py index 3c1e72ac..7bbd4e16 100644 --- a/tarantool/connection_pool.py +++ b/tarantool/connection_pool.py @@ -115,7 +115,7 @@ class PoolUnit(): addr: dict """ - ``{"host": host, "port": port}`` info. + ``{"host": host, "port": port, "socket_fd": socket_fd}`` info. :type: :obj:`dict` """ @@ -161,6 +161,14 @@ class PoolUnit(): :type: :obj:`bool` """ + def get_address(self): + """ + Get an address string representation. + """ + if self.addr['socket_fd'] is not None: + return f'fd://{self.addr["socket_fd"]}' + return f'{self.addr["host"]}:{self.addr["port"]}' + # Based on https://realpython.com/python-interface/ class StrategyInterface(metaclass=abc.ABCMeta): @@ -398,6 +406,7 @@ def __init__(self, { "host': "str" or None, # mandatory "port": int or "str", # mandatory + "socket_fd": int, # optional "transport": "str", # optional "ssl_key_file": "str", # optional "ssl_cert_file": "str", # optional @@ -499,6 +508,7 @@ def __init__(self, conn=Connection( host=addr['host'], port=addr['port'], + socket_fd=addr['socket_fd'], user=user, password=password, socket_timeout=socket_timeout, @@ -529,15 +539,16 @@ def _make_key(self, addr): """ Make a unique key for a server based on its address. - :param addr: `{"host": host, "port": port}` dictionary. + :param addr: `{"host": host, "port": port, "socket_fd": socket_fd}` dictionary. :type addr: :obj:`dict` :rtype: :obj:`str` :meta private: """ - - return f"{addr['host']}:{addr['port']}" + if addr['socket_fd'] is None: + return f"{addr['host']}:{addr['port']}" + return addr['socket_fd'] def _get_new_state(self, unit): """ @@ -557,7 +568,7 @@ def _get_new_state(self, unit): try: conn.connect() except NetworkError as exc: - msg = (f"Failed to connect to {unit.addr['host']}:{unit.addr['port']}, " + msg = (f"Failed to connect to {unit.get_address()}, " f"reason: {repr(exc)}") warn(msg, ClusterConnectWarning) return InstanceState(Status.UNHEALTHY) @@ -565,7 +576,7 @@ def _get_new_state(self, unit): try: resp = conn.call('box.info') except NetworkError as exc: - msg = (f"Failed to get box.info for {unit.addr['host']}:{unit.addr['port']}, " + msg = (f"Failed to get box.info for {unit.get_address()}, " f"reason: {repr(exc)}") warn(msg, PoolTolopogyWarning) return InstanceState(Status.UNHEALTHY) @@ -573,7 +584,7 @@ def _get_new_state(self, unit): try: read_only = resp.data[0]['ro'] except (IndexError, KeyError) as exc: - msg = (f"Incorrect box.info response from {unit.addr['host']}:{unit.addr['port']}" + msg = (f"Incorrect box.info response from {unit.get_address()}" f"reason: {repr(exc)}") warn(msg, PoolTolopogyWarning) return InstanceState(Status.UNHEALTHY) @@ -582,11 +593,11 @@ def _get_new_state(self, unit): status = resp.data[0]['status'] if status != 'running': - msg = f"{unit.addr['host']}:{unit.addr['port']} instance status is not 'running'" + msg = f"{unit.get_address()} instance status is not 'running'" warn(msg, PoolTolopogyWarning) return InstanceState(Status.UNHEALTHY) except (IndexError, KeyError) as exc: - msg = (f"Incorrect box.info response from {unit.addr['host']}:{unit.addr['port']}" + msg = (f"Incorrect box.info response from {unit.get_address()}" f"reason: {repr(exc)}") warn(msg, PoolTolopogyWarning) return InstanceState(Status.UNHEALTHY) diff --git a/tarantool/const.py b/tarantool/const.py index 1e2b0895..53749cec 100644 --- a/tarantool/const.py +++ b/tarantool/const.py @@ -103,6 +103,12 @@ IPROTO_FEATURE_SPACE_AND_INDEX_NAMES = 5 IPROTO_FEATURE_WATCH_ONCE = 6 +# Default value for host. +DEFAULT_HOST = None +# Default value for port. +DEFAULT_PORT = None +# Default value for socket_fd. +DEFAULT_SOCKET_FD = None # Default value for connection timeout (seconds) CONNECTION_TIMEOUT = None # Default value for socket timeout (seconds) diff --git a/tarantool/mesh_connection.py b/tarantool/mesh_connection.py index 74b8af43..0f86c0dd 100644 --- a/tarantool/mesh_connection.py +++ b/tarantool/mesh_connection.py @@ -28,6 +28,9 @@ DEFAULT_SSL_PASSWORD, DEFAULT_SSL_PASSWORD_FILE, CLUSTER_DISCOVERY_DELAY, + DEFAULT_HOST, + DEFAULT_SOCKET_FD, + DEFAULT_PORT, ) from tarantool.request import ( @@ -35,6 +38,9 @@ ) default_addr_opts = { + 'host': DEFAULT_HOST, + 'port': DEFAULT_PORT, + 'socket_fd': DEFAULT_SOCKET_FD, 'transport': DEFAULT_TRANSPORT, 'ssl_key_file': DEFAULT_SSL_KEY_FILE, 'ssl_cert_file': DEFAULT_SSL_CERT_FILE, @@ -91,7 +97,8 @@ def parse_error(uri, msg): return parse_error(uri, 'port should be a number') for key, val in default_addr_opts.items(): - result[key] = val + if key not in result: + result[key] = val if opts_str != "": for opt_str in opts_str.split('&'): @@ -127,9 +134,6 @@ def format_error(address, err): if not isinstance(address, dict): return format_error(address, 'address must be a dict') - if 'port' not in address or address['port'] is None: - return format_error(address, 'port is not set or None') - result = {} for key, val in address.items(): result[key] = val @@ -138,6 +142,17 @@ def format_error(address, err): if key not in result: result[key] = val + if result['socket_fd'] is not None: + # Looks like socket fd. + if result['host'] is not None or result['port'] is not None: + return format_error(result, + "specifying both socket_fd and host/port is not allowed") + + if not isinstance(result['socket_fd'], int): + return format_error(result, + 'socket_fd must be an int') + return result, None + if isinstance(result['port'], int): # Looks like an inet address. @@ -192,6 +207,7 @@ def update_connection(conn, address): conn.host = address["host"] conn.port = address["port"] + conn.socket_fd = address["socket_fd"] conn.transport = address['transport'] conn.ssl_key_file = address['ssl_key_file'] conn.ssl_cert_file = address['ssl_cert_file'] @@ -268,7 +284,10 @@ class MeshConnection(Connection): Represents a connection to a cluster of Tarantool servers. """ - def __init__(self, host=None, port=None, + def __init__(self, + host=DEFAULT_HOST, + port=DEFAULT_PORT, + socket_fd=DEFAULT_SOCKET_FD, user=None, password=None, socket_timeout=SOCKET_TIMEOUT, @@ -302,6 +321,11 @@ def __init__(self, host=None, port=None, Value would be used to add one more server in :paramref:`~tarantool.MeshConnection.params.addrs` list. + :param socket_fd: Refer to + :paramref:`~tarantol.Connection.params.socket_fd`. + Value would be used to add one more server in + :paramref:`~tarantool.MeshConnection.params.addrs` list. + :param user: Refer to :paramref:`~tarantool.Connection.params.user`. Value would be used to add one more server in @@ -447,9 +471,10 @@ def __init__(self, host=None, port=None, # Don't change user provided arguments. addrs = addrs[:] - if host and port: + if (host and port) or socket_fd: addrs.insert(0, {'host': host, 'port': port, + 'socket_fd': socket_fd, 'transport': transport, 'ssl_key_file': ssl_key_file, 'ssl_cert_file': ssl_cert_file, @@ -484,6 +509,7 @@ def __init__(self, host=None, port=None, super().__init__( host=addr['host'], port=addr['port'], + socket_fd=addr['socket_fd'], user=user, password=password, socket_timeout=socket_timeout, @@ -604,6 +630,7 @@ def _opt_refresh_instances(self): # an instance list and connect to one of new instances. current_addr = {'host': self.host, 'port': self.port, + 'socket_fd': self.socket_fd, 'transport': self.transport, 'ssl_key_file': self.ssl_key_file, 'ssl_cert_file': self.ssl_cert_file, diff --git a/test/suites/__init__.py b/test/suites/__init__.py index d56b2889..7d092585 100644 --- a/test/suites/__init__.py +++ b/test/suites/__init__.py @@ -15,6 +15,7 @@ from .test_execute import TestSuiteExecute from .test_dbapi import TestSuiteDBAPI from .test_encoding import TestSuiteEncoding +from .test_socket_fd import TestSuiteSocketFD from .test_ssl import TestSuiteSsl from .test_decimal import TestSuiteDecimal from .test_uuid import TestSuiteUUID @@ -33,7 +34,7 @@ TestSuiteEncoding, TestSuitePool, TestSuiteSsl, TestSuiteDecimal, TestSuiteUUID, TestSuiteDatetime, TestSuiteInterval, TestSuitePackage, TestSuiteErrorExt, - TestSuitePush, TestSuiteConnection, TestSuiteCrud,) + TestSuitePush, TestSuiteConnection, TestSuiteCrud, TestSuiteSocketFD) def load_tests(loader, tests, pattern): diff --git a/test/suites/lib/skip.py b/test/suites/lib/skip.py index 00b1a21d..625caf6a 100644 --- a/test/suites/lib/skip.py +++ b/test/suites/lib/skip.py @@ -306,3 +306,14 @@ def skip_or_run_iproto_basic_features_test(func): return skip_or_run_test_tarantool(func, '2.10.0', 'does not support iproto ID and iproto basic features') + + +def skip_or_run_box_session_new_tests(func): + """ + Decorator to skip or run tests that use box.session.new. + + Tarantool supports box.session.new only in current master since + commit 324872a. + See https://github.com/tarantool/tarantool/issues/8801. + """ + return skip_or_run_test_tarantool(func, '3.0.0', 'does not support box.session.new') diff --git a/test/suites/sidecar.py b/test/suites/sidecar.py new file mode 100644 index 00000000..6bee5af7 --- /dev/null +++ b/test/suites/sidecar.py @@ -0,0 +1,16 @@ +# pylint: disable=missing-module-docstring +import os + +import tarantool + +socket_fd = int(os.environ["SOCKET_FD"]) + +conn = tarantool.connect(None, None, socket_fd=socket_fd) + +# Check user. +assert conn.eval("return box.session.user()").data[0] == "test" + +# Check db operations. +conn.insert("test", [1]) +conn.insert("test", [2]) +assert conn.select("test").data == [[1], [2]] diff --git a/test/suites/test_mesh.py b/test/suites/test_mesh.py index 606cc7d1..a906597b 100644 --- a/test/suites/test_mesh.py +++ b/test/suites/test_mesh.py @@ -135,7 +135,6 @@ def test_01_contructor(self): # Verify that a bad address given at initialization leads # to an error. bad_addrs = [ - {"port": 1234}, # no host {"host": "localhost"}, # no port {"host": "localhost", "port": "1234"}, # port is str ] diff --git a/test/suites/test_socket_fd.py b/test/suites/test_socket_fd.py new file mode 100644 index 00000000..5cf94777 --- /dev/null +++ b/test/suites/test_socket_fd.py @@ -0,0 +1,200 @@ +""" +This module tests work with connection over socket fd. +""" +import os.path +# pylint: disable=missing-class-docstring,missing-function-docstring + +import socket +import sys +import unittest + +import tarantool +from .lib.skip import skip_or_run_box_session_new_tests +from .lib.tarantool_server import TarantoolServer, find_port +from .utils import assert_admin_success + + +def find_python(): + for _dir in os.environ["PATH"].split(os.pathsep): + exe = os.path.join(_dir, "python") + if os.access(exe, os.X_OK): + return os.path.abspath(exe) + raise RuntimeError("Can't find python executable in " + os.environ["PATH"]) + + +class TestSuiteSocketFD(unittest.TestCase): + EVAL_USER = "return box.session.user()" + + @classmethod + def setUpClass(cls): + print(' SOCKET FD '.center(70, '='), file=sys.stderr) + print('-' * 70, file=sys.stderr) + + cls.srv = TarantoolServer() + cls.srv.script = 'test/suites/box.lua' + cls.srv.start() + cls.tcp_port = find_port() + + # Start tcp server to test work with blocking sockets. + # pylint: disable=consider-using-f-string + resp = cls.srv.admin(""" + local socket = require('socket') + + box.cfg{} + box.schema.user.create('test', {password = 'test', if_not_exists = true}) + box.schema.user.grant('test', 'read,write,execute,create', 'universe', + nil, {if_not_exists = true}) + box.schema.user.grant('guest', 'execute', 'universe', + nil, {if_not_exists = true}) + + socket.tcp_server('0.0.0.0', %d, function(s) + if not s:nonblock(true) then + s:close() + return + end + box.session.new({ + type = 'binary', + fd = s:fd(), + user = 'test', + }) + s:detach() + end) + + box.schema.create_space('test', { + format = {{type='unsigned', name='id'}}, + if_not_exists = true, + }) + box.space.test:create_index('primary') + + return true + """ % cls.tcp_port) + assert_admin_success(resp) + + @skip_or_run_box_session_new_tests + def setUp(self): + # Prevent a remote tarantool from clean our session. + if self.srv.is_started(): + self.srv.touch_lock() + + def _get_tt_sock(self): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect((self.srv.host, self.tcp_port)) + return sock + + def test_01_incorrect_params(self): + cases = { + "host and socket_fd": { + "args": {"host": "123", "socket_fd": 3}, + "msg": "specifying both socket_fd and host/port is not allowed", + }, + "port and socket_fd": { + "args": {"port": 14, "socket_fd": 3}, + "msg": "specifying both socket_fd and host/port is not allowed", + }, + "empty": { + "args": {}, + "msg": r"host/port.* port.* or socket_fd", + }, + "only host": { + "args": {"host": "localhost"}, + "msg": "when specifying host, it is also necessary to specify port", + }, + } + + for name, case in cases.items(): + with self.subTest(msg=name): + with self.assertRaisesRegex(tarantool.Error, case["msg"]): + tarantool.Connection(**case["args"]) + + def test_02_socket_fd_connect(self): + sock = self._get_tt_sock() + conn = tarantool.connect(None, None, socket_fd=sock.fileno()) + sock.detach() + try: + self.assertSequenceEqual(conn.eval(self.EVAL_USER), ["test"]) + finally: + conn.close() + + def test_03_socket_fd_re_auth(self): + sock = self._get_tt_sock() + conn = tarantool.connect(None, None, socket_fd=sock.fileno(), user="guest") + sock.detach() + try: + self.assertSequenceEqual(conn.eval(self.EVAL_USER), ["guest"]) + finally: + conn.close() + + @unittest.skipIf(sys.platform.startswith("win"), + "Skip on Windows since it uses remote server") + def test_04_tarantool_made_socket(self): + python_exe = find_python() + cwd = os.getcwd() + side_script_path = os.path.join(cwd, "test", "suites", "sidecar.py") + + # pylint: disable=consider-using-f-string + ret_code, err = self.srv.admin(""" + local socket = require('socket') + local popen = require('popen') + local os = require('os') + local s1, s2 = socket.socketpair('AF_UNIX', 'SOCK_STREAM', 0) + + --[[ Tell sidecar which fd use to connect. --]] + os.setenv('SOCKET_FD', tostring(s2:fd())) + + --[[ Tell sidecar where find `tarantool` module. --]] + os.setenv('PYTHONPATH', (os.getenv('PYTHONPATH') or '') .. ':' .. '%s') + + box.session.new({ + type = 'binary', + fd = s1:fd(), + user = 'test', + }) + s1:detach() + + local ph, err = popen.new({'%s', '%s'}, { + stdout = popen.opts.PIPE, + stderr = popen.opts.PIPE, + inherit_fds = {s2:fd()}, + }) + + if err ~= nil then + return 1, err + end + + ph:wait() + + local status_code = ph:info().status.exit_code + local stderr = ph:read({stderr=true}):rstrip() + return status_code, stderr + """ % (cwd, python_exe, side_script_path)) + self.assertIsNone(err, err) + self.assertEqual(ret_code, 0) + + def test_05_socket_fd_pool(self): + sock = self._get_tt_sock() + pool = tarantool.ConnectionPool( + addrs=[{'host': None, 'port': None, 'socket_fd': sock.fileno()}] + ) + sock.detach() + try: + self.assertSequenceEqual(pool.eval(self.EVAL_USER, mode=tarantool.Mode.ANY), ["test"]) + finally: + pool.close() + + def test_06_socket_fd_mesh(self): + sock = self._get_tt_sock() + mesh = tarantool.MeshConnection( + host=None, + port=None, + socket_fd=sock.fileno() + ) + sock.detach() + try: + self.assertSequenceEqual(mesh.eval(self.EVAL_USER), ["test"]) + finally: + mesh.close() + + @classmethod + def tearDownClass(cls): + cls.srv.stop() + cls.srv.clean() From 4a682699edd20fa2888d9bcb4855147334a5fc8e Mon Sep 17 00:00:00 2001 From: Yaroslav Lobankov Date: Thu, 14 Mar 2024 17:09:50 +0400 Subject: [PATCH 25/39] ci: bump actions in reusable_testing.yml Bump version of the `actions/checkout` and `actions/download-artifact` actions to v4. Bump version of the `actions/setup-python` action to v5. It is needed for fixing the following GitHub warning: Node.js 16 actions are deprecated. Please update the following actions to use Node.js 20 --- .github/workflows/reusable_testing.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/reusable_testing.yml b/.github/workflows/reusable_testing.yml index e9441532..406a7301 100644 --- a/.github/workflows/reusable_testing.yml +++ b/.github/workflows/reusable_testing.yml @@ -14,12 +14,12 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Clone the tarantool-python connector - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: ${{ github.repository_owner }}/tarantool-python - name: Download the tarantool build artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: ${{ inputs.artifact_name }} @@ -29,7 +29,7 @@ jobs: run: sudo dpkg -i tarantool*.deb - name: Setup python3 for tests - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.11' From a12f6e7f8e1b23a8ed2e5076cdcfa130af15a81f Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Wed, 27 Mar 2024 10:55:00 +0300 Subject: [PATCH 26/39] release 1.2.0 Overview This release introduces socket fd connect support. Added - The ability to connect to the Tarantool using an existing socket fd (#304). --- CHANGELOG.md | 2 +- debian/changelog | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3bd49f1..28f93b19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Unreleased +## 1.2.0 - 2024-03-27 ### Added - The ability to connect to the Tarantool using an existing socket fd (#304). diff --git a/debian/changelog b/debian/changelog index c8d5153d..07127a11 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,15 @@ +python3-tarantool (1.2.0-0) unstable; urgency=medium + + ## Overview + + This release introduces socket fd connect support. + + ## Added + + - The ability to connect to the Tarantool using an existing socket fd (#304). + + -- Georgy Moiseev Wed, 27 Mar 2024 11:00:00 +0300 + python3-tarantool (1.1.2-0) unstable; urgency=medium ## Overview From 3d397aa86bf71088e91a7d7ee3a74d2583a426b0 Mon Sep 17 00:00:00 2001 From: Nikolay Shirokovskiy Date: Fri, 29 Mar 2024 14:20:07 +0300 Subject: [PATCH 27/39] test: use coverage instead of equality to test error payload We are going to add missing 'user' payload field for ACCESS_DENIED error which will break current tests. Let rewrite tests to allow adding new payload fields for this error. Need for https://github.com/tarantool/tarantool/issues/9108 --- test/suites/test_dml.py | 6 +++--- test/suites/test_error_ext.py | 6 +++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/test/suites/test_dml.py b/test/suites/test_dml.py index f557451e..50b71905 100644 --- a/test/suites/test_dml.py +++ b/test/suites/test_dml.py @@ -419,13 +419,13 @@ def test_16_extra_error_info_fields(self): "Create access to function 'forbidden_function' is denied for user 'test'") self.assertEqual(exc.extra_info.errno, 0) self.assertEqual(exc.extra_info.errcode, 42) - self.assertEqual( - exc.extra_info.fields, + self.assertGreaterEqual( + exc.extra_info.fields.items(), { 'object_type': 'function', 'object_name': 'forbidden_function', 'access_type': 'Create' - }) + }.items()) self.assertEqual(exc.extra_info.prev, None) else: self.fail('Expected error') diff --git a/test/suites/test_error_ext.py b/test/suites/test_error_ext.py index 52622794..c6cdf4b5 100644 --- a/test/suites/test_error_ext.py +++ b/test/suites/test_error_ext.py @@ -327,7 +327,11 @@ def test_tarantool_decode(self): self.assertEqual(err.message, expected_err.message) self.assertEqual(err.errno, expected_err.errno) self.assertEqual(err.errcode, expected_err.errcode) - self.assertEqual(err.fields, expected_err.fields) + if expected_err.fields is not None: + self.assertGreaterEqual(err.fields.items(), + expected_err.fields.items()) + else: + self.assertEqual(err.fields, None) err = err.prev expected_err = expected_err.prev From acc1446a921044be814f68b245c585226ca183a0 Mon Sep 17 00:00:00 2001 From: Nikolay Shirokovskiy Date: Thu, 11 Apr 2024 13:26:19 +0300 Subject: [PATCH 28/39] test: use coverage instead of equality to test error payload p.2 We are going to add 'name' payload field for every client error. So we need to tweak more test to handle this. Need for https://github.com/tarantool/tarantool/issues/9875 --- test/suites/test_dml.py | 14 ++++++++++++-- test/suites/test_error_ext.py | 13 ++++++++----- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/test/suites/test_dml.py b/test/suites/test_dml.py index 50b71905..26539eec 100644 --- a/test/suites/test_dml.py +++ b/test/suites/test_dml.py @@ -391,7 +391,12 @@ def test_15_extra_error_info_stacked(self): self.assertEqual(exc.extra_info.message, "Timeout exceeded") self.assertEqual(exc.extra_info.errno, 0) self.assertEqual(exc.extra_info.errcode, 78) - self.assertEqual(exc.extra_info.fields, None) + actual_fields = exc.extra_info.fields + if actual_fields is None: + actual_fields = {} + expected_fields = {} + self.assertGreaterEqual(actual_fields.items(), + expected_fields.items()) self.assertNotEqual(exc.extra_info.prev, None) prev = exc.extra_info.prev self.assertEqual(prev.type, 'ClientError') @@ -400,7 +405,12 @@ def test_15_extra_error_info_stacked(self): self.assertEqual(prev.message, "Unknown error") self.assertEqual(prev.errno, 0) self.assertEqual(prev.errcode, 0) - self.assertEqual(prev.fields, None) + actual_fields = prev.fields + if actual_fields is None: + actual_fields = {} + expected_fields = {} + self.assertGreaterEqual(actual_fields.items(), + expected_fields.items()) else: self.fail('Expected error') diff --git a/test/suites/test_error_ext.py b/test/suites/test_error_ext.py index c6cdf4b5..d5e96e75 100644 --- a/test/suites/test_error_ext.py +++ b/test/suites/test_error_ext.py @@ -327,11 +327,14 @@ def test_tarantool_decode(self): self.assertEqual(err.message, expected_err.message) self.assertEqual(err.errno, expected_err.errno) self.assertEqual(err.errcode, expected_err.errcode) - if expected_err.fields is not None: - self.assertGreaterEqual(err.fields.items(), - expected_err.fields.items()) - else: - self.assertEqual(err.fields, None) + expected_fields = expected_err.fields + if expected_fields is None: + expected_fields = {} + actual_fields = err.fields + if actual_fields is None: + actual_fields = {} + self.assertGreaterEqual(actual_fields.items(), + expected_fields.items()) err = err.prev expected_err = expected_err.prev From 5507a6eb445d4850b7eeb8804cd7ff19ee54401f Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Fri, 24 May 2024 10:43:49 +0300 Subject: [PATCH 29/39] codespell: ignore unittest naming After resent update, codespell started to treat assertIn as a typo, and CI started to fail. --- .codespellrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.codespellrc b/.codespellrc index fe94c2d8..291bd5cf 100644 --- a/.codespellrc +++ b/.codespellrc @@ -1,3 +1,3 @@ [codespell] skip = tarantool/msgpack_ext/types/timezones/timezones.py -ignore-words-list = ans,gost,ro +ignore-words-list = ans,gost,ro,assertIn From 717f275f55284c945973ad52d4735ff7aa275104 Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Fri, 24 May 2024 11:09:08 +0300 Subject: [PATCH 30/39] ci: fix deb build It seems that something has been changed in the package dependencies and `install` no longer works here without `update`. --- .github/workflows/packing.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/packing.yml b/.github/workflows/packing.yml index b5b84f32..c2456477 100644 --- a/.github/workflows/packing.yml +++ b/.github/workflows/packing.yml @@ -414,6 +414,7 @@ jobs: - name: Install deb packing tools run: | + sudo apt update sudo apt install -y devscripts equivs - name: Make changelog entry for non-release build From e50f7624dea0c0d9ddcd372703e2648037af7133 Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Wed, 4 Sep 2024 09:56:07 +0300 Subject: [PATCH 31/39] setup: fix import order --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4ee0797c..87bcc4c4 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ import codecs import os -from setuptools import setup, find_packages +from setuptools import find_packages, setup from setuptools.command.build_py import build_py # Extra commands for documentation management From 7c5e9a047931553768e5ca1117ff8557a8377b7f Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Wed, 4 Sep 2024 09:33:02 +0300 Subject: [PATCH 32/39] ci: fix download artifact vulnerability Versions of actions/download-artifact before 4.1.7 are vulnerable to arbitrary file write when downloading and extracting a specifically crafted artifact that contains path traversal filenames [1]. 1. https://github.com/tarantool/tarantool-python/security/dependabot/4 --- .github/workflows/packing.yml | 20 ++++++++++---------- .github/workflows/reusable_testing.yml | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/packing.yml b/.github/workflows/packing.yml index c2456477..c286abf1 100644 --- a/.github/workflows/packing.yml +++ b/.github/workflows/packing.yml @@ -43,7 +43,7 @@ jobs: run: make pip-dist-check - name: Archive pip artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4.4.0 with: name: pip_dist path: pip_dist @@ -84,7 +84,7 @@ jobs: tarantool-version: '2.11' - name: Download pip package artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4.1.8 with: name: pip_dist path: pip_dist @@ -134,7 +134,7 @@ jobs: run: python3 .github/scripts/remove_source_code.py - name: Download pip package artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4.1.8 with: name: pip_dist path: pip_dist @@ -202,7 +202,7 @@ jobs: run: pip3 install twine - name: Download pip package artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4.1.8 with: name: pip_dist path: pip_dist @@ -271,7 +271,7 @@ jobs: run: make rpm-dist-check - name: Archive RPM artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4.4.0 with: name: rpm_dist_${{ matrix.target.os }}_${{ matrix.target.dist }} path: rpm_dist @@ -324,7 +324,7 @@ jobs: dnf install -y tarantool tarantool-devel - name: Download RPM artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4.1.8 with: name: rpm_dist_${{ matrix.target.os }}_${{ matrix.target.dist }} path: rpm_dist @@ -372,7 +372,7 @@ jobs: run: sudo apt install -y curl make - name: Download RPM artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4.1.8 with: name: rpm_dist_${{ matrix.target.os }}_${{ matrix.target.dist }} path: rpm_dist @@ -433,7 +433,7 @@ jobs: run: make deb-dist-check - name: Archive deb artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4.4.0 with: name: deb_dist path: deb_dist @@ -490,7 +490,7 @@ jobs: DEBIAN_FRONTEND: noninteractive - name: Download deb artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4.1.8 with: name: deb_dist path: deb_dist @@ -542,7 +542,7 @@ jobs: run: sudo apt install -y curl make - name: Download deb artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4.1.8 with: name: deb_dist path: deb_dist diff --git a/.github/workflows/reusable_testing.yml b/.github/workflows/reusable_testing.yml index 406a7301..aec47845 100644 --- a/.github/workflows/reusable_testing.yml +++ b/.github/workflows/reusable_testing.yml @@ -19,7 +19,7 @@ jobs: repository: ${{ github.repository_owner }}/tarantool-python - name: Download the tarantool build artifact - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v4.1.8 with: name: ${{ inputs.artifact_name }} From d291a451865cbd9cd7859dfca0206533f0a1a806 Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Wed, 4 Sep 2024 09:43:43 +0300 Subject: [PATCH 33/39] ci: cancel previous runs on PR push After this patch, current PR pipeline runs will be cancelled if new commits/force push triggers new pipelines. --- .github/workflows/packing.yml | 4 ++++ .github/workflows/testing.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.github/workflows/packing.yml b/.github/workflows/packing.yml index c286abf1..c719b31a 100644 --- a/.github/workflows/packing.yml +++ b/.github/workflows/packing.yml @@ -6,6 +6,10 @@ on: pull_request_target: types: [labeled] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: pack_pip: # We want to run on external PRs, but not on our own internal diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 82f299b4..05927f4c 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -6,6 +6,10 @@ on: pull_request_target: types: [labeled] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: run_tests_ce_linux: # We want to run on external PRs, but not on our own internal From 70c92fdee672b67e9536b36fd5f727c44ae8146f Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Wed, 4 Sep 2024 11:08:30 +0300 Subject: [PATCH 34/39] package: drop Python 3.6 support Python 3.6 EOL is 2021-12-23 [1]. Current build script no longer supports Python 3.6 due to `packaging` Python dependency bump. (And fixing the issue is more than just fixating older `packaging` version as a dependency.) https://devguide.python.org/versions/ --- .github/workflows/testing.yml | 9 ++++----- CHANGELOG.md | 5 +++++ requirements.txt | 1 - setup.py | 4 ++-- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 05927f4c..b072be05 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -33,7 +33,6 @@ jobs: - '2.11' - 'master' python: - - '3.6' - '3.7' - '3.8' - '3.9' @@ -157,7 +156,7 @@ jobs: path: 'release/linux/x86_64/2.10/' - bundle: 'sdk-gc64-2.11.0-0-r563.linux.x86_64' path: 'release/linux/x86_64/2.11/' - python: ['3.6', '3.11'] + python: ['3.7', '3.11'] steps: - name: Clone the connector @@ -225,7 +224,7 @@ jobs: tarantool: - '2.11' python: - - '3.6' + - '3.7' - '3.11' steps: - name: Clone the connector repo @@ -278,7 +277,7 @@ jobs: tarantool: - '2.11.0.g247a9a418-1' python: - - '3.6' + - '3.7' - '3.11' steps: @@ -348,7 +347,7 @@ jobs: tarantool: - '2.11.0.g247a9a418-1' python: - - '3.6' + - '3.7' - '3.11' steps: - name: Clone the connector repo diff --git a/CHANGELOG.md b/CHANGELOG.md index 28f93b19..8edd47e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Changed +- Drop Python 3.6 support (PR #327). + ## 1.2.0 - 2024-03-27 ### Added diff --git a/requirements.txt b/requirements.txt index d88dbea3..afcf7b25 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ msgpack pytz -dataclasses; python_version <= '3.6' diff --git a/setup.py b/setup.py index 87bcc4c4..3fa65c9c 100755 --- a/setup.py +++ b/setup.py @@ -112,7 +112,7 @@ def get_dependencies(filename): command_options=command_options, install_requires=get_dependencies('requirements.txt'), setup_requires=[ - 'setuptools_scm==6.4.2', + 'setuptools_scm==7.1.0', ], - python_requires='>=3.6', + python_requires='>=3.7', ) From 61e96fcedfef75ee7c4dae4d9bca3c1153e6db85 Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Wed, 4 Sep 2024 11:09:53 +0300 Subject: [PATCH 35/39] ci: bump installer script --- .github/workflows/packing.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/packing.yml b/.github/workflows/packing.yml index c719b31a..6613f218 100644 --- a/.github/workflows/packing.yml +++ b/.github/workflows/packing.yml @@ -324,7 +324,7 @@ jobs: - name: Install tarantool run: | - curl -L https://tarantool.io/yeohchA/release/2/installer.sh | bash + curl -L https://tarantool.io/release/2/installer.sh | bash dnf install -y tarantool tarantool-devel - name: Download RPM artifacts @@ -488,7 +488,7 @@ jobs: - name: Install tarantool ${{ matrix.tarantool }} run: | apt install -y curl - curl -L https://tarantool.io/yeohchA/release/2/installer.sh | bash + curl -L https://tarantool.io/release/2/installer.sh | bash apt install -y tarantool tarantool-dev env: DEBIAN_FRONTEND: noninteractive From 938db75a4d11b33a8e91565fd4f6952249cb7592 Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Sun, 22 Sep 2024 19:28:49 +0300 Subject: [PATCH 36/39] pylint: ignore too-many-positional-arguments Pylint 3.3.0 includes a new warning: too-many-positional-arguments [1]. We already ignore a similar too-many-arguments warning in several places since "fixing" it is backward-incompatible. 1. https://github.com/pylint-dev/pylint/issues/9099 --- tarantool/__init__.py | 2 +- tarantool/connection.py | 2 +- tarantool/connection_pool.py | 2 +- tarantool/mesh_connection.py | 2 +- tarantool/request.py | 8 ++++---- test/suites/lib/tarantool_server.py | 2 +- test/suites/test_ssl.py | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tarantool/__init__.py b/tarantool/__init__.py index d7e99358..97a1e860 100644 --- a/tarantool/__init__.py +++ b/tarantool/__init__.py @@ -1,7 +1,7 @@ """ This package provides API for interaction with a Tarantool server. """ -# pylint: disable=too-many-arguments +# pylint: disable=too-many-arguments,too-many-positional-arguments from tarantool.connection import Connection from tarantool.mesh_connection import MeshConnection diff --git a/tarantool/connection.py b/tarantool/connection.py index 1284fa25..682b0700 100644 --- a/tarantool/connection.py +++ b/tarantool/connection.py @@ -812,7 +812,7 @@ def __init__(self, .. _mp_bin: https://github.com/msgpack/msgpack/blob/master/spec.md#bin-format-family .. _mp_array: https://github.com/msgpack/msgpack/blob/master/spec.md#array-format-family """ - # pylint: disable=too-many-arguments,too-many-locals,too-many-statements + # pylint: disable=too-many-arguments,too-many-locals,too-many-statements,too-many-positional-arguments if host is None and port is None and socket_fd is None: raise ConfigurationError("need to specify host/port, " diff --git a/tarantool/connection_pool.py b/tarantool/connection_pool.py index 7bbd4e16..f1e675b3 100644 --- a/tarantool/connection_pool.py +++ b/tarantool/connection_pool.py @@ -482,7 +482,7 @@ def __init__(self, .. _box.info.status: .. _box.info: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_info/ """ - # pylint: disable=too-many-arguments,too-many-locals + # pylint: disable=too-many-arguments,too-many-locals,too-many-positional-arguments if not isinstance(addrs, list) or len(addrs) == 0: raise ConfigurationError("addrs must be non-empty list") diff --git a/tarantool/mesh_connection.py b/tarantool/mesh_connection.py index 0f86c0dd..02a20d7d 100644 --- a/tarantool/mesh_connection.py +++ b/tarantool/mesh_connection.py @@ -463,7 +463,7 @@ def __init__(self, :class:`~tarantool.Connection` exceptions, :class:`~tarantool.MeshConnection.connect` exceptions """ - # pylint: disable=too-many-arguments,too-many-locals + # pylint: disable=too-many-arguments,too-many-locals,too-many-positional-arguments if addrs is None: addrs = [] diff --git a/tarantool/request.py b/tarantool/request.py index d8d4dd22..95164b79 100644 --- a/tarantool/request.py +++ b/tarantool/request.py @@ -269,7 +269,7 @@ def __init__(self, conn, salt, user, password, auth_type=AUTH_TYPE_CHAP_SHA1): :param auth_type: Refer to :paramref:`~tarantool.Connection.auth_type`. :type auth_type: :obj:`str`, optional """ - # pylint: disable=too-many-arguments + # pylint: disable=too-many-arguments,too-many-positional-arguments super().__init__(conn) @@ -405,7 +405,7 @@ def __init__(self, conn, space_no, index_no, key, offset, limit, iterator): :raise: :exc:`~AssertionError` """ - # pylint: disable=too-many-arguments + # pylint: disable=too-many-arguments,too-many-positional-arguments super().__init__(conn) request_body = self._dumps({IPROTO_SPACE_ID: space_no, @@ -446,7 +446,7 @@ def __init__(self, conn, space_no, index_no, key, op_list): :raise: :exc:`~AssertionError` """ - # pylint: disable=too-many-arguments + # pylint: disable=too-many-arguments,too-many-positional-arguments super().__init__(conn) @@ -569,7 +569,7 @@ def __init__(self, conn, space_no, index_no, tuple_value, op_list): :raise: :exc:`~AssertionError` """ - # pylint: disable=too-many-arguments + # pylint: disable=too-many-arguments,too-many-positional-arguments super().__init__(conn) diff --git a/test/suites/lib/tarantool_server.py b/test/suites/lib/tarantool_server.py index c8f34b37..55f6ca63 100644 --- a/test/suites/lib/tarantool_server.py +++ b/test/suites/lib/tarantool_server.py @@ -56,7 +56,7 @@ class TarantoolServer(): """ Class to start up a new Tarantool server. """ - # pylint: disable=too-many-instance-attributes,too-many-arguments,duplicate-code + # pylint: disable=too-many-instance-attributes,too-many-arguments,duplicate-code,too-many-positional-arguments default_tarantool = { "bin": "tarantool", diff --git a/test/suites/test_ssl.py b/test/suites/test_ssl.py index 9452e83b..b7d1da97 100644 --- a/test/suites/test_ssl.py +++ b/test/suites/test_ssl.py @@ -34,7 +34,7 @@ def is_test_ssl(): class SslTestCase: - # pylint: disable=too-few-public-methods,too-many-instance-attributes,too-many-arguments + # pylint: disable=too-few-public-methods,too-many-instance-attributes,too-many-arguments,too-many-positional-arguments def __init__(self, name="", From 1574e4cd8c59c096d0510af418dc8febc59ef923 Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Sun, 22 Sep 2024 19:51:59 +0300 Subject: [PATCH 37/39] pylint: ignore unknown-option-value Disabling any new pylint warnings per-entity results in older pylint failing due to unknown option. Since we support several pylint versions due to Python version restrictions, we need to support multiple pylint versions as well. --- .pylintrc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.pylintrc b/.pylintrc index 3fd8b942..385d67ad 100644 --- a/.pylintrc +++ b/.pylintrc @@ -6,3 +6,8 @@ good-names=i,j,k,ex,Run,_,ok,t,tz [FORMAT] # Allow links in docstings, allow tables ignore-long-lines=^(?:\s*(# )?(?:\.\.\s.+?:)?\s*?)|(\s\+.+\+)|(\s\|.+\|)$ + +[MESSAGES CONTROL] +# Ignore unknown options to support per-entity ignores for newest warnings/errors +# which are not supported for older versions. +disable=unknown-option-value From b6422af1914db169fbf7f8b074731002e725a3d3 Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Sun, 22 Sep 2024 19:41:06 +0300 Subject: [PATCH 38/39] test: fixate linters version strictly Any new linter release, including bugfix ones, may result in failing CI. Since tarantool-python CI is included in several integration pipelines, such things may block work on repos like tarantool/tarantool and tarantool/luajit. On the other hand, if CI will start to fail due to a new linter release, it is unlikely that something user-critical will be discovered. So for now let's stick to the following approach: one will bump linters manually from time to time and fix/disable emerging issues. --- requirements-test.txt | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/requirements-test.txt b/requirements-test.txt index 56f4b595..4d16b8ca 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,8 +1,10 @@ git+https://github.com/baztian/dbapi-compliance.git@ea7cb1b4#egg=dbapi-compliance pyyaml==6.0 importlib-metadata >= 1.0 ; python_version < '3.8' -pylint ~= 3.0 ; python_version >= '3.8' -pylint ~= 2.13 ; python_version < '3.8' -flake8 ~= 6.1 ; python_version >= '3.8' -flake8 ~= 5.0 ; python_version < '3.8' -codespell ~= 2.2 +pylint == 3.3.0 ; python_version >= '3.9' +pylint == 3.2.7 ; python_version == '3.8' +pylint == 2.17.7 ; python_version == '3.7' +flake8 == 6.1.0 ; python_version >= '3.8' +flake8 == 5.0.4 ; python_version < '3.8' +codespell == 2.3.0 ; python_version >= '3.8' +codespell == 2.2.5 ; python_version < '3.8' From 28310fc4e927c77d3be75e98360bf3b09f72482a Mon Sep 17 00:00:00 2001 From: Alexey Potapenko Date: Mon, 31 Mar 2025 20:33:50 +0300 Subject: [PATCH 39/39] ci: bump ubuntu version Bump actions to use ubuntu-24.04 for fixing the following GitHub warning: The Ubuntu 20.04 Actions runner image will begin deprecation on 2025-02-01. Additionally this patch updates test requirements, list of OS to test, forces `xz` compression type to support older OS versions, hot-fixes to solve `CMake` 3.5 problem on latest Ubuntu and Debian. Part of #TNTP-1918 --- .github/workflows/packing.yml | 94 +++++++++++++++++++------- .github/workflows/reusable_testing.yml | 3 +- .github/workflows/testing.yml | 73 ++++++++++---------- .readthedocs.yaml | 3 + debian/control | 2 +- debian/rules | 4 ++ requirements-test.txt | 3 +- 7 files changed, 119 insertions(+), 63 deletions(-) diff --git a/.github/workflows/packing.yml b/.github/workflows/packing.yml index 6613f218..70970387 100644 --- a/.github/workflows/packing.yml +++ b/.github/workflows/packing.yml @@ -20,14 +20,14 @@ jobs: if: (github.event_name == 'push') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false steps: - name: Clone the connector repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Checkout all tags for correct version computation. with: fetch-depth: 0 @@ -38,7 +38,9 @@ jobs: python-version: '3.11' - name: Install tools for packing and verification - run: pip3 install wheel twine + run: | + pip3 install wheel twine + pip3 install --upgrade setuptools - name: Pack source and binary files run: make pip-dist @@ -65,14 +67,14 @@ jobs: if: (github.event_name == 'push') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false steps: - name: Clone the connector repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python uses: actions/setup-python@v4 @@ -83,7 +85,7 @@ jobs: run: python3 .github/scripts/remove_source_code.py - name: Install tarantool - uses: tarantool/setup-tarantool@v2 + uses: tarantool/setup-tarantool@v3 with: tarantool-version: '2.11' @@ -99,10 +101,13 @@ jobs: - name: Install test requirements run: pip3 install -r requirements-test.txt + # Installation of the specific CMake version is a hotfix for + # https://github.com/tarantool/checks/issues/64 - name: Install the crud module for testing purposes run: | curl -L https://tarantool.io/release/2/installer.sh | bash sudo apt install -y tt + pip3 install cmake==3.15.3 tt rocks install crud - name: Run tests @@ -127,7 +132,7 @@ jobs: steps: - name: Clone the connector repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python uses: actions/setup-python@v4 @@ -188,14 +193,14 @@ jobs: - run_tests_pip_package_linux - run_tests_pip_package_windows - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false steps: - name: Clone the connector repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python and basic packing tools uses: actions/setup-python@v4 @@ -226,7 +231,7 @@ jobs: if: (github.event_name == 'push') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 container: image: ${{ matrix.target.os }}:${{ matrix.target.dist }} @@ -250,7 +255,7 @@ jobs: run: dnf install -y git - name: Clone the connector repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Checkout all tags for correct version computation. with: fetch-depth: 0 @@ -293,7 +298,7 @@ jobs: if: (github.event_name == 'push') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 container: image: ${{ matrix.target.os }}:${{ matrix.target.dist }} @@ -312,7 +317,7 @@ jobs: steps: - name: Clone the connector repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python and test running tools # cmake rocks fail to install as expected without findutils: @@ -343,6 +348,7 @@ jobs: run: | curl -L https://tarantool.io/release/2/installer.sh | bash sudo dnf install -y tt + pip3 install cmake==3.15.3 tt rocks install crud - name: Run tests @@ -354,7 +360,7 @@ jobs: needs: - run_tests_rpm - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false @@ -370,7 +376,7 @@ jobs: steps: - name: Clone the connector repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install tools for package publishing run: sudo apt install -y curl make @@ -404,14 +410,14 @@ jobs: if: (github.event_name == 'push') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false steps: - name: Clone the connector repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Checkout all tags for correct version computation with: fetch-depth: 0 @@ -420,6 +426,7 @@ jobs: run: | sudo apt update sudo apt install -y devscripts equivs + sudo apt install python3-setuptools python3-stdeb dh-python - name: Make changelog entry for non-release build if: startsWith(github.ref, 'refs/tags') != true @@ -453,7 +460,7 @@ jobs: if: (github.event_name == 'push') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 container: image: ${{ matrix.target.os }}:${{ matrix.target.dist }} @@ -463,18 +470,20 @@ jobs: matrix: target: + - os: debian + dist: bullseye # 11 + - os: debian + dist: bookworm # 12 - os: ubuntu dist: focal # 20.04 - os: ubuntu dist: jammy # 22.04 - - os: debian - dist: buster # 10 - - os: debian - dist: bullseye # 11 + - os: ubuntu + dist: noble # 24.04 steps: - name: Clone the connector repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Prepare apt run: apt update @@ -504,17 +513,48 @@ jobs: env: DEBIAN_FRONTEND: noninteractive + # Usage of venv is mandatory starting with Debian 12 and Ubuntu 24.04. + - name: Create venv + run: | + apt install -y python3-venv + python3 -m venv .venv + - name: Install test requirements run: pip3 install -r requirements-test.txt + if: matrix.target.dist != 'bookworm' && matrix.target.dist != 'noble' + + - name: Install test requirements + run: | + . .venv/bin/activate + pip3 install -r requirements-test.txt + if: matrix.target.dist == 'bookworm' || matrix.target.dist == 'noble' - name: Install the crud module for testing purposes run: | curl -L https://tarantool.io/release/2/installer.sh | bash apt install -y tt tt rocks install crud + if: matrix.target.dist != 'bookworm' && matrix.target.dist != 'noble' + + - name: Install the crud module for testing purposes + run: | + . .venv/bin/activate + curl -L https://tarantool.io/release/3/installer.sh | bash + apt install -y tt + tt rocks install crud + if: matrix.target.dist == 'bookworm' || matrix.target.dist == 'noble' - name: Run tests run: make test-pure-install + if: matrix.target.dist != 'bookworm' && matrix.target.dist != 'noble' + + - name: Run tests + run: | + . .venv/bin/activate + export PYTHONPATH=$PYTHONPATH:/usr/lib/python3.11:/usr/lib/python3.12:/usr/bin:/usr/lib/python3/dist-packages + export PATH=$PATH:/usr/lib/python3/dist-packages + make test-pure-install + if: matrix.target.dist == 'bookworm' || matrix.target.dist == 'noble' publish_deb: if: startsWith(github.ref, 'refs/tags') @@ -522,7 +562,7 @@ jobs: needs: - run_tests_deb - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false @@ -533,14 +573,18 @@ jobs: dist: focal # 20.04 - os: ubuntu dist: jammy # 22.04 + - os: ubuntu + dist: noble # 24.04 - os: debian dist: buster # 10 - os: debian dist: bullseye # 11 + - os: debian + dist: bookworm # 12 steps: - name: Clone the connector repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install tools for package publishing run: sudo apt install -y curl make diff --git a/.github/workflows/reusable_testing.yml b/.github/workflows/reusable_testing.yml index aec47845..79f9f2a4 100644 --- a/.github/workflows/reusable_testing.yml +++ b/.github/workflows/reusable_testing.yml @@ -11,7 +11,7 @@ on: jobs: run_tests: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - name: Clone the tarantool-python connector uses: actions/checkout@v4 @@ -43,6 +43,7 @@ jobs: run: | curl -L https://tarantool.io/release/2/installer.sh | bash sudo apt install -y tt + pip3 install cmake==3.15.3 tt rocks install crud - run: make test diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index b072be05..8f94a4e6 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -21,20 +21,16 @@ jobs: (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: tarantool: - - '1.10' - - '2.8' - - '2.10' - '2.11' + - '3.3' - 'master' python: - - '3.7' - - '3.8' - '3.9' - '3.10' - '3.11' @@ -47,30 +43,35 @@ jobs: # "This page is taking too long to load." error. Thus we use # pairwise testing. include: - - tarantool: '2.11' + - tarantool: '3.3' python: '3.11' msgpack-deps: 'msgpack==0.5.0' - - tarantool: '2.11' + - tarantool: '3.3' python: '3.11' msgpack-deps: 'msgpack==0.6.2' - - tarantool: '2.11' + - tarantool: '3.3' python: '3.11' msgpack-deps: 'msgpack==1.0.4' steps: - name: Clone the connector - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup tt run: | - curl -L https://tarantool.io/release/2/installer.sh | sudo bash + curl -L https://tarantool.io/release/3/installer.sh | sudo bash sudo apt install -y tt tt version - tt init + tt init + + # Installation of the specific CMake version is a hotfix for + # https://github.com/tarantool/checks/issues/64 + - name: Install old CMake + run: pip3 install cmake==3.15.3 - name: Install tarantool ${{ matrix.tarantool }} if: matrix.tarantool != 'master' - uses: tarantool/setup-tarantool@v2 + uses: tarantool/setup-tarantool@v3 with: tarantool-version: ${{ matrix.tarantool }} @@ -123,8 +124,7 @@ jobs: run: pip3 install -r requirements-test.txt - name: Install the crud module for testing purposes - run: | - tt rocks install crud + run: tt rocks install crud - name: Run tests run: make test @@ -142,34 +142,30 @@ jobs: github.event.pull_request.head.repo.full_name != github.repository && github.event.label.name == 'full-ci') - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: tarantool: - - bundle: 'sdk-1.10.15-0-r563' - path: 'release/linux/x86_64/1.10/' - - bundle: 'sdk-2.8.4-0-r563' - path: 'release/linux/x86_64/2.8/' - - bundle: 'sdk-gc64-2.10.7-0-r563.linux.x86_64' - path: 'release/linux/x86_64/2.10/' - bundle: 'sdk-gc64-2.11.0-0-r563.linux.x86_64' path: 'release/linux/x86_64/2.11/' - python: ['3.7', '3.11'] + - bundle: 'sdk-gc64-3.3.1-0-r55.linux.x86_64' + path: 'release/linux/x86_64/3.3/' + python: ['3.9', '3.11'] steps: - name: Clone the connector # `ref` as merge request is needed for pull_request_target because this # target runs in the context of the base commit of the pull request. - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name == 'pull_request_target' with: ref: refs/pull/${{ github.event.pull_request.number }}/merge - name: Clone the connector if: github.event_name != 'pull_request_target' - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Tarantool EE SDK run: | @@ -194,8 +190,9 @@ jobs: # See more here: https://github.com/tarantool/tt/issues/282 run: | source tarantool-enterprise/env.sh - curl -L https://tarantool.io/release/2/installer.sh | bash + curl -L https://tarantool.io/release/3/installer.sh | bash sudo apt install -y tt + pip3 install cmake==3.15.3 tt rocks install crud TARANTOOL_DIR=$PWD/tarantool-enterprise - name: Run tests @@ -215,7 +212,7 @@ jobs: if: (github.event_name == 'push') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: fail-fast: false @@ -223,15 +220,15 @@ jobs: matrix: tarantool: - '2.11' + - '3.3' python: - - '3.7' - '3.11' steps: - name: Clone the connector repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install tarantool ${{ matrix.tarantool }} - uses: tarantool/setup-tarantool@v2 + uses: tarantool/setup-tarantool@v3 with: tarantool-version: ${{ matrix.tarantool }} @@ -246,13 +243,19 @@ jobs: - name: Install the package with pip run: pip3 install git+$GITHUB_SERVER_URL/$GITHUB_REPOSITORY@$GITHUB_REF + # Installation of the specific CMake version is a hotfix for + # https://github.com/tarantool/checks/issues/64 + - name: Install old CMake + run: pip3 install cmake==3.15.3 + - name: Install test requirements run: pip3 install -r requirements-test.txt - name: Install the crud module for testing purposes run: | - curl -L https://tarantool.io/release/2/installer.sh | bash + curl -L https://tarantool.io/release/3/installer.sh | bash sudo apt install -y tt + pip3 install cmake==3.15.3 tt rocks install crud - name: Run tests @@ -275,14 +278,14 @@ jobs: matrix: # Use reduced test matrix cause Windows pipelines are long. tarantool: + # https://github.com/tarantool/tarantool-python/issues/331 - '2.11.0.g247a9a418-1' python: - - '3.7' - '3.11' steps: - name: Clone the connector - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python for tests uses: actions/setup-python@v4 @@ -345,13 +348,13 @@ jobs: matrix: # Use reduced test matrix cause Windows pipelines are long. tarantool: + # https://github.com/tarantool/tarantool-python/issues/331 - '2.11.0.g247a9a418-1' python: - - '3.7' - '3.11' steps: - name: Clone the connector repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python for tests uses: actions/setup-python@v4 diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 9375a9f6..bc990993 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -13,3 +13,6 @@ python: - method: pip path: . - requirements: docs/requirements.txt + +sphinx: + configuration: docs/source/conf.py diff --git a/debian/control b/debian/control index dc7869fc..a5cfe113 100644 --- a/debian/control +++ b/debian/control @@ -4,7 +4,7 @@ Section: python Priority: optional # See https://github.com/astraw/stdeb/issues/175 for dependencies Build-Depends: python3, python3-dev, python3-pip, python3-setuptools, - python3-distutils, python3-wheel, python3-stdeb, dh-python, + python3-wheel, python3-stdeb, dh-python, debhelper (>= 10) Standards-Version: 3.9.1 Homepage: https://github.com/tarantool/tarantool-python diff --git a/debian/rules b/debian/rules index b55a2b66..df1413c3 100755 --- a/debian/rules +++ b/debian/rules @@ -18,3 +18,7 @@ override_dh_auto_install: override_dh_python2: dh_python2 --no-guessing-versions + +# Force `xz` compression for older system with dpkg version < 1.15.6 +override_dh_builddeb: + dh_builddeb -- -Zxz diff --git a/requirements-test.txt b/requirements-test.txt index 4d16b8ca..0f2ad987 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,5 +1,5 @@ git+https://github.com/baztian/dbapi-compliance.git@ea7cb1b4#egg=dbapi-compliance -pyyaml==6.0 +pyyaml >= 6.0.2 importlib-metadata >= 1.0 ; python_version < '3.8' pylint == 3.3.0 ; python_version >= '3.9' pylint == 3.2.7 ; python_version == '3.8' @@ -8,3 +8,4 @@ flake8 == 6.1.0 ; python_version >= '3.8' flake8 == 5.0.4 ; python_version < '3.8' codespell == 2.3.0 ; python_version >= '3.8' codespell == 2.2.5 ; python_version < '3.8' +setuptools >= 75.3.2