diff --git a/.codespellrc b/.codespellrc index fe94c2d8..291bd5cf 100644 --- a/.codespellrc +++ b/.codespellrc @@ -1,3 +1,3 @@ [codespell] skip = tarantool/msgpack_ext/types/timezones/timezones.py -ignore-words-list = ans,gost,ro +ignore-words-list = ans,gost,ro,assertIn diff --git a/.github/workflows/packing.yml b/.github/workflows/packing.yml index eabf5096..70970387 100644 --- a/.github/workflows/packing.yml +++ b/.github/workflows/packing.yml @@ -6,6 +6,10 @@ on: pull_request_target: types: [labeled] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: pack_pip: # We want to run on external PRs, but not on our own internal @@ -16,14 +20,14 @@ jobs: if: (github.event_name == 'push') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false steps: - name: Clone the connector repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Checkout all tags for correct version computation. with: fetch-depth: 0 @@ -34,7 +38,9 @@ jobs: python-version: '3.11' - name: Install tools for packing and verification - run: pip3 install wheel twine + run: | + pip3 install wheel twine + pip3 install --upgrade setuptools - name: Pack source and binary files run: make pip-dist @@ -43,7 +49,7 @@ jobs: run: make pip-dist-check - name: Archive pip artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4.4.0 with: name: pip_dist path: pip_dist @@ -61,14 +67,14 @@ jobs: if: (github.event_name == 'push') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false steps: - name: Clone the connector repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python uses: actions/setup-python@v4 @@ -79,12 +85,12 @@ jobs: run: python3 .github/scripts/remove_source_code.py - name: Install tarantool - uses: tarantool/setup-tarantool@v2 + uses: tarantool/setup-tarantool@v3 with: tarantool-version: '2.11' - name: Download pip package artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4.1.8 with: name: pip_dist path: pip_dist @@ -95,10 +101,13 @@ jobs: - name: Install test requirements run: pip3 install -r requirements-test.txt + # Installation of the specific CMake version is a hotfix for + # https://github.com/tarantool/checks/issues/64 - name: Install the crud module for testing purposes run: | curl -L https://tarantool.io/release/2/installer.sh | bash sudo apt install -y tt + pip3 install cmake==3.15.3 tt rocks install crud - name: Run tests @@ -123,7 +132,7 @@ jobs: steps: - name: Clone the connector repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python uses: actions/setup-python@v4 @@ -134,7 +143,7 @@ jobs: run: python3 .github/scripts/remove_source_code.py - name: Download pip package artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4.1.8 with: name: pip_dist path: pip_dist @@ -148,16 +157,16 @@ jobs: - name: Setup WSL for tarantool uses: Vampire/setup-wsl@v1 with: - distribution: Ubuntu-20.04 + distribution: Ubuntu-22.04 - name: Install tarantool - shell: wsl-bash_Ubuntu-20.04 {0} + shell: wsl-bash_Ubuntu-22.04 {0} run: | curl -L https://tarantool.io/release/2/installer.sh | bash -s sudo apt install -y tarantool tarantool-dev - name: Setup test tarantool instance - shell: wsl-bash_Ubuntu-20.04 {0} + shell: wsl-bash_Ubuntu-22.04 {0} run: | rm -f ./tarantool.pid ./tarantool.log TNT_PID=$(tarantool ./test/suites/lib/tarantool_python_ci.lua > tarantool.log 2>&1 & echo $!) @@ -172,7 +181,7 @@ jobs: - name: Stop test tarantool instance if: ${{ always() }} - shell: wsl-bash_Ubuntu-20.04 {0} + shell: wsl-bash_Ubuntu-22.04 {0} run: | cat tarantool.log || true kill $(cat tarantool.pid) || true @@ -184,14 +193,14 @@ jobs: - run_tests_pip_package_linux - run_tests_pip_package_windows - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false steps: - name: Clone the connector repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python and basic packing tools uses: actions/setup-python@v4 @@ -202,7 +211,7 @@ jobs: run: pip3 install twine - name: Download pip package artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4.1.8 with: name: pip_dist path: pip_dist @@ -222,7 +231,7 @@ jobs: if: (github.event_name == 'push') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 container: image: ${{ matrix.target.os }}:${{ matrix.target.dist }} @@ -246,7 +255,7 @@ jobs: run: dnf install -y git - name: Clone the connector repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Checkout all tags for correct version computation. with: fetch-depth: 0 @@ -271,7 +280,7 @@ jobs: run: make rpm-dist-check - name: Archive RPM artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4.4.0 with: name: rpm_dist_${{ matrix.target.os }}_${{ matrix.target.dist }} path: rpm_dist @@ -289,7 +298,7 @@ jobs: if: (github.event_name == 'push') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 container: image: ${{ matrix.target.os }}:${{ matrix.target.dist }} @@ -308,7 +317,7 @@ jobs: steps: - name: Clone the connector repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python and test running tools # cmake rocks fail to install as expected without findutils: @@ -320,11 +329,11 @@ jobs: - name: Install tarantool run: | - curl -L https://tarantool.io/yeohchA/release/2/installer.sh | bash + curl -L https://tarantool.io/release/2/installer.sh | bash dnf install -y tarantool tarantool-devel - name: Download RPM artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4.1.8 with: name: rpm_dist_${{ matrix.target.os }}_${{ matrix.target.dist }} path: rpm_dist @@ -339,6 +348,7 @@ jobs: run: | curl -L https://tarantool.io/release/2/installer.sh | bash sudo dnf install -y tt + pip3 install cmake==3.15.3 tt rocks install crud - name: Run tests @@ -350,7 +360,7 @@ jobs: needs: - run_tests_rpm - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false @@ -366,13 +376,13 @@ jobs: steps: - name: Clone the connector repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install tools for package publishing run: sudo apt install -y curl make - name: Download RPM artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4.1.8 with: name: rpm_dist_${{ matrix.target.os }}_${{ matrix.target.dist }} path: rpm_dist @@ -400,21 +410,23 @@ jobs: if: (github.event_name == 'push') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false steps: - name: Clone the connector repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Checkout all tags for correct version computation with: fetch-depth: 0 - name: Install deb packing tools run: | + sudo apt update sudo apt install -y devscripts equivs + sudo apt install python3-setuptools python3-stdeb dh-python - name: Make changelog entry for non-release build if: startsWith(github.ref, 'refs/tags') != true @@ -432,7 +444,7 @@ jobs: run: make deb-dist-check - name: Archive deb artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4.4.0 with: name: deb_dist path: deb_dist @@ -448,7 +460,7 @@ jobs: if: (github.event_name == 'push') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 container: image: ${{ matrix.target.os }}:${{ matrix.target.dist }} @@ -458,18 +470,20 @@ jobs: matrix: target: + - os: debian + dist: bullseye # 11 + - os: debian + dist: bookworm # 12 - os: ubuntu dist: focal # 20.04 - os: ubuntu dist: jammy # 22.04 - - os: debian - dist: buster # 10 - - os: debian - dist: bullseye # 11 + - os: ubuntu + dist: noble # 24.04 steps: - name: Clone the connector repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Prepare apt run: apt update @@ -483,13 +497,13 @@ jobs: - name: Install tarantool ${{ matrix.tarantool }} run: | apt install -y curl - curl -L https://tarantool.io/yeohchA/release/2/installer.sh | bash + curl -L https://tarantool.io/release/2/installer.sh | bash apt install -y tarantool tarantool-dev env: DEBIAN_FRONTEND: noninteractive - name: Download deb artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4.1.8 with: name: deb_dist path: deb_dist @@ -499,17 +513,48 @@ jobs: env: DEBIAN_FRONTEND: noninteractive + # Usage of venv is mandatory starting with Debian 12 and Ubuntu 24.04. + - name: Create venv + run: | + apt install -y python3-venv + python3 -m venv .venv + - name: Install test requirements run: pip3 install -r requirements-test.txt + if: matrix.target.dist != 'bookworm' && matrix.target.dist != 'noble' + + - name: Install test requirements + run: | + . .venv/bin/activate + pip3 install -r requirements-test.txt + if: matrix.target.dist == 'bookworm' || matrix.target.dist == 'noble' - name: Install the crud module for testing purposes run: | curl -L https://tarantool.io/release/2/installer.sh | bash apt install -y tt tt rocks install crud + if: matrix.target.dist != 'bookworm' && matrix.target.dist != 'noble' + + - name: Install the crud module for testing purposes + run: | + . .venv/bin/activate + curl -L https://tarantool.io/release/3/installer.sh | bash + apt install -y tt + tt rocks install crud + if: matrix.target.dist == 'bookworm' || matrix.target.dist == 'noble' - name: Run tests run: make test-pure-install + if: matrix.target.dist != 'bookworm' && matrix.target.dist != 'noble' + + - name: Run tests + run: | + . .venv/bin/activate + export PYTHONPATH=$PYTHONPATH:/usr/lib/python3.11:/usr/lib/python3.12:/usr/bin:/usr/lib/python3/dist-packages + export PATH=$PATH:/usr/lib/python3/dist-packages + make test-pure-install + if: matrix.target.dist == 'bookworm' || matrix.target.dist == 'noble' publish_deb: if: startsWith(github.ref, 'refs/tags') @@ -517,7 +562,7 @@ jobs: needs: - run_tests_deb - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false @@ -528,20 +573,24 @@ jobs: dist: focal # 20.04 - os: ubuntu dist: jammy # 22.04 + - os: ubuntu + dist: noble # 24.04 - os: debian dist: buster # 10 - os: debian dist: bullseye # 11 + - os: debian + dist: bookworm # 12 steps: - name: Clone the connector repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install tools for package publishing run: sudo apt install -y curl make - name: Download deb artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4.1.8 with: name: deb_dist path: deb_dist diff --git a/.github/workflows/reusable_testing.yml b/.github/workflows/reusable_testing.yml index e9441532..79f9f2a4 100644 --- a/.github/workflows/reusable_testing.yml +++ b/.github/workflows/reusable_testing.yml @@ -11,15 +11,15 @@ on: jobs: run_tests: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - name: Clone the tarantool-python connector - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: ${{ github.repository_owner }}/tarantool-python - name: Download the tarantool build artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4.1.8 with: name: ${{ inputs.artifact_name }} @@ -29,7 +29,7 @@ jobs: run: sudo dpkg -i tarantool*.deb - name: Setup python3 for tests - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.11' @@ -43,6 +43,7 @@ jobs: run: | curl -L https://tarantool.io/release/2/installer.sh | bash sudo apt install -y tt + pip3 install cmake==3.15.3 tt rocks install crud - run: make test diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 199744f2..8f94a4e6 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -6,6 +6,10 @@ on: pull_request_target: types: [labeled] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: run_tests_ce_linux: # We want to run on external PRs, but not on our own internal @@ -17,20 +21,16 @@ jobs: (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: tarantool: - - '1.10' - - '2.8' - - '2.10' - '2.11' + - '3.3' + - 'master' python: - - '3.6' - - '3.7' - - '3.8' - '3.9' - '3.10' - '3.11' @@ -43,25 +43,64 @@ jobs: # "This page is taking too long to load." error. Thus we use # pairwise testing. include: - - tarantool: '2.11' + - tarantool: '3.3' python: '3.11' msgpack-deps: 'msgpack==0.5.0' - - tarantool: '2.11' + - tarantool: '3.3' python: '3.11' msgpack-deps: 'msgpack==0.6.2' - - tarantool: '2.11' + - tarantool: '3.3' python: '3.11' msgpack-deps: 'msgpack==1.0.4' steps: - name: Clone the connector - uses: actions/checkout@v3 + uses: actions/checkout@v4 + + - name: Setup tt + run: | + curl -L https://tarantool.io/release/3/installer.sh | sudo bash + sudo apt install -y tt + tt version + tt init + + # Installation of the specific CMake version is a hotfix for + # https://github.com/tarantool/checks/issues/64 + - name: Install old CMake + run: pip3 install cmake==3.15.3 - name: Install tarantool ${{ matrix.tarantool }} - uses: tarantool/setup-tarantool@v2 + if: matrix.tarantool != 'master' + uses: tarantool/setup-tarantool@v3 with: tarantool-version: ${{ matrix.tarantool }} + - name: Get Tarantool master latest commit + if: matrix.tarantool == 'master' + run: | + commit_hash=$(git ls-remote https://github.com/tarantool/tarantool.git --branch master | head -c 8) + echo "LATEST_COMMIT=${commit_hash}" >> $GITHUB_ENV + shell: bash + + - name: Cache Tarantool master + if: matrix.tarantool == 'master' + id: cache-latest + uses: actions/cache@v3 + with: + path: | + ${{ github.workspace }}/bin + ${{ github.workspace }}/include + key: cache-latest-${{ env.LATEST_COMMIT }} + + - name: Setup Tarantool master + if: matrix.tarantool == 'master' && steps.cache-latest.outputs.cache-hit != 'true' + run: | + tt install tarantool master + + - name: Add Tarantool master to PATH + if: matrix.tarantool == 'master' + run: echo "${GITHUB_WORKSPACE}/bin" >> $GITHUB_PATH + - name: Setup Python for tests uses: actions/setup-python@v4 with: @@ -85,10 +124,7 @@ jobs: run: pip3 install -r requirements-test.txt - name: Install the crud module for testing purposes - run: | - curl -L https://tarantool.io/release/2/installer.sh | bash - sudo apt install -y tt - tt rocks install crud + run: tt rocks install crud - name: Run tests run: make test @@ -106,34 +142,30 @@ jobs: github.event.pull_request.head.repo.full_name != github.repository && github.event.label.name == 'full-ci') - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: tarantool: - - bundle: 'sdk-1.10.15-0-r563' - path: 'release/linux/x86_64/1.10/' - - bundle: 'sdk-2.8.4-0-r563' - path: 'release/linux/x86_64/2.8/' - - bundle: 'sdk-gc64-2.10.7-0-r563.linux.x86_64' - path: 'release/linux/x86_64/2.10/' - bundle: 'sdk-gc64-2.11.0-0-r563.linux.x86_64' path: 'release/linux/x86_64/2.11/' - python: ['3.6', '3.11'] + - bundle: 'sdk-gc64-3.3.1-0-r55.linux.x86_64' + path: 'release/linux/x86_64/3.3/' + python: ['3.9', '3.11'] steps: - name: Clone the connector # `ref` as merge request is needed for pull_request_target because this # target runs in the context of the base commit of the pull request. - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name == 'pull_request_target' with: ref: refs/pull/${{ github.event.pull_request.number }}/merge - name: Clone the connector if: github.event_name != 'pull_request_target' - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Tarantool EE SDK run: | @@ -158,8 +190,9 @@ jobs: # See more here: https://github.com/tarantool/tt/issues/282 run: | source tarantool-enterprise/env.sh - curl -L https://tarantool.io/release/2/installer.sh | bash + curl -L https://tarantool.io/release/3/installer.sh | bash sudo apt install -y tt + pip3 install cmake==3.15.3 tt rocks install crud TARANTOOL_DIR=$PWD/tarantool-enterprise - name: Run tests @@ -179,7 +212,7 @@ jobs: if: (github.event_name == 'push') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: fail-fast: false @@ -187,15 +220,15 @@ jobs: matrix: tarantool: - '2.11' + - '3.3' python: - - '3.6' - '3.11' steps: - name: Clone the connector repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install tarantool ${{ matrix.tarantool }} - uses: tarantool/setup-tarantool@v2 + uses: tarantool/setup-tarantool@v3 with: tarantool-version: ${{ matrix.tarantool }} @@ -210,13 +243,19 @@ jobs: - name: Install the package with pip run: pip3 install git+$GITHUB_SERVER_URL/$GITHUB_REPOSITORY@$GITHUB_REF + # Installation of the specific CMake version is a hotfix for + # https://github.com/tarantool/checks/issues/64 + - name: Install old CMake + run: pip3 install cmake==3.15.3 + - name: Install test requirements run: pip3 install -r requirements-test.txt - name: Install the crud module for testing purposes run: | - curl -L https://tarantool.io/release/2/installer.sh | bash + curl -L https://tarantool.io/release/3/installer.sh | bash sudo apt install -y tt + pip3 install cmake==3.15.3 tt rocks install crud - name: Run tests @@ -239,14 +278,14 @@ jobs: matrix: # Use reduced test matrix cause Windows pipelines are long. tarantool: + # https://github.com/tarantool/tarantool-python/issues/331 - '2.11.0.g247a9a418-1' python: - - '3.6' - '3.11' steps: - name: Clone the connector - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python for tests uses: actions/setup-python@v4 @@ -262,16 +301,16 @@ jobs: - name: Setup WSL for tarantool uses: Vampire/setup-wsl@v2 with: - distribution: Ubuntu-20.04 + distribution: Ubuntu-22.04 - name: Install tarantool ${{ matrix.tarantool }} for WSL (2.10 and newer) - shell: wsl-bash_Ubuntu-20.04 {0} + shell: wsl-bash_Ubuntu-22.04 {0} run: | curl -L https://tarantool.io/release/2/installer.sh | bash -s sudo apt install -y tarantool=${{ matrix.tarantool }} tarantool-dev=${{ matrix.tarantool }} - name: Setup test tarantool instance - shell: wsl-bash_Ubuntu-20.04 {0} + shell: wsl-bash_Ubuntu-22.04 {0} run: | rm -f ./tarantool.pid ./tarantool.log TNT_PID=$(tarantool ./test/suites/lib/tarantool_python_ci.lua > tarantool.log 2>&1 & echo $!) @@ -286,7 +325,7 @@ jobs: - name: Stop test tarantool instance if: ${{ always() }} - shell: wsl-bash_Ubuntu-20.04 {0} + shell: wsl-bash_Ubuntu-22.04 {0} run: | cat tarantool.log || true kill $(cat tarantool.pid) || true @@ -309,13 +348,13 @@ jobs: matrix: # Use reduced test matrix cause Windows pipelines are long. tarantool: + # https://github.com/tarantool/tarantool-python/issues/331 - '2.11.0.g247a9a418-1' python: - - '3.6' - '3.11' steps: - name: Clone the connector repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python for tests uses: actions/setup-python@v4 @@ -334,16 +373,16 @@ jobs: - name: Setup WSL for tarantool uses: Vampire/setup-wsl@v2 with: - distribution: Ubuntu-20.04 + distribution: Ubuntu-22.04 - name: Install tarantool ${{ matrix.tarantool }} for WSL - shell: wsl-bash_Ubuntu-20.04 {0} + shell: wsl-bash_Ubuntu-22.04 {0} run: | curl -L https://tarantool.io/release/2/installer.sh | bash -s sudo apt install -y tarantool=${{ matrix.tarantool }} tarantool-dev=${{ matrix.tarantool }} - name: Setup test tarantool instance - shell: wsl-bash_Ubuntu-20.04 {0} + shell: wsl-bash_Ubuntu-22.04 {0} run: | rm -f ./tarantool.pid ./tarantool.log TNT_PID=$(tarantool ./test/suites/lib/tarantool_python_ci.lua > tarantool.log 2>&1 & echo $!) @@ -358,7 +397,7 @@ jobs: - name: Stop test tarantool instance if: ${{ always() }} - shell: wsl-bash_Ubuntu-20.04 {0} + shell: wsl-bash_Ubuntu-22.04 {0} run: | cat tarantool.log || true kill $(cat tarantool.pid) || true diff --git a/.pylintrc b/.pylintrc index 3fd8b942..385d67ad 100644 --- a/.pylintrc +++ b/.pylintrc @@ -6,3 +6,8 @@ good-names=i,j,k,ex,Run,_,ok,t,tz [FORMAT] # Allow links in docstings, allow tables ignore-long-lines=^(?:\s*(# )?(?:\.\.\s.+?:)?\s*?)|(\s\+.+\+)|(\s\|.+\|)$ + +[MESSAGES CONTROL] +# Ignore unknown options to support per-entity ignores for newest warnings/errors +# which are not supported for older versions. +disable=unknown-option-value diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 9375a9f6..bc990993 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -13,3 +13,6 @@ python: - method: pip path: . - requirements: docs/requirements.txt + +sphinx: + configuration: docs/source/conf.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ad980b7..8edd47e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,34 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Changed +- Drop Python 3.6 support (PR #327). + +## 1.2.0 - 2024-03-27 + +### Added +- The ability to connect to the Tarantool using an existing socket fd (#304). + +## 1.1.2 - 2023-09-20 + +### Fixed +- Exception rethrow in crud API (PR #310). +- Work with timestamps larger than year 2038 for some platforms (like Windows) (PR #311). + It covers + - building new `tarantool.Datetime` objects from timestamp, + - parsing datetime objects received from Tarantool. + +## 1.1.1 - 2023-07-19 + +### Changed +- Validate `tarantool.Interval` limits with the same rules as in Tarantool (PR #302). + +### Fixed +- `tarantool.Interval` arithmetic with weeks (PR #302). +- `tarantool.Interval` weeks display in `str()` and `repr()` (PR #302). + ## 1.1.0 - 2023-06-30 ### Added @@ -198,7 +226,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Support pandas way to build datetime from timestamp (PR #252). `timestamp_since_utc_epoch` is a parameter to set timestamp - convertion behavior for timezone-aware datetimes. + conversion behavior for timezone-aware datetimes. If ``False`` (default), behaves similar to Tarantool `datetime.new()`: @@ -218,7 +246,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Thus, if ``False``, datetime is computed from timestamp since epoch and then timezone is applied without any - convertion. In that case, `dt.timestamp` won't be equal to + conversion. In that case, `dt.timestamp` won't be equal to initialization `timestamp` for all timezones with non-zero offset. If ``True``, behaves similar to `pandas.Timestamp`: diff --git a/Makefile b/Makefile index 25809cd8..d3919795 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ install: PYTHON_FILES=tarantool test setup.py docs/source/conf.py -TEXT_FILES=README.rst docs/source/*.rst +TEXT_FILES=README.rst CHANGELOG.md docs/source/*.rst .PHONY: lint lint: python3 -m pylint --recursive=y $(PYTHON_FILES) diff --git a/debian/changelog b/debian/changelog index 3d653acf..07127a11 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,50 @@ +python3-tarantool (1.2.0-0) unstable; urgency=medium + + ## Overview + + This release introduces socket fd connect support. + + ## Added + + - The ability to connect to the Tarantool using an existing socket fd (#304). + + -- Georgy Moiseev Wed, 27 Mar 2024 11:00:00 +0300 + +python3-tarantool (1.1.2-0) unstable; urgency=medium + + ## Overview + + This release introduce several bugfixes and behavior improvements. + + ## Fixed + + - Exception rethrow in crud API (PR #310). + - Work with timestamps larger than year 2038 for some platforms (like Windows) (PR #311). + It covers + - building new `tarantool.Datetime` objects from timestamp, + - parsing datetime objects received from Tarantool. + + -- Georgy Moiseev Wed, 20 Sep 2023 10:00:00 +0300 + +python3-tarantool (1.1.1-0) unstable; urgency=medium + + ## Overview + + This release introduces various datetime interval fixes and quality of life + improvements. + + ## Breaking changes + - Forbid to create datetime intervals out of Tarantool limits (PR #302). + + ## Changed + - Validate `tarantool.Interval` limits with the same rules as in Tarantool (PR #302). + + ## Fixed + - `tarantool.Interval` arithmetic with weeks (PR #302). + - `tarantool.Interval` weeks display in `str()` and `repr()` (PR #302). + + -- Georgy Moiseev Wed, 19 Jul 2023 18:00:00 +0300 + python3-tarantool (1.1.0-0) unstable; urgency=medium ## Overview diff --git a/debian/control b/debian/control index dc7869fc..a5cfe113 100644 --- a/debian/control +++ b/debian/control @@ -4,7 +4,7 @@ Section: python Priority: optional # See https://github.com/astraw/stdeb/issues/175 for dependencies Build-Depends: python3, python3-dev, python3-pip, python3-setuptools, - python3-distutils, python3-wheel, python3-stdeb, dh-python, + python3-wheel, python3-stdeb, dh-python, debhelper (>= 10) Standards-Version: 3.9.1 Homepage: https://github.com/tarantool/tarantool-python diff --git a/debian/rules b/debian/rules index b55a2b66..df1413c3 100755 --- a/debian/rules +++ b/debian/rules @@ -18,3 +18,7 @@ override_dh_auto_install: override_dh_python2: dh_python2 --no-guessing-versions + +# Force `xz` compression for older system with dpkg version < 1.15.6 +override_dh_builddeb: + dh_builddeb -- -Zxz diff --git a/requirements-test.txt b/requirements-test.txt index 94d16351..0f2ad987 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,6 +1,11 @@ git+https://github.com/baztian/dbapi-compliance.git@ea7cb1b4#egg=dbapi-compliance -pyyaml==6.0 +pyyaml >= 6.0.2 importlib-metadata >= 1.0 ; python_version < '3.8' -pylint -flake8 -codespell +pylint == 3.3.0 ; python_version >= '3.9' +pylint == 3.2.7 ; python_version == '3.8' +pylint == 2.17.7 ; python_version == '3.7' +flake8 == 6.1.0 ; python_version >= '3.8' +flake8 == 5.0.4 ; python_version < '3.8' +codespell == 2.3.0 ; python_version >= '3.8' +codespell == 2.2.5 ; python_version < '3.8' +setuptools >= 75.3.2 diff --git a/requirements.txt b/requirements.txt index d88dbea3..afcf7b25 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ msgpack pytz -dataclasses; python_version <= '3.6' diff --git a/setup.py b/setup.py index 4ee0797c..3fa65c9c 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ import codecs import os -from setuptools import setup, find_packages +from setuptools import find_packages, setup from setuptools.command.build_py import build_py # Extra commands for documentation management @@ -112,7 +112,7 @@ def get_dependencies(filename): command_options=command_options, install_requires=get_dependencies('requirements.txt'), setup_requires=[ - 'setuptools_scm==6.4.2', + 'setuptools_scm==7.1.0', ], - python_requires='>=3.6', + python_requires='>=3.7', ) diff --git a/tarantool/__init__.py b/tarantool/__init__.py index 91f80e10..97a1e860 100644 --- a/tarantool/__init__.py +++ b/tarantool/__init__.py @@ -1,7 +1,7 @@ """ This package provides API for interaction with a Tarantool server. """ -# pylint: disable=too-many-arguments +# pylint: disable=too-many-arguments,too-many-positional-arguments from tarantool.connection import Connection from tarantool.mesh_connection import MeshConnection @@ -51,7 +51,7 @@ __version__ = '0.0.0-dev' -def connect(host="localhost", port=33013, user=None, password=None, +def connect(host="localhost", port=33013, socket_fd=None, user=None, password=None, encoding=ENCODING_DEFAULT, transport=DEFAULT_TRANSPORT, ssl_key_file=DEFAULT_SSL_KEY_FILE, ssl_cert_file=DEFAULT_SSL_CERT_FILE, @@ -64,6 +64,8 @@ def connect(host="localhost", port=33013, user=None, password=None, :param port: Refer to :paramref:`~tarantool.Connection.params.port`. + :param socket_fd: Refer to :paramref:`~tarantool.Connection.params.socket_fd`. + :param user: Refer to :paramref:`~tarantool.Connection.params.user`. :param password: Refer to @@ -93,6 +95,7 @@ def connect(host="localhost", port=33013, user=None, password=None, """ return Connection(host, port, + socket_fd=socket_fd, user=user, password=password, socket_timeout=SOCKET_TIMEOUT, diff --git a/tarantool/connection.py b/tarantool/connection.py index ea089c08..682b0700 100644 --- a/tarantool/connection.py +++ b/tarantool/connection.py @@ -4,6 +4,7 @@ # pylint: disable=too-many-lines,duplicate-code import os +import select import time import errno from enum import Enum @@ -51,6 +52,9 @@ RECONNECT_DELAY, DEFAULT_TRANSPORT, SSL_TRANSPORT, + DEFAULT_HOST, + DEFAULT_PORT, + DEFAULT_SOCKET_FD, DEFAULT_SSL_KEY_FILE, DEFAULT_SSL_CERT_FILE, DEFAULT_SSL_CA_FILE, @@ -215,6 +219,7 @@ def delete(self, space_name, key, *, index=None, on_push=None, on_push_ctx=None) """ Reference implementation: :meth:`~tarantool.Connection.delete`. """ + # pylint: disable=too-many-arguments raise NotImplementedError @@ -224,6 +229,7 @@ def upsert(self, space_name, tuple_value, op_list, *, index=None, """ Reference implementation: :meth:`~tarantool.Connection.upsert`. """ + # pylint: disable=too-many-arguments raise NotImplementedError @@ -232,6 +238,7 @@ def update(self, space_name, key, op_list, *, index=None, on_push=None, on_push_ """ Reference implementation: :meth:`~tarantool.Connection.update`. """ + # pylint: disable=too-many-arguments raise NotImplementedError @@ -249,6 +256,7 @@ def select(self, space_name, key, *, offset=None, limit=None, """ Reference implementation: :meth:`~tarantool.Connection.select`. """ + # pylint: disable=too-many-arguments raise NotImplementedError @@ -590,7 +598,10 @@ class Connection(ConnectionInterface): :value: :exc:`~tarantool.error.CrudModuleError` """ - def __init__(self, host, port, + def __init__(self, + host=DEFAULT_HOST, + port=DEFAULT_PORT, + socket_fd=DEFAULT_SOCKET_FD, user=None, password=None, socket_timeout=SOCKET_TIMEOUT, @@ -619,8 +630,11 @@ def __init__(self, host, port, Unix sockets. :type host: :obj:`str` or :obj:`None` - :param port: Server port or Unix socket path. - :type port: :obj:`int` or :obj:`str` + :param port: Server port, or Unix socket path. + :type port: :obj:`int` or :obj:`str` or :obj:`None` + + :param socket_fd: socket fd number. + :type socket_fd: :obj:`int` or :obj:`None` :param user: User name for authentication on the Tarantool server. @@ -798,7 +812,19 @@ def __init__(self, host, port, .. _mp_bin: https://github.com/msgpack/msgpack/blob/master/spec.md#bin-format-family .. _mp_array: https://github.com/msgpack/msgpack/blob/master/spec.md#array-format-family """ - # pylint: disable=too-many-arguments,too-many-locals,too-many-statements + # pylint: disable=too-many-arguments,too-many-locals,too-many-statements,too-many-positional-arguments + + if host is None and port is None and socket_fd is None: + raise ConfigurationError("need to specify host/port, " + "port (in case of Unix sockets) " + "or socket_fd") + + if socket_fd is not None and (host is not None or port is not None): + raise ConfigurationError("specifying both socket_fd and host/port is not allowed") + + if host is not None and port is None: + raise ConfigurationError("when specifying host, " + "it is also necessary to specify port") if msgpack.version >= (1, 0, 0) and encoding not in (None, 'utf-8'): raise ConfigurationError("msgpack>=1.0.0 only supports None and " @@ -816,6 +842,7 @@ def __init__(self, host, port, recv.restype = ctypes.c_int self.host = host self.port = port + self.socket_fd = socket_fd self.user = user self.password = password self.socket_timeout = socket_timeout @@ -893,10 +920,37 @@ def connect_basic(self): :meta private: """ - if self.host is None: - self.connect_unix() - else: + if self.socket_fd is not None: + self.connect_socket_fd() + elif self.host is not None: self.connect_tcp() + else: + self.connect_unix() + + def connect_socket_fd(self): + """ + Establish a connection using an existing socket fd. + + +---------------------+--------------------------+-------------------------+ + | socket_fd / timeout | >= 0 | `None` | + +=====================+==========================+=========================+ + | blocking | Set non-blocking socket | Don't change, `select` | + | | lib call `select` | isn't needed | + +---------------------+--------------------------+-------------------------+ + | non-blocking | Don't change, socket lib | Don't change, call | + | | call `select` | `select` ourselves | + +---------------------+--------------------------+-------------------------+ + + :meta private: + """ + + self.connected = True + if self._socket: + self._socket.close() + + self._socket = socket.socket(fileno=self.socket_fd) + if self.socket_timeout is not None: + self._socket.settimeout(self.socket_timeout) def connect_tcp(self): """ @@ -1120,6 +1174,11 @@ def _recv(self, to_read): while to_read > 0: try: tmp = self._socket.recv(to_read) + except BlockingIOError: + ready, _, _ = select.select([self._socket.fileno()], [], [], self.socket_timeout) + if not ready: + raise NetworkError(TimeoutError()) # pylint: disable=raise-missing-from + continue except OverflowError as exc: self._socket.close() err = socket.error( @@ -1159,6 +1218,41 @@ def _read_response(self): # Read the packet return self._recv(length) + def _sendall(self, bytes_to_send): + """ + Sends bytes to the transport (socket). + + :param bytes_to_send: Message to send. + :type bytes_to_send: :obj:`bytes` + + :raise: :exc:`~tarantool.error.NetworkError` + + :meta private: + """ + + total_sent = 0 + while total_sent < len(bytes_to_send): + try: + sent = self._socket.send(bytes_to_send[total_sent:]) + if sent == 0: + err = socket.error( + errno.ECONNRESET, + "Lost connection to server during query" + ) + raise NetworkError(err) + total_sent += sent + except BlockingIOError as exc: + total_sent += exc.characters_written + _, ready, _ = select.select([], [self._socket.fileno()], [], self.socket_timeout) + if not ready: + raise NetworkError(TimeoutError()) # pylint: disable=raise-missing-from + except socket.error as exc: + err = socket.error( + errno.ECONNRESET, + "Lost connection to server during query" + ) + raise NetworkError(err) from exc + def _send_request_wo_reconnect(self, request, on_push=None, on_push_ctx=None): """ Send request without trying to reconnect. @@ -1187,7 +1281,7 @@ def _send_request_wo_reconnect(self, request, on_push=None, on_push_ctx=None): response = None while True: try: - self._socket.sendall(bytes(request)) + self._sendall(bytes(request)) response = request.response_class(self, self._read_response()) break except SchemaReloadException as exc: @@ -1618,6 +1712,7 @@ def delete(self, space_name, key, *, index=0, on_push=None, on_push_ctx=None): .. _delete: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/delete/ """ + # pylint: disable=too-many-arguments self._schemaful_connection_check() @@ -1680,6 +1775,7 @@ def upsert(self, space_name, tuple_value, op_list, *, index=0, on_push=None, on_ .. _upsert: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/upsert/ """ + # pylint: disable=too-many-arguments self._schemaful_connection_check() @@ -1771,6 +1867,7 @@ def update(self, space_name, key, op_list, *, index=0, on_push=None, on_push_ctx .. _update: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/update/ """ + # pylint: disable=too-many-arguments self._schemaful_connection_check() @@ -1961,6 +2058,7 @@ def select(self, space_name, key=None, *, offset=0, limit=0xffffffff, index=0, i .. _select: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/select/ """ + # pylint: disable=too-many-arguments self._schemaful_connection_check() @@ -2158,7 +2256,7 @@ def crud_insert(self, space_name: str, values: Union[tuple, list], crud_resp = call_crud(self, "crud.insert", space_name, values, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) @@ -2192,7 +2290,7 @@ def crud_insert_object(self, space_name: str, values: dict, crud_resp = call_crud(self, "crud.insert_object", space_name, values, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) @@ -2230,7 +2328,7 @@ def crud_insert_many(self, space_name: str, values: Union[tuple, list], if crud_resp[0] is not None: res = CrudResult(crud_resp[0]) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: errs = [] for err in crud_resp[1]: errs.append(CrudError(err)) @@ -2271,7 +2369,7 @@ def crud_insert_object_many(self, space_name: str, values: Union[tuple, list], if crud_resp[0] is not None: res = CrudResult(crud_resp[0]) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: errs = [] for err in crud_resp[1]: errs.append(CrudError(err)) @@ -2306,7 +2404,7 @@ def crud_get(self, space_name: str, key: int, opts: Optional[dict] = None) -> Cr crud_resp = call_crud(self, "crud.get", space_name, key, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) @@ -2345,7 +2443,7 @@ def crud_update(self, space_name: str, key: int, operations: Optional[list] = No crud_resp = call_crud(self, "crud.update", space_name, key, operations, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) @@ -2377,7 +2475,7 @@ def crud_delete(self, space_name: str, key: int, opts: Optional[dict] = None) -> crud_resp = call_crud(self, "crud.delete", space_name, key, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) @@ -2411,7 +2509,7 @@ def crud_replace(self, space_name: str, values: Union[tuple, list], crud_resp = call_crud(self, "crud.replace", space_name, values, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) @@ -2445,7 +2543,7 @@ def crud_replace_object(self, space_name: str, values: dict, crud_resp = call_crud(self, "crud.replace_object", space_name, values, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) @@ -2483,7 +2581,7 @@ def crud_replace_many(self, space_name: str, values: Union[tuple, list], if crud_resp[0] is not None: res = CrudResult(crud_resp[0]) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: errs = [] for err in crud_resp[1]: errs.append(CrudError(err)) @@ -2524,7 +2622,7 @@ def crud_replace_object_many(self, space_name: str, values: Union[tuple, list], if crud_resp[0] is not None: res = CrudResult(crud_resp[0]) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: errs = [] for err in crud_resp[1]: errs.append(CrudError(err)) @@ -2567,7 +2665,7 @@ def crud_upsert(self, space_name: str, values: Union[tuple, list], crud_resp = call_crud(self, "crud.upsert", space_name, values, operations, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) @@ -2608,7 +2706,7 @@ def crud_upsert_object(self, space_name: str, values: dict, crud_resp = call_crud(self, "crud.upsert_object", space_name, values, operations, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) @@ -2646,7 +2744,7 @@ def crud_upsert_many(self, space_name: str, values_operation: Union[tuple, list] if crud_resp[0] is not None: res = CrudResult(crud_resp[0]) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: errs = [] for err in crud_resp[1]: errs.append(CrudError(err)) @@ -2687,7 +2785,7 @@ def crud_upsert_object_many(self, space_name: str, values_operation: Union[tuple if crud_resp[0] is not None: res = CrudResult(crud_resp[0]) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: errs = [] for err in crud_resp[1]: errs.append(CrudError(err)) @@ -2726,7 +2824,7 @@ def crud_select(self, space_name: str, conditions: Optional[list] = None, crud_resp = call_crud(self, "crud.select", space_name, conditions, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) @@ -2758,7 +2856,7 @@ def crud_min(self, space_name: str, index_name: str, opts: Optional[dict] = None crud_resp = call_crud(self, "crud.min", space_name, index_name, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) @@ -2790,7 +2888,7 @@ def crud_max(self, space_name: str, index_name: str, opts: Optional[dict] = None crud_resp = call_crud(self, "crud.max", space_name, index_name, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) @@ -2819,9 +2917,7 @@ def crud_truncate(self, space_name: str, opts: Optional[dict] = None) -> bool: crud_resp = call_crud(self, "crud.truncate", space_name, opts) - # In absence of an error, crud does not give - # variable err as nil (as in most cases). - if len(crud_resp) != 1: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return crud_resp[0] @@ -2850,9 +2946,7 @@ def crud_len(self, space_name: str, opts: Optional[dict] = None) -> int: crud_resp = call_crud(self, "crud.len", space_name, opts) - # In absence of an error, crud does not give - # variable err as nil (as in most cases). - if len(crud_resp) != 1: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return crud_resp[0] @@ -2877,9 +2971,7 @@ def crud_storage_info(self, opts: Optional[dict] = None) -> dict: crud_resp = call_crud(self, "crud.storage_info", opts) - # In absence of an error, crud does not give - # variable err as nil (as in most cases). - if len(crud_resp) != 1: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return crud_resp[0] @@ -2915,7 +3007,7 @@ def crud_count(self, space_name: str, conditions: Optional[list] = None, crud_resp = call_crud(self, "crud.count", space_name, conditions, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return crud_resp[0] @@ -2938,6 +3030,7 @@ def crud_stats(self, space_name: str = None) -> CrudResult: crud_resp = call_crud(self, "crud.stats", space_name) + # There are no errors in `crud.stats`. res = None if len(crud_resp.data[0]) > 0: res = CrudResult(crud_resp.data[0]) diff --git a/tarantool/connection_pool.py b/tarantool/connection_pool.py index 1008991f..f1e675b3 100644 --- a/tarantool/connection_pool.py +++ b/tarantool/connection_pool.py @@ -115,7 +115,7 @@ class PoolUnit(): addr: dict """ - ``{"host": host, "port": port}`` info. + ``{"host": host, "port": port, "socket_fd": socket_fd}`` info. :type: :obj:`dict` """ @@ -161,6 +161,14 @@ class PoolUnit(): :type: :obj:`bool` """ + def get_address(self): + """ + Get an address string representation. + """ + if self.addr['socket_fd'] is not None: + return f'fd://{self.addr["socket_fd"]}' + return f'{self.addr["host"]}:{self.addr["port"]}' + # Based on https://realpython.com/python-interface/ class StrategyInterface(metaclass=abc.ABCMeta): @@ -398,6 +406,7 @@ def __init__(self, { "host': "str" or None, # mandatory "port": int or "str", # mandatory + "socket_fd": int, # optional "transport": "str", # optional "ssl_key_file": "str", # optional "ssl_cert_file": "str", # optional @@ -473,7 +482,7 @@ def __init__(self, .. _box.info.status: .. _box.info: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_info/ """ - # pylint: disable=too-many-arguments,too-many-locals + # pylint: disable=too-many-arguments,too-many-locals,too-many-positional-arguments if not isinstance(addrs, list) or len(addrs) == 0: raise ConfigurationError("addrs must be non-empty list") @@ -499,6 +508,7 @@ def __init__(self, conn=Connection( host=addr['host'], port=addr['port'], + socket_fd=addr['socket_fd'], user=user, password=password, socket_timeout=socket_timeout, @@ -529,15 +539,16 @@ def _make_key(self, addr): """ Make a unique key for a server based on its address. - :param addr: `{"host": host, "port": port}` dictionary. + :param addr: `{"host": host, "port": port, "socket_fd": socket_fd}` dictionary. :type addr: :obj:`dict` :rtype: :obj:`str` :meta private: """ - - return f"{addr['host']}:{addr['port']}" + if addr['socket_fd'] is None: + return f"{addr['host']}:{addr['port']}" + return addr['socket_fd'] def _get_new_state(self, unit): """ @@ -557,7 +568,7 @@ def _get_new_state(self, unit): try: conn.connect() except NetworkError as exc: - msg = (f"Failed to connect to {unit.addr['host']}:{unit.addr['port']}, " + msg = (f"Failed to connect to {unit.get_address()}, " f"reason: {repr(exc)}") warn(msg, ClusterConnectWarning) return InstanceState(Status.UNHEALTHY) @@ -565,7 +576,7 @@ def _get_new_state(self, unit): try: resp = conn.call('box.info') except NetworkError as exc: - msg = (f"Failed to get box.info for {unit.addr['host']}:{unit.addr['port']}, " + msg = (f"Failed to get box.info for {unit.get_address()}, " f"reason: {repr(exc)}") warn(msg, PoolTolopogyWarning) return InstanceState(Status.UNHEALTHY) @@ -573,7 +584,7 @@ def _get_new_state(self, unit): try: read_only = resp.data[0]['ro'] except (IndexError, KeyError) as exc: - msg = (f"Incorrect box.info response from {unit.addr['host']}:{unit.addr['port']}" + msg = (f"Incorrect box.info response from {unit.get_address()}" f"reason: {repr(exc)}") warn(msg, PoolTolopogyWarning) return InstanceState(Status.UNHEALTHY) @@ -582,11 +593,11 @@ def _get_new_state(self, unit): status = resp.data[0]['status'] if status != 'running': - msg = f"{unit.addr['host']}:{unit.addr['port']} instance status is not 'running'" + msg = f"{unit.get_address()} instance status is not 'running'" warn(msg, PoolTolopogyWarning) return InstanceState(Status.UNHEALTHY) except (IndexError, KeyError) as exc: - msg = (f"Incorrect box.info response from {unit.addr['host']}:{unit.addr['port']}" + msg = (f"Incorrect box.info response from {unit.get_address()}" f"reason: {repr(exc)}") warn(msg, PoolTolopogyWarning) return InstanceState(Status.UNHEALTHY) @@ -820,6 +831,7 @@ def replace(self, space_name, values, *, mode=Mode.RW, on_push=None, on_push_ctx .. _replace: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/replace/ """ + # pylint: disable=too-many-arguments return self._send(mode, 'replace', space_name, values, on_push=on_push, on_push_ctx=on_push_ctx) @@ -850,6 +862,7 @@ def insert(self, space_name, values, *, mode=Mode.RW, on_push=None, on_push_ctx= .. _insert: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/insert/ """ + # pylint: disable=too-many-arguments return self._send(mode, 'insert', space_name, values, on_push=on_push, on_push_ctx=on_push_ctx) @@ -883,6 +896,7 @@ def delete(self, space_name, key, *, index=0, mode=Mode.RW, on_push=None, on_pus .. _delete: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/delete/ """ + # pylint: disable=too-many-arguments return self._send(mode, 'delete', space_name, key, index=index, on_push=on_push, on_push_ctx=on_push_ctx) @@ -920,6 +934,7 @@ def upsert(self, space_name, tuple_value, op_list, *, index=0, mode=Mode.RW, .. _upsert: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/upsert/ """ + # pylint: disable=too-many-arguments return self._send(mode, 'upsert', space_name, tuple_value, op_list, index=index, on_push=on_push, on_push_ctx=on_push_ctx) @@ -957,6 +972,7 @@ def update(self, space_name, key, op_list, *, index=0, mode=Mode.RW, .. _update: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/update/ """ + # pylint: disable=too-many-arguments return self._send(mode, 'update', space_name, key, op_list, index=index, on_push=on_push, on_push_ctx=on_push_ctx) @@ -1023,6 +1039,7 @@ def select(self, space_name, key, *, offset=0, limit=0xffffffff, .. _select: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/select/ """ + # pylint: disable=too-many-arguments return self._send(mode, 'select', space_name, key, offset=offset, limit=limit, index=index, iterator=iterator, on_push=on_push, on_push_ctx=on_push_ctx) @@ -1214,6 +1231,7 @@ def crud_update(self, space_name, key, operations=None, opts=None, *, mode=Mode. :raise: :exc:`~tarantool.error.CrudModuleError`, :exc:`~tarantool.error.DatabaseError` """ + # pylint: disable=too-many-arguments return self._send(mode, 'crud_update', space_name, key, operations, opts) @@ -1379,6 +1397,7 @@ def crud_upsert(self, space_name, values, operations=None, opts=None, *, mode=Mo :raise: :exc:`~tarantool.error.CrudModuleError`, :exc:`~tarantool.error.DatabaseError` """ + # pylint: disable=too-many-arguments return self._send(mode, 'crud_upsert', space_name, values, operations, opts) @@ -1409,6 +1428,7 @@ def crud_upsert_object(self, space_name, values, operations=None, opts=None, *, :raise: :exc:`~tarantool.error.CrudModuleError`, :exc:`~tarantool.error.DatabaseError` """ + # pylint: disable=too-many-arguments return self._send(mode, 'crud_upsert_object', space_name, values, operations, opts) diff --git a/tarantool/const.py b/tarantool/const.py index 1e2b0895..53749cec 100644 --- a/tarantool/const.py +++ b/tarantool/const.py @@ -103,6 +103,12 @@ IPROTO_FEATURE_SPACE_AND_INDEX_NAMES = 5 IPROTO_FEATURE_WATCH_ONCE = 6 +# Default value for host. +DEFAULT_HOST = None +# Default value for port. +DEFAULT_PORT = None +# Default value for socket_fd. +DEFAULT_SOCKET_FD = None # Default value for connection timeout (seconds) CONNECTION_TIMEOUT = None # Default value for socket timeout (seconds) diff --git a/tarantool/crud.py b/tarantool/crud.py index d10726c5..3a642564 100644 --- a/tarantool/crud.py +++ b/tarantool/crud.py @@ -69,5 +69,6 @@ def call_crud(conn, *args): if exc.code in (ER_NO_SUCH_PROC, ER_ACCESS_DENIED): exc_msg = ". Ensure that you're calling crud.router and user has sufficient grants" raise DatabaseError(exc.code, exc.message + exc_msg, extra_info=exc.extra_info) from exc + raise exc return crud_resp diff --git a/tarantool/mesh_connection.py b/tarantool/mesh_connection.py index ebbc1a40..02a20d7d 100644 --- a/tarantool/mesh_connection.py +++ b/tarantool/mesh_connection.py @@ -28,6 +28,9 @@ DEFAULT_SSL_PASSWORD, DEFAULT_SSL_PASSWORD_FILE, CLUSTER_DISCOVERY_DELAY, + DEFAULT_HOST, + DEFAULT_SOCKET_FD, + DEFAULT_PORT, ) from tarantool.request import ( @@ -35,6 +38,9 @@ ) default_addr_opts = { + 'host': DEFAULT_HOST, + 'port': DEFAULT_PORT, + 'socket_fd': DEFAULT_SOCKET_FD, 'transport': DEFAULT_TRANSPORT, 'ssl_key_file': DEFAULT_SSL_KEY_FILE, 'ssl_cert_file': DEFAULT_SSL_CERT_FILE, @@ -91,7 +97,8 @@ def parse_error(uri, msg): return parse_error(uri, 'port should be a number') for key, val in default_addr_opts.items(): - result[key] = val + if key not in result: + result[key] = val if opts_str != "": for opt_str in opts_str.split('&'): @@ -127,9 +134,6 @@ def format_error(address, err): if not isinstance(address, dict): return format_error(address, 'address must be a dict') - if 'port' not in address or address['port'] is None: - return format_error(address, 'port is not set or None') - result = {} for key, val in address.items(): result[key] = val @@ -138,6 +142,17 @@ def format_error(address, err): if key not in result: result[key] = val + if result['socket_fd'] is not None: + # Looks like socket fd. + if result['host'] is not None or result['port'] is not None: + return format_error(result, + "specifying both socket_fd and host/port is not allowed") + + if not isinstance(result['socket_fd'], int): + return format_error(result, + 'socket_fd must be an int') + return result, None + if isinstance(result['port'], int): # Looks like an inet address. @@ -192,6 +207,7 @@ def update_connection(conn, address): conn.host = address["host"] conn.port = address["port"] + conn.socket_fd = address["socket_fd"] conn.transport = address['transport'] conn.ssl_key_file = address['ssl_key_file'] conn.ssl_cert_file = address['ssl_cert_file'] @@ -268,7 +284,10 @@ class MeshConnection(Connection): Represents a connection to a cluster of Tarantool servers. """ - def __init__(self, host=None, port=None, + def __init__(self, + host=DEFAULT_HOST, + port=DEFAULT_PORT, + socket_fd=DEFAULT_SOCKET_FD, user=None, password=None, socket_timeout=SOCKET_TIMEOUT, @@ -298,7 +317,12 @@ def __init__(self, host=None, port=None, :paramref:`~tarantool.MeshConnection.params.addrs` list. :param port: Refer to - :paramref:`~tarantool.Connection.params.host`. + :paramref:`~tarantool.Connection.params.port`. + Value would be used to add one more server in + :paramref:`~tarantool.MeshConnection.params.addrs` list. + + :param socket_fd: Refer to + :paramref:`~tarantol.Connection.params.socket_fd`. Value would be used to add one more server in :paramref:`~tarantool.MeshConnection.params.addrs` list. @@ -439,7 +463,7 @@ def __init__(self, host=None, port=None, :class:`~tarantool.Connection` exceptions, :class:`~tarantool.MeshConnection.connect` exceptions """ - # pylint: disable=too-many-arguments,too-many-locals + # pylint: disable=too-many-arguments,too-many-locals,too-many-positional-arguments if addrs is None: addrs = [] @@ -447,9 +471,10 @@ def __init__(self, host=None, port=None, # Don't change user provided arguments. addrs = addrs[:] - if host and port: + if (host and port) or socket_fd: addrs.insert(0, {'host': host, 'port': port, + 'socket_fd': socket_fd, 'transport': transport, 'ssl_key_file': ssl_key_file, 'ssl_cert_file': ssl_cert_file, @@ -484,6 +509,7 @@ def __init__(self, host=None, port=None, super().__init__( host=addr['host'], port=addr['port'], + socket_fd=addr['socket_fd'], user=user, password=password, socket_timeout=socket_timeout, @@ -604,6 +630,7 @@ def _opt_refresh_instances(self): # an instance list and connect to one of new instances. current_addr = {'host': self.host, 'port': self.port, + 'socket_fd': self.socket_fd, 'transport': self.transport, 'ssl_key_file': self.ssl_key_file, 'ssl_cert_file': self.ssl_cert_file, diff --git a/tarantool/msgpack_ext/types/datetime.py b/tarantool/msgpack_ext/types/datetime.py index 1a546cfc..d7068f57 100644 --- a/tarantool/msgpack_ext/types/datetime.py +++ b/tarantool/msgpack_ext/types/datetime.py @@ -6,7 +6,6 @@ from calendar import monthrange from copy import deepcopy from datetime import datetime, timedelta -import sys import pytz @@ -282,7 +281,7 @@ def __init__(self, *, timestamp=None, year=None, month=None, .. _datetime.new(): https://www.tarantool.io/en/doc/latest/reference/reference_lua/datetime/new/ """ - # pylint: disable=too-many-branches,too-many-locals,too-many-statements + # pylint: disable=too-many-branches,too-many-locals,too-many-statements,too-many-arguments tzinfo = None if tz != '': @@ -314,13 +313,12 @@ def __init__(self, *, timestamp=None, year=None, month=None, timestamp += nsec // NSEC_IN_SEC nsec = nsec % NSEC_IN_SEC - if (sys.platform.startswith("win")) and (timestamp < 0): - # Fails to create a datetime from negative timestamp on Windows. - _datetime = _EPOCH + timedelta(seconds=timestamp) - else: - # Timezone-naive datetime objects are treated by many datetime methods - # as local times, so we represent time in UTC explicitly if not provided. - _datetime = datetime.fromtimestamp(timestamp, pytz.UTC) + # datetime.fromtimestamp may raise OverflowError, if the timestamp + # is out of the range of values supported by the platform C localtime() + # function, and OSError on localtime() failure. It’s common for this + # to be restricted to years from 1970 through 2038, yet we want + # to support a wider range. + _datetime = _EPOCH + timedelta(seconds=timestamp) if nsec is not None: _datetime = _datetime.replace(microsecond=nsec // NSEC_IN_MKSEC) diff --git a/tarantool/msgpack_ext/types/interval.py b/tarantool/msgpack_ext/types/interval.py index 10dc4847..734a9088 100644 --- a/tarantool/msgpack_ext/types/interval.py +++ b/tarantool/msgpack_ext/types/interval.py @@ -16,6 +16,60 @@ 8: 'adjust', } +# https://github.com/tarantool/tarantool/blob/ff57f990f359f6d7866c1947174d8ba0e97b1ea6/src/lua/datetime.lua#L112-L146 +SECS_PER_DAY = 86400 + +MIN_DATE_YEAR = -5879610 +MIN_DATE_MONTH = 6 +MIN_DATE_DAY = 22 +MAX_DATE_YEAR = 5879611 +MAX_DATE_MONTH = 7 +MAX_DATE_DAY = 11 + +AVERAGE_DAYS_YEAR = 365.25 +AVERAGE_WEEK_YEAR = AVERAGE_DAYS_YEAR / 7 +INT_MAX = 2147483647 +MAX_YEAR_RANGE = MAX_DATE_YEAR - MIN_DATE_YEAR +MAX_MONTH_RANGE = MAX_YEAR_RANGE * 12 +MAX_WEEK_RANGE = MAX_YEAR_RANGE * AVERAGE_WEEK_YEAR +MAX_DAY_RANGE = MAX_YEAR_RANGE * AVERAGE_DAYS_YEAR +MAX_HOUR_RANGE = MAX_DAY_RANGE * 24 +MAX_MIN_RANGE = MAX_HOUR_RANGE * 60 +MAX_SEC_RANGE = MAX_DAY_RANGE * SECS_PER_DAY +MAX_NSEC_RANGE = INT_MAX + +max_val = { + 'year': MAX_YEAR_RANGE, + 'month': MAX_MONTH_RANGE, + 'week': MAX_WEEK_RANGE, + 'day': MAX_DAY_RANGE, + 'hour': MAX_HOUR_RANGE, + 'minute': MAX_MIN_RANGE, + 'sec': MAX_SEC_RANGE, + 'nsec': MAX_NSEC_RANGE, +} + + +def verify_range(intv): + """ + Check allowed values. Approach is the same as in tarantool/tarantool. + + :param intv: Raw interval to verify. + :type intv: :class:`~tarantool.Interval` + + :raise: :exc:`ValueError` + + :meta private: + """ + + for field_name, range_max in max_val.items(): + val = getattr(intv, field_name) + # Tarantool implementation has a bug + # https://github.com/tarantool/tarantool/issues/8878 + if (val > range_max) or (val < -range_max): + raise ValueError(f"value {val} of {field_name} is out of " + f"allowed range [{-range_max}, {range_max}]") + # https://github.com/tarantool/c-dt/blob/cec6acebb54d9e73ea0b99c63898732abd7683a6/dt_arithmetic.h#L34 class Adjust(Enum): @@ -49,8 +103,8 @@ class Interval(): .. code-block:: python - di = tarantool.Interval(year=-1, month=2, day=3, - hour=4, minute=-5, sec=6, + di = tarantool.Interval(year=-1, month=2, week=-3, + day=4, hour=5, minute=-6, sec=7, nsec=308543321, adjust=tarantool.IntervalAdjust.NONE) @@ -92,7 +146,10 @@ def __init__(self, *, year=0, month=0, week=0, :param adjust: Interval adjustment rule. Refer to :meth:`~tarantool.Datetime.__add__`. :type adjust: :class:`~tarantool.IntervalAdjust`, optional + + :raise: :exc:`ValueError` """ + # pylint: disable=too-many-arguments self.year = year self.month = month @@ -104,6 +161,8 @@ def __init__(self, *, year=0, month=0, week=0, self.nsec = nsec self.adjust = adjust + verify_range(self) + def __add__(self, other): """ Valid operations: @@ -145,6 +204,7 @@ def __add__(self, other): return Interval( year=self.year + other.year, month=self.month + other.month, + week=self.week + other.week, day=self.day + other.day, hour=self.hour + other.hour, minute=self.minute + other.minute, @@ -194,6 +254,7 @@ def __sub__(self, other): return Interval( year=self.year - other.year, month=self.month - other.month, + week=self.week - other.week, day=self.day - other.day, hour=self.hour - other.hour, minute=self.minute - other.minute, @@ -231,8 +292,8 @@ def __eq__(self, other): return True def __repr__(self): - return f'tarantool.Interval(year={self.year}, month={self.month}, day={self.day}, ' + \ - f'hour={self.hour}, minute={self.minute}, sec={self.sec}, ' + \ + return f'tarantool.Interval(year={self.year}, month={self.month}, week={self.week}, ' + \ + f'day={self.day}, hour={self.hour}, minute={self.minute}, sec={self.sec}, ' + \ f'nsec={self.nsec}, adjust={self.adjust})' __str__ = __repr__ diff --git a/tarantool/request.py b/tarantool/request.py index d8d4dd22..95164b79 100644 --- a/tarantool/request.py +++ b/tarantool/request.py @@ -269,7 +269,7 @@ def __init__(self, conn, salt, user, password, auth_type=AUTH_TYPE_CHAP_SHA1): :param auth_type: Refer to :paramref:`~tarantool.Connection.auth_type`. :type auth_type: :obj:`str`, optional """ - # pylint: disable=too-many-arguments + # pylint: disable=too-many-arguments,too-many-positional-arguments super().__init__(conn) @@ -405,7 +405,7 @@ def __init__(self, conn, space_no, index_no, key, offset, limit, iterator): :raise: :exc:`~AssertionError` """ - # pylint: disable=too-many-arguments + # pylint: disable=too-many-arguments,too-many-positional-arguments super().__init__(conn) request_body = self._dumps({IPROTO_SPACE_ID: space_no, @@ -446,7 +446,7 @@ def __init__(self, conn, space_no, index_no, key, op_list): :raise: :exc:`~AssertionError` """ - # pylint: disable=too-many-arguments + # pylint: disable=too-many-arguments,too-many-positional-arguments super().__init__(conn) @@ -569,7 +569,7 @@ def __init__(self, conn, space_no, index_no, tuple_value, op_list): :raise: :exc:`~AssertionError` """ - # pylint: disable=too-many-arguments + # pylint: disable=too-many-arguments,too-many-positional-arguments super().__init__(conn) diff --git a/tarantool/utils.py b/tarantool/utils.py index c7d9e246..b7ab436d 100644 --- a/tarantool/utils.py +++ b/tarantool/utils.py @@ -1,5 +1,5 @@ """ -This module provides untility functions for the package. +This module provides utility functions for the package. """ from base64 import decodebytes as base64_decode diff --git a/test/suites/__init__.py b/test/suites/__init__.py index d56b2889..7d092585 100644 --- a/test/suites/__init__.py +++ b/test/suites/__init__.py @@ -15,6 +15,7 @@ from .test_execute import TestSuiteExecute from .test_dbapi import TestSuiteDBAPI from .test_encoding import TestSuiteEncoding +from .test_socket_fd import TestSuiteSocketFD from .test_ssl import TestSuiteSsl from .test_decimal import TestSuiteDecimal from .test_uuid import TestSuiteUUID @@ -33,7 +34,7 @@ TestSuiteEncoding, TestSuitePool, TestSuiteSsl, TestSuiteDecimal, TestSuiteUUID, TestSuiteDatetime, TestSuiteInterval, TestSuitePackage, TestSuiteErrorExt, - TestSuitePush, TestSuiteConnection, TestSuiteCrud,) + TestSuitePush, TestSuiteConnection, TestSuiteCrud, TestSuiteSocketFD) def load_tests(loader, tests, pattern): diff --git a/test/suites/box.lua b/test/suites/box.lua index eb4330dd..c8110d18 100644 --- a/test/suites/box.lua +++ b/test/suites/box.lua @@ -19,3 +19,5 @@ box.cfg{ pid_file = "box.pid", auth_type = (auth_type:len() > 0) and auth_type or nil, } + +rawset(_G, 'ready', true) diff --git a/test/suites/crud_mock_server.lua b/test/suites/crud_mock_server.lua new file mode 100644 index 00000000..252059df --- /dev/null +++ b/test/suites/crud_mock_server.lua @@ -0,0 +1,21 @@ +#!/usr/bin/env tarantool + +local admin_listen = os.getenv("ADMIN") +local primary_listen = os.getenv("LISTEN") + +require('console').listen(admin_listen) +box.cfg{ + listen = primary_listen, + memtx_memory = 0.1 * 1024^3, -- 0.1 GiB + pid_file = "box.pid", +} + +box.schema.user.grant('guest', 'execute', 'universe', nil, {if_not_exists = true}) + +local function mock_replace() + error('Unexpected connection error') +end + +rawset(_G, 'crud', {replace = mock_replace}) + +rawset(_G, 'ready', true) diff --git a/test/suites/crud_server.lua b/test/suites/crud_server.lua index 8bb58fcb..f939818a 100644 --- a/test/suites/crud_server.lua +++ b/test/suites/crud_server.lua @@ -91,3 +91,5 @@ if crud_imported == false or vshard_imported == false then else configure_crud_instance(primary_listen, crud, vshard) end + +rawset(_G, 'ready', true) diff --git a/test/suites/lib/skip.py b/test/suites/lib/skip.py index 00b1a21d..625caf6a 100644 --- a/test/suites/lib/skip.py +++ b/test/suites/lib/skip.py @@ -306,3 +306,14 @@ def skip_or_run_iproto_basic_features_test(func): return skip_or_run_test_tarantool(func, '2.10.0', 'does not support iproto ID and iproto basic features') + + +def skip_or_run_box_session_new_tests(func): + """ + Decorator to skip or run tests that use box.session.new. + + Tarantool supports box.session.new only in current master since + commit 324872a. + See https://github.com/tarantool/tarantool/issues/8801. + """ + return skip_or_run_test_tarantool(func, '3.0.0', 'does not support box.session.new') diff --git a/test/suites/lib/tarantool_server.py b/test/suites/lib/tarantool_server.py index 1dd5e053..55f6ca63 100644 --- a/test/suites/lib/tarantool_server.py +++ b/test/suites/lib/tarantool_server.py @@ -56,7 +56,7 @@ class TarantoolServer(): """ Class to start up a new Tarantool server. """ - # pylint: disable=too-many-instance-attributes,too-many-arguments,duplicate-code + # pylint: disable=too-many-instance-attributes,too-many-arguments,duplicate-code,too-many-positional-arguments default_tarantool = { "bin": "tarantool", @@ -302,27 +302,22 @@ def prepare_args(self): return shlex.split(self.binary if not self.script else self.script_dst) - def wait_until_started(self): + def wait_until_ready(self): """ - Wait until server is started. + Wait until server is configured and ready to work. Server consists of two parts: 1) wait until server is listening on sockets - 2) wait until server tells us his status + 2) wait until server finishes executing its script """ while True: try: temp = TarantoolAdmin('0.0.0.0', self.args['admin']) - while True: - ans = temp('box.info.status')[0] - if ans in ('running', 'hot_standby', 'orphan') or ans.startswith('replica'): - temp.disconnect() - return True - if ans in ('loading',): - continue - - raise ValueError(f"Strange output for `box.info.status`: {ans}") + ans = temp('ready')[0] + temp.disconnect() + if isinstance(ans, bool) and ans: + return True except socket.error as exc: if exc.errno == errno.ECONNREFUSED: time.sleep(0.1) @@ -352,7 +347,7 @@ def start(self): cwd=self.vardir, stdout=self.log_des, stderr=self.log_des) - self.wait_until_started() + self.wait_until_ready() def stop(self): """ diff --git a/test/suites/sidecar.py b/test/suites/sidecar.py new file mode 100644 index 00000000..6bee5af7 --- /dev/null +++ b/test/suites/sidecar.py @@ -0,0 +1,16 @@ +# pylint: disable=missing-module-docstring +import os + +import tarantool + +socket_fd = int(os.environ["SOCKET_FD"]) + +conn = tarantool.connect(None, None, socket_fd=socket_fd) + +# Check user. +assert conn.eval("return box.session.user()").data[0] == "test" + +# Check db operations. +conn.insert("test", [1]) +conn.insert("test", [2]) +assert conn.select("test").data == [[1], [2]] diff --git a/test/suites/test_connection.py b/test/suites/test_connection.py index 52234608..4402f0b0 100644 --- a/test/suites/test_connection.py +++ b/test/suites/test_connection.py @@ -5,8 +5,9 @@ import sys import unittest - import decimal + +import pkg_resources import msgpack import tarantool @@ -14,6 +15,7 @@ from .lib.skip import skip_or_run_decimal_test, skip_or_run_varbinary_test from .lib.tarantool_server import TarantoolServer +from .utils import assert_admin_success class TestSuiteConnection(unittest.TestCase): @@ -26,35 +28,48 @@ def setUpClass(cls): cls.srv.start() cls.adm = cls.srv.admin - cls.adm(r""" + resp = cls.adm(""" box.schema.user.create('test', {password = 'test', if_not_exists = true}) - box.schema.user.grant('test', 'read,write,execute', 'universe') - - box.schema.create_space('space_varbin') - - box.space['space_varbin']:format({ - { - 'id', - type = 'number', - is_nullable = false - }, - { - 'varbin', - type = 'varbinary', - is_nullable = false, - } - }) - - box.space['space_varbin']:create_index('id', { - type = 'tree', - parts = {1, 'number'}, - unique = true}) - - box.space['space_varbin']:create_index('varbin', { - type = 'tree', - parts = {2, 'varbinary'}, - unique = true}) + box.schema.user.grant('test', 'read,write,execute', 'universe', + nil, {if_not_exists = true}) + + return true """) + assert_admin_success(resp) + + if cls.srv.admin.tnt_version >= pkg_resources.parse_version('2.2.1'): + resp = cls.adm(""" + box.schema.create_space('space_varbin', {if_not_exists = true}) + + box.space['space_varbin']:format({ + { + 'id', + type = 'number', + is_nullable = false + }, + { + 'varbin', + type = 'varbinary', + is_nullable = false, + } + }) + + box.space['space_varbin']:create_index('id', { + type = 'tree', + parts = {1, 'number'}, + unique = true, + if_not_exists = true}) + + box.space['space_varbin']:create_index('varbin', { + type = 'tree', + parts = {2, 'varbinary'}, + unique = true, + if_not_exists = true}) + + return true + """) + assert_admin_success(resp) + cls.con = None def setUp(self): diff --git a/test/suites/test_crud.py b/test/suites/test_crud.py index d1d7e186..2bbc5513 100644 --- a/test/suites/test_crud.py +++ b/test/suites/test_crud.py @@ -22,6 +22,14 @@ def create_server(): return srv +def create_mock_server(): + srv = TarantoolServer() + srv.script = 'test/suites/crud_mock_server.lua' + srv.start() + + return srv + + @unittest.skipIf(sys.platform.startswith("win"), "Crud tests on windows platform are not supported: " "complexity of the vshard replicaset configuration") @@ -33,14 +41,19 @@ def setUpClass(cls): print('-' * 70, file=sys.stderr) # Create server and extract helpful fields for tests. cls.srv = create_server() + cls.mock_srv = create_mock_server() cls.host = cls.srv.host cls.port = cls.srv.args['primary'] + cls.mock_host = cls.mock_srv.host + cls.mock_port = cls.mock_srv.args['primary'] def setUp(self): time.sleep(1) # Open connections to instance. self.conn = tarantool.Connection(host=self.host, port=self.port, user='guest', password='', fetch_schema=False) + self.mock_conn = tarantool.Connection(host=self.mock_host, port=self.mock_port, + user='guest', password='', fetch_schema=False) self.conn_mesh = tarantool.MeshConnection(host=self.host, port=self.port, user='guest', password='', fetch_schema=False) self.conn_pool = tarantool.ConnectionPool([{'host': self.host, 'port': self.port}], @@ -736,9 +749,15 @@ def test_crud_module_via_pool_connection(self): # Exception try testing. self._exception_operation_with_crud(testing_function, case, mode=tarantool.Mode.RW) + def test_error_rethrow(self): + self.assertRaisesRegex( + DatabaseError, "Unexpected connection error", + lambda: self.mock_conn.crud_replace('tester', [2, 100, 'Alice'], {'timeout': 10})) + def tearDown(self): # Close connections to instance. self.conn.close() + self.mock_conn.close() self.conn_mesh.close() self.conn_pool.close() @@ -747,3 +766,5 @@ def tearDownClass(cls): # Stop instance. cls.srv.stop() cls.srv.clean() + cls.mock_srv.stop() + cls.mock_srv.clean() diff --git a/test/suites/test_datetime.py b/test/suites/test_datetime.py index 40f84215..80958fdc 100644 --- a/test/suites/test_datetime.py +++ b/test/suites/test_datetime.py @@ -16,6 +16,7 @@ from .lib.tarantool_server import TarantoolServer from .lib.skip import skip_or_run_datetime_test, skip_or_run_datetime_2_11_test +from .utils import assert_admin_success class TestSuiteDatetime(unittest.TestCase): @@ -28,25 +29,28 @@ def setUpClass(cls): cls.srv.start() cls.adm = cls.srv.admin - cls.adm(r""" + resp = cls.adm(""" _, datetime = pcall(require, 'datetime') - box.schema.space.create('test') + box.schema.space.create('test', {if_not_exists = true}) box.space['test']:create_index('primary', { type = 'tree', parts = {1, 'string'}, - unique = true}) + unique = true, + if_not_exists = true}) pcall(function() box.schema.space.create('test_pk') box.space['test_pk']:create_index('primary', { type = 'tree', parts = {1, 'datetime'}, - unique = true}) + unique = true, + if_not_exists = true}) end) box.schema.user.create('test', {password = 'test', if_not_exists = true}) - box.schema.user.grant('test', 'read,write,execute', 'universe') + box.schema.user.grant('test', 'read,write,execute', 'universe', + nil, {if_not_exists = true}) local function add(arg1, arg2) return arg1 + arg2 @@ -57,7 +61,10 @@ def setUpClass(cls): return arg1 - arg2 end rawset(_G, 'sub', sub) + + return true """) + assert_admin_success(resp) cls.con = tarantool.Connection(cls.srv.host, cls.srv.args['primary'], user='test', password='test') @@ -67,7 +74,11 @@ def setUp(self): if self.srv.is_started(): self.srv.touch_lock() - self.adm("box.space['test']:truncate()") + resp = self.adm(""" + box.space['test']:truncate() + return true + """) + assert_admin_success(resp) def test_datetime_class_api(self): datetime = tarantool.Datetime(year=2022, month=8, day=31, hour=18, minute=7, sec=54, @@ -142,6 +153,24 @@ def test_datetime_class_api_wth_tz(self): 'type': ValueError, 'msg': 'Failed to create datetime with ambiguous timezone "AET"' }, + 'under_min_timestamp_1': { + 'args': [], + 'kwargs': {'timestamp': -62135596801}, + 'type': OverflowError, + 'msg': 'date value out of range' + }, + 'under_min_timestamp_2': { + 'args': [], + 'kwargs': {'timestamp': -62135596800, 'nsec': -1}, + 'type': OverflowError, + 'msg': 'date value out of range' + }, + 'over_max_timestamp': { + 'args': [], + 'kwargs': {'timestamp': 253402300800}, + 'type': OverflowError, + 'msg': 'date value out of range' + }, } def test_datetime_class_invalid_init(self): @@ -282,6 +311,28 @@ def test_datetime_class_invalid_init(self): 'tarantool': r"datetime.new({timestamp=1661969274, nsec=308543321, " r"tz='Europe/Moscow'})", }, + 'min_datetime': { # Python datetime.MINYEAR is 1. + 'python': tarantool.Datetime(year=1, month=1, day=1, hour=0, minute=0, sec=0), + 'msgpack': (b'\x00\x09\x6e\x88\xf1\xff\xff\xff'), + 'tarantool': r"datetime.new({year=1, month=1, day=1, hour=0, min=0, sec=0})", + }, + 'max_datetime': { # Python datetime.MAXYEAR is 9999. + 'python': tarantool.Datetime(year=9999, month=12, day=31, hour=23, minute=59, sec=59, + nsec=999999999), + 'msgpack': (b'\x7f\x41\xf4\xff\x3a\x00\x00\x00\xff\xc9\x9a\x3b\x00\x00\x00\x00'), + 'tarantool': r"datetime.new({year=9999, month=12, day=31, hour=23, min=59, sec=59," + r"nsec=999999999})", + }, + 'min_datetime_timestamp': { # Python datetime.MINYEAR is 1. + 'python': tarantool.Datetime(timestamp=-62135596800), + 'msgpack': (b'\x00\x09\x6e\x88\xf1\xff\xff\xff'), + 'tarantool': r"datetime.new({timestamp=-62135596800})", + }, + 'max_datetime_timestamp': { # Python datetime.MAXYEAR is 9999. + 'python': tarantool.Datetime(timestamp=253402300799, nsec=999999999), + 'msgpack': (b'\x7f\x41\xf4\xff\x3a\x00\x00\x00\xff\xc9\x9a\x3b\x00\x00\x00\x00'), + 'tarantool': r"datetime.new({timestamp=253402300799, nsec=999999999})", + }, } def test_msgpack_decode(self): @@ -480,6 +531,12 @@ def test_tarantool_datetime_subtraction_different_timezones(self): 'res_add': tarantool.Datetime(year=2009, month=3, day=31), 'res_sub': tarantool.Datetime(year=2009, month=1, day=31), }, + 'week': { + 'arg_1': tarantool.Datetime(year=2008, month=2, day=3), + 'arg_2': tarantool.Interval(week=1), + 'res_add': tarantool.Datetime(year=2008, month=2, day=10), + 'res_sub': tarantool.Datetime(year=2008, month=1, day=27), + }, } def test_python_interval_addition(self): diff --git a/test/suites/test_dbapi.py b/test/suites/test_dbapi.py index 83156b39..bd8c7487 100644 --- a/test/suites/test_dbapi.py +++ b/test/suites/test_dbapi.py @@ -12,6 +12,7 @@ from tarantool import dbapi from .lib.tarantool_server import TarantoolServer from .lib.skip import skip_or_run_sql_test +from .utils import assert_admin_success class TestSuiteDBAPI(dbapi20.DatabaseAPI20Test): @@ -39,6 +40,14 @@ def setUpClass(cls): "port": cls.srv.args['primary'] } + # grant full access to guest + resp = cls.srv.admin(""" + box.schema.user.grant('guest', 'create,read,write,execute', 'universe', + nil, {if_not_exists = true}) + return true + """) + assert_admin_success(resp) + @skip_or_run_sql_test def setUp(self): # prevent a remote tarantool from clean our session @@ -46,10 +55,6 @@ def setUp(self): self.srv.touch_lock() self.con.flush_schema() - # grant full access to guest - self.srv.admin("box.schema.user.grant('guest', 'create,read,write," - "execute', 'universe')") - @classmethod def tearDownClass(cls): cls.con.close() diff --git a/test/suites/test_decimal.py b/test/suites/test_decimal.py index b880eaa3..2875a7da 100644 --- a/test/suites/test_decimal.py +++ b/test/suites/test_decimal.py @@ -16,6 +16,7 @@ from .lib.tarantool_server import TarantoolServer from .lib.skip import skip_or_run_decimal_test +from .utils import assert_admin_success class TestSuiteDecimal(unittest.TestCase): @@ -28,26 +29,32 @@ def setUpClass(cls): cls.srv.start() cls.adm = cls.srv.admin - cls.adm(r""" - _, decimal = pcall(require, 'decimal') + resp = cls.adm(""" + decimal_supported, decimal = pcall(require, 'decimal') - box.schema.space.create('test') + box.schema.space.create('test', {if_not_exists = true}) box.space['test']:create_index('primary', { type = 'tree', parts = {1, 'string'}, - unique = true}) + unique = true, + if_not_exists = true}) - pcall(function() + if decimal_supported then box.schema.space.create('test_pk') box.space['test_pk']:create_index('primary', { type = 'tree', parts = {1, 'decimal'}, - unique = true}) - end) + unique = true, + if_not_exists = true}) + end box.schema.user.create('test', {password = 'test', if_not_exists = true}) - box.schema.user.grant('test', 'read,write,execute', 'universe') + box.schema.user.grant('test', 'read,write,execute', 'universe', + nil, {if_not_exists = true}) + + return true """) + assert_admin_success(resp) cls.con = tarantool.Connection(cls.srv.host, cls.srv.args['primary'], user='test', password='test') @@ -57,7 +64,11 @@ def setUp(self): if self.srv.is_started(): self.srv.touch_lock() - self.adm("box.space['test']:truncate()") + resp = self.adm(""" + box.space['test']:truncate() + return true + """) + assert_admin_success(resp) valid_cases = { 'simple_decimal_1': { diff --git a/test/suites/test_dml.py b/test/suites/test_dml.py index 0263d451..26539eec 100644 --- a/test/suites/test_dml.py +++ b/test/suites/test_dml.py @@ -10,6 +10,7 @@ from .lib.tarantool_server import TarantoolServer from .lib.skip import skip_or_run_error_extra_info_test +from .utils import assert_admin_success class TestSuiteRequest(unittest.TestCase): @@ -22,29 +23,35 @@ def setUpClass(cls): cls.srv.start() cls.con = tarantool.Connection(cls.srv.host, cls.srv.args['primary']) cls.adm = cls.srv.admin - cls.space_created = cls.adm("box.schema.create_space('space_1')") - cls.adm(""" - box.space['space_1']:create_index('primary', { - type = 'tree', - parts = {1, 'num'}, - unique = true}) - """.replace('\n', ' ')) - cls.adm(""" - box.space['space_1']:create_index('secondary', { - type = 'tree', - parts = {2, 'num', 3, 'str'}, - unique = false}) - """.replace('\n', ' ')) - cls.space_created = cls.adm("box.schema.create_space('space_2')") - cls.adm(""" - box.space['space_2']:create_index('primary', { - type = 'hash', - parts = {1, 'num'}, - unique = true}) - """.replace('\n', ' ')) - cls.adm("json = require('json')") - cls.adm("fiber = require('fiber')") - cls.adm("uuid = require('uuid')") + cls.space_created = cls.adm("box.schema.create_space('space_1', {if_not_exists = true})") + resp = cls.adm(""" + box.space['space_1']:create_index('primary', { + type = 'tree', + parts = {1, 'num'}, + unique = true, + if_not_exists = true}) + + box.space['space_1']:create_index('secondary', { + type = 'tree', + parts = {2, 'num', 3, 'str'}, + unique = false, + if_not_exists = true}) + + box.schema.create_space('space_2', {if_not_exists = true}) + + box.space['space_2']:create_index('primary', { + type = 'hash', + parts = {1, 'num'}, + unique = true, + if_not_exists = true}) + + json = require('json') + fiber = require('fiber') + uuid = require('uuid') + + return true + """) + assert_admin_success(resp) if not sys.platform.startswith("win"): cls.sock_srv = TarantoolServer(create_unix_socket=True) @@ -60,10 +67,11 @@ def setUp(self): def test_00_00_authenticate(self): self.assertIsNone(self.srv.admin(""" - box.schema.user.create('test', { password = 'test' }) + box.schema.user.create('test', { password = 'test', if_not_exists = true }) """)) self.assertIsNone(self.srv.admin(""" - box.schema.user.grant('test', 'execute,read,write', 'universe') + box.schema.user.grant('test', 'execute,read,write', 'universe', + nil, {if_not_exists = true}) """)) self.assertEqual(self.con.authenticate('test', 'test')._data, None) @@ -210,7 +218,7 @@ def test_07_call_16(self): self.assertEqual(len(ans[0]), 1) self.assertIsInstance(ans[0][0], str) - self.assertSequenceEqual(con.call('box.tuple.new', [1, 2, 3, 'fld_1']), + self.assertSequenceEqual(con.call('box.tuple.new', [[1, 2, 3, 'fld_1']]), [[1, 2, 3, 'fld_1']]) self.assertSequenceEqual(con.call('box.tuple.new', 'fld_1'), [['fld_1']]) finally: @@ -236,7 +244,7 @@ def test_07_call_17(self): self.assertEqual(len(ans), 1) self.assertIsInstance(ans[0], str) - self.assertSequenceEqual(con.call('box.tuple.new', [1, 2, 3, 'fld_1']), + self.assertSequenceEqual(con.call('box.tuple.new', [[1, 2, 3, 'fld_1']]), [[1, 2, 3, 'fld_1']]) self.assertSequenceEqual(con.call('box.tuple.new', 'fld_1'), [['fld_1']]) @@ -311,7 +319,7 @@ def test_11_select_all_hash(self): space.select((), iterator=tarantool.const.ITERATOR_EQ) def test_12_update_fields(self): - self.srv.admin( + resp = self.srv.admin( """ do local sp = box.schema.create_space('sp', { @@ -325,7 +333,9 @@ def test_12_update_fields(self): parts = {1, 'unsigned'} }) end + return true """) + assert_admin_success(resp) self.con.insert('sp', [2, 'help', 4]) self.assertSequenceEqual( self.con.update('sp', (2,), [('+', 'thi', 3)]), @@ -381,7 +391,12 @@ def test_15_extra_error_info_stacked(self): self.assertEqual(exc.extra_info.message, "Timeout exceeded") self.assertEqual(exc.extra_info.errno, 0) self.assertEqual(exc.extra_info.errcode, 78) - self.assertEqual(exc.extra_info.fields, None) + actual_fields = exc.extra_info.fields + if actual_fields is None: + actual_fields = {} + expected_fields = {} + self.assertGreaterEqual(actual_fields.items(), + expected_fields.items()) self.assertNotEqual(exc.extra_info.prev, None) prev = exc.extra_info.prev self.assertEqual(prev.type, 'ClientError') @@ -390,7 +405,12 @@ def test_15_extra_error_info_stacked(self): self.assertEqual(prev.message, "Unknown error") self.assertEqual(prev.errno, 0) self.assertEqual(prev.errcode, 0) - self.assertEqual(prev.fields, None) + actual_fields = prev.fields + if actual_fields is None: + actual_fields = {} + expected_fields = {} + self.assertGreaterEqual(actual_fields.items(), + expected_fields.items()) else: self.fail('Expected error') @@ -409,13 +429,13 @@ def test_16_extra_error_info_fields(self): "Create access to function 'forbidden_function' is denied for user 'test'") self.assertEqual(exc.extra_info.errno, 0) self.assertEqual(exc.extra_info.errcode, 42) - self.assertEqual( - exc.extra_info.fields, + self.assertGreaterEqual( + exc.extra_info.fields.items(), { 'object_type': 'function', 'object_name': 'forbidden_function', 'access_type': 'Create' - }) + }.items()) self.assertEqual(exc.extra_info.prev, None) else: self.fail('Expected error') diff --git a/test/suites/test_encoding.py b/test/suites/test_encoding.py index 14a08f54..dcda5983 100644 --- a/test/suites/test_encoding.py +++ b/test/suites/test_encoding.py @@ -1,16 +1,18 @@ """ This module tests various type encoding cases. """ -# pylint: disable=missing-class-docstring,missing-function-docstring +# pylint: disable=missing-class-docstring,missing-function-docstring,duplicate-code import sys import unittest +import pkg_resources import tarantool from tarantool.error import DatabaseError from .lib.skip import skip_or_run_varbinary_test, skip_or_run_error_extra_info_test from .lib.tarantool_server import TarantoolServer +from .utils import assert_admin_success class TestSuiteEncoding(unittest.TestCase): @@ -24,10 +26,14 @@ def setUpClass(cls): cls.srv.script = 'test/suites/box.lua' cls.srv.start() - cls.srv.admin(""" - box.schema.user.create('test', { password = 'test' }) - box.schema.user.grant('test', 'execute,read,write', 'universe') + resp = cls.srv.admin(""" + box.schema.user.create('test', { password = 'test', if_not_exists = true }) + box.schema.user.grant('test', 'execute,read,write', 'universe', + nil, {if_not_exists = true}) + + return true """) + assert_admin_success(resp) args = [cls.srv.host, cls.srv.args['primary']] kwargs = {'user': 'test', 'password': 'test'} @@ -35,41 +41,47 @@ def setUpClass(cls): cls.con_encoding_none = tarantool.Connection(*args, encoding=None, **kwargs) cls.conns = [cls.con_encoding_utf8, cls.con_encoding_none] - cls.srv.admin("box.schema.create_space('space_str')") - cls.srv.admin(""" + resp = cls.srv.admin(""" + box.schema.create_space('space_str', {if_not_exists = true}) box.space['space_str']:create_index('primary', { type = 'tree', parts = {1, 'str'}, - unique = true}) - """.replace('\n', ' ')) - - cls.srv.admin("box.schema.create_space('space_varbin')") - cls.srv.admin(r""" - box.space['space_varbin']:format({ - { - 'id', - type = 'number', - is_nullable = false - }, - { - 'varbin', - type = 'varbinary', - is_nullable = false, - } - }) - """.replace('\n', ' ')) - cls.srv.admin(""" - box.space['space_varbin']:create_index('id', { - type = 'tree', - parts = {1, 'number'}, - unique = true}) - """.replace('\n', ' ')) - cls.srv.admin(""" - box.space['space_varbin']:create_index('varbin', { - type = 'tree', - parts = {2, 'varbinary'}, - unique = true}) - """.replace('\n', ' ')) + unique = true, + if_not_exists = true}) + + return true + """) + assert_admin_success(resp) + + if cls.srv.admin.tnt_version >= pkg_resources.parse_version('2.2.1'): + resp = cls.srv.admin(""" + box.schema.create_space('space_varbin', {if_not_exists = true}) + box.space['space_varbin']:format({ + { + 'id', + type = 'number', + is_nullable = false + }, + { + 'varbin', + type = 'varbinary', + is_nullable = false, + } + }) + box.space['space_varbin']:create_index('id', { + type = 'tree', + parts = {1, 'number'}, + unique = true, + if_not_exists = true}) + box.space['space_varbin']:create_index('varbin', { + type = 'tree', + parts = {2, 'varbinary'}, + unique = true, + if_not_exists = true}) + + return true + """) + assert_admin_success(resp) def assertNotRaises(self, func, *args, **kwargs): try: diff --git a/test/suites/test_error_ext.py b/test/suites/test_error_ext.py index 8d1a63cf..d5e96e75 100644 --- a/test/suites/test_error_ext.py +++ b/test/suites/test_error_ext.py @@ -15,6 +15,7 @@ from .lib.tarantool_server import TarantoolServer from .lib.skip import skip_or_run_error_ext_type_test +from .utils import assert_admin_success class TestSuiteErrorExt(unittest.TestCase): @@ -27,18 +28,23 @@ def setUpClass(cls): cls.srv.start() cls.adm = cls.srv.admin - cls.adm(r""" - box.schema.space.create('test') + resp = cls.adm(""" + box.schema.space.create('test', {if_not_exists = true}) box.space['test']:create_index('primary', { type = 'tree', parts = {1, 'string'}, - unique = true}) + unique = true, + if_not_exists = true}) box.schema.user.create('test', {password = 'test', if_not_exists = true}) - box.schema.user.grant('test', 'read,write,execute,create', 'universe') + box.schema.user.grant('test', 'read,write,execute,create', 'universe', + nil, {if_not_exists = true}) box.schema.user.create('no_grants', {if_not_exists = true}) + + return true """) + assert_admin_success(resp) cls.conn_encoding_utf8 = tarantool.Connection( cls.srv.host, cls.srv.args['primary'], @@ -78,7 +84,11 @@ def setUp(self): if self.srv.is_started(): self.srv.touch_lock() - self.adm("box.space['test']:truncate()") + resp = self.adm(""" + box.space['test']:truncate() + return true + """) + assert_admin_success(resp) # msgpack data for different encodings are actually the same, # but sometimes python msgpack module use different string @@ -317,7 +327,14 @@ def test_tarantool_decode(self): self.assertEqual(err.message, expected_err.message) self.assertEqual(err.errno, expected_err.errno) self.assertEqual(err.errcode, expected_err.errcode) - self.assertEqual(err.fields, expected_err.fields) + expected_fields = expected_err.fields + if expected_fields is None: + expected_fields = {} + actual_fields = err.fields + if actual_fields is None: + actual_fields = {} + self.assertGreaterEqual(actual_fields.items(), + expected_fields.items()) err = err.prev expected_err = expected_err.prev diff --git a/test/suites/test_execute.py b/test/suites/test_execute.py index 66fe0982..46e85ce7 100644 --- a/test/suites/test_execute.py +++ b/test/suites/test_execute.py @@ -9,6 +9,7 @@ import tarantool from .lib.tarantool_server import TarantoolServer from .lib.skip import skip_or_run_sql_test +from .utils import assert_admin_success class TestSuiteExecute(unittest.TestCase): @@ -32,6 +33,14 @@ def setUpClass(cls): cls.srv.start() cls.con = tarantool.Connection(cls.srv.host, cls.srv.args['primary']) + # grant full access to guest + resp = cls.srv.admin(""" + box.schema.user.grant('guest', 'create,read,write,execute', 'universe', + nil, {if_not_exists = true}) + return true + """) + assert_admin_success(resp) + @skip_or_run_sql_test def setUp(self): # prevent a remote tarantool from clean our session @@ -39,10 +48,6 @@ def setUp(self): self.srv.touch_lock() self.con.flush_schema() - # grant full access to guest - self.srv.admin("box.schema.user.grant('guest', 'create,read,write," - "execute', 'universe')") - @classmethod def tearDownClass(cls): cls.con.close() diff --git a/test/suites/test_interval.py b/test/suites/test_interval.py index 77f2cf52..2b6f6abe 100644 --- a/test/suites/test_interval.py +++ b/test/suites/test_interval.py @@ -13,9 +13,20 @@ from tarantool.error import MsgpackError from tarantool.msgpack_ext.packer import default as packer_default from tarantool.msgpack_ext.unpacker import ext_hook as unpacker_ext_hook +from tarantool.msgpack_ext.types.interval import ( + MAX_YEAR_RANGE, + MAX_MONTH_RANGE, + MAX_WEEK_RANGE, + MAX_DAY_RANGE, + MAX_HOUR_RANGE, + MAX_MIN_RANGE, + MAX_SEC_RANGE, + MAX_NSEC_RANGE, +) from .lib.tarantool_server import TarantoolServer from .lib.skip import skip_or_run_datetime_test +from .utils import assert_admin_success class TestSuiteInterval(unittest.TestCase): @@ -28,17 +39,19 @@ def setUpClass(cls): cls.srv.start() cls.adm = cls.srv.admin - cls.adm(r""" + resp = cls.adm(""" _, datetime = pcall(require, 'datetime') - box.schema.space.create('test') + box.schema.space.create('test', {if_not_exists = true}) box.space['test']:create_index('primary', { type = 'tree', parts = {1, 'string'}, - unique = true}) + unique = true, + if_not_exists = true}) box.schema.user.create('test', {password = 'test', if_not_exists = true}) - box.schema.user.grant('test', 'read,write,execute', 'universe') + box.schema.user.grant('test', 'read,write,execute', 'universe', + nil, {if_not_exists = true}) local function add(arg1, arg2) return arg1 + arg2 @@ -49,7 +62,10 @@ def setUpClass(cls): return arg1 - arg2 end rawset(_G, 'sub', sub) + + return true """) + assert_admin_success(resp) cls.con = tarantool.Connection(cls.srv.host, cls.srv.args['primary'], user='test', password='test') @@ -59,7 +75,11 @@ def setUp(self): if self.srv.is_started(): self.srv.touch_lock() - self.adm("box.space['test']:truncate()") + resp = self.adm(""" + box.space['test']:truncate() + return true + """) + assert_admin_success(resp) def test_interval_positional_init(self): self.assertRaisesRegex( @@ -71,42 +91,58 @@ def test_interval_positional_init(self): 'python': tarantool.Interval(year=1), 'msgpack': (b'\x02\x00\x01\x08\x01'), 'tarantool': r"datetime.interval.new({year=1})", + 'str': 'tarantool.Interval(year=1, month=0, week=0, day=0, hour=0, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', }, 'big_year': { 'python': tarantool.Interval(year=1000), 'msgpack': (b'\x02\x00\xcd\x03\xe8\x08\x01'), 'tarantool': r"datetime.interval.new({year=1000})", + 'str': 'tarantool.Interval(year=1000, month=0, week=0, day=0, hour=0, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', }, 'date': { 'python': tarantool.Interval(year=1, month=2, day=3), 'msgpack': (b'\x04\x00\x01\x01\x02\x03\x03\x08\x01'), 'tarantool': r"datetime.interval.new({year=1, month=2, day=3})", + 'str': 'tarantool.Interval(year=1, month=2, week=0, day=3, hour=0, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', }, 'big_month_date': { 'python': tarantool.Interval(year=1, month=100000, day=3), 'msgpack': (b'\x04\x00\x01\x01\xce\x00\x01\x86\xa0\x03\x03\x08\x01'), 'tarantool': r"datetime.interval.new({year=1, month=100000, day=3})", + 'str': 'tarantool.Interval(year=1, month=100000, week=0, day=3, hour=0, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', }, 'time': { 'python': tarantool.Interval(hour=1, minute=2, sec=3), 'msgpack': (b'\x04\x04\x01\x05\x02\x06\x03\x08\x01'), 'tarantool': r"datetime.interval.new({hour=1, min=2, sec=3})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=1, ' + 'minute=2, sec=3, nsec=0, adjust=Adjust.NONE)', }, 'big_seconds_time': { 'python': tarantool.Interval(hour=1, minute=2, sec=3000), 'msgpack': (b'\x04\x04\x01\x05\x02\x06\xcd\x0b\xb8\x08\x01'), 'tarantool': r"datetime.interval.new({hour=1, min=2, sec=3000})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=1, ' + 'minute=2, sec=3000, nsec=0, adjust=Adjust.NONE)', }, 'datetime': { 'python': tarantool.Interval(year=1, month=2, day=3, hour=1, minute=2, sec=3000), 'msgpack': (b'\x07\x00\x01\x01\x02\x03\x03\x04\x01\x05\x02\x06\xcd\x0b\xb8\x08\x01'), 'tarantool': r"datetime.interval.new({year=1, month=2, day=3, hour=1, " r"min=2, sec=3000})", + 'str': 'tarantool.Interval(year=1, month=2, week=0, day=3, hour=1, ' + 'minute=2, sec=3000, nsec=0, adjust=Adjust.NONE)', }, 'nanoseconds': { 'python': tarantool.Interval(nsec=10000000), 'msgpack': (b'\x02\x07\xce\x00\x98\x96\x80\x08\x01'), 'tarantool': r"datetime.interval.new({nsec=10000000})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + 'minute=0, sec=0, nsec=10000000, adjust=Adjust.NONE)', }, 'datetime_with_nanoseconds': { 'python': tarantool.Interval(year=1, month=2, day=3, hour=1, minute=2, @@ -115,6 +151,8 @@ def test_interval_positional_init(self): b'\x00\x98\x96\x80\x08\x01'), 'tarantool': r"datetime.interval.new({year=1, month=2, day=3, hour=1, " r"min=2, sec=3000, nsec=10000000})", + 'str': 'tarantool.Interval(year=1, month=2, week=0, day=3, hour=1, ' + 'minute=2, sec=3000, nsec=10000000, adjust=Adjust.NONE)', }, 'datetime_none_adjust': { 'python': tarantool.Interval(year=1, month=2, day=3, hour=1, minute=2, @@ -124,6 +162,8 @@ def test_interval_positional_init(self): b'\x00\x98\x96\x80\x08\x01'), 'tarantool': r"datetime.interval.new({year=1, month=2, day=3, hour=1, " r"min=2, sec=3000, nsec=10000000, adjust='none'})", + 'str': 'tarantool.Interval(year=1, month=2, week=0, day=3, hour=1, ' + 'minute=2, sec=3000, nsec=10000000, adjust=Adjust.NONE)', }, 'datetime_excess_adjust': { 'python': tarantool.Interval(year=1, month=2, day=3, hour=1, minute=2, @@ -133,6 +173,8 @@ def test_interval_positional_init(self): b'\x00\x98\x96\x80'), 'tarantool': r"datetime.interval.new({year=1, month=2, day=3, hour=1, " r"min=2, sec=3000, nsec=10000000, adjust='excess'})", + 'str': 'tarantool.Interval(year=1, month=2, week=0, day=3, hour=1, ' + 'minute=2, sec=3000, nsec=10000000, adjust=Adjust.EXCESS)', }, 'datetime_last_adjust': { 'python': tarantool.Interval(year=1, month=2, day=3, hour=1, minute=2, @@ -142,11 +184,295 @@ def test_interval_positional_init(self): b'\x00\x98\x96\x80\x08\x02'), 'tarantool': r"datetime.interval.new({year=1, month=2, day=3, hour=1, " r"min=2, sec=3000, nsec=10000000, adjust='last'})", + 'str': 'tarantool.Interval(year=1, month=2, week=0, day=3, hour=1, ' + 'minute=2, sec=3000, nsec=10000000, adjust=Adjust.LAST)', }, 'all_zeroes': { 'python': tarantool.Interval(adjust=tarantool.IntervalAdjust.EXCESS), 'msgpack': (b'\x00'), 'tarantool': r"datetime.interval.new({adjust='excess'})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.EXCESS)', + }, + 'weeks': { + 'python': tarantool.Interval(week=3), + 'msgpack': (b'\x02\x02\x03\x08\x01'), + 'tarantool': r"datetime.interval.new({week=3})", + 'str': 'tarantool.Interval(year=0, month=0, week=3, day=0, hour=0, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'date_with_week': { + 'python': tarantool.Interval(year=1, month=2, week=3, day=4), + 'msgpack': (b'\x05\x00\x01\x01\x02\x02\x03\x03\x04\x08\x01'), + 'tarantool': r"datetime.interval.new({year=1, month=2, week=3, day=4})", + 'str': 'tarantool.Interval(year=1, month=2, week=3, day=4, hour=0, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'datetime_with_week': { + 'python': tarantool.Interval(year=1, month=2, week=3, day=4, hour=1, minute=2, + sec=3000, nsec=10000000), + 'msgpack': (b'\x09\x00\x01\x01\x02\x02\x03\x03\x04\x04\x01\x05\x02\x06\xcd\x0b\xb8' + b'\x07\xce\x00\x98\x96\x80\x08\x01'), + 'tarantool': r"datetime.interval.new({year=1, month=2, week=3, day=4, hour=1, " + r"min=2, sec=3000, nsec=10000000})", + 'str': 'tarantool.Interval(year=1, month=2, week=3, day=4, hour=1, ' + 'minute=2, sec=3000, nsec=10000000, adjust=Adjust.NONE)', + }, + 'min_year_interval': { + 'python': tarantool.Interval(year=-int(MAX_YEAR_RANGE)), + 'msgpack': (b'\x02\x00\xd2\xff\x4c\x91\x8b\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{year=-{int(MAX_YEAR_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{year=-{int(MAX_YEAR_RANGE)} + 1}}) - " + r"datetime.interval.new({year=1})", + 'str': f'tarantool.Interval(year=-{int(MAX_YEAR_RANGE)}, month=0, week=0, day=0, ' + 'hour=0, minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'max_year_interval': { + 'python': tarantool.Interval(year=int(MAX_YEAR_RANGE)), + 'msgpack': (b'\x02\x00\xce\x00\xb3\x6e\x75\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{year={int(MAX_YEAR_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{year={int(MAX_YEAR_RANGE)} - 1}}) + " + r"datetime.interval.new({year=1})", + 'str': f'tarantool.Interval(year={int(MAX_YEAR_RANGE)}, month=0, week=0, day=0, ' + 'hour=0, minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'min_month_interval': { + 'python': tarantool.Interval(month=-int(MAX_MONTH_RANGE)), + 'msgpack': (b'\x02\x01\xd2\xf7\x96\xd2\x84\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{month=-{int(MAX_MONTH_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{month=-{int(MAX_MONTH_RANGE)} + 1}}) - " + r"datetime.interval.new({month=1})", + 'str': f'tarantool.Interval(year=0, month=-{int(MAX_MONTH_RANGE)}, week=0, day=0, ' + 'hour=0, minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'max_month_interval': { + 'python': tarantool.Interval(month=int(MAX_MONTH_RANGE)), + 'msgpack': (b'\x02\x01\xce\x08\x69\x2d\x7c\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{month={int(MAX_MONTH_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{month={int(MAX_MONTH_RANGE)} - 1}}) + " + r"datetime.interval.new({month=1})", + 'str': f'tarantool.Interval(year=0, month={int(MAX_MONTH_RANGE)}, week=0, day=0, ' + 'hour=0, minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'min_week_interval': { + 'python': tarantool.Interval(week=-int(MAX_WEEK_RANGE)), + 'msgpack': (b'\x02\x02\xd2\xdb\x6d\x85\xa8\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{week=-{int(MAX_WEEK_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{week=-{int(MAX_WEEK_RANGE)} + 1}}) - " + r"datetime.interval.new({week=1})", + 'str': f'tarantool.Interval(year=0, month=0, week=-{int(MAX_WEEK_RANGE)}, day=0, ' + 'hour=0, minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'max_week_interval': { + 'python': tarantool.Interval(week=int(MAX_WEEK_RANGE)), + 'msgpack': (b'\x02\x02\xce\x24\x92\x7a\x58\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{week={int(MAX_WEEK_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{week={int(MAX_WEEK_RANGE)} - 1}}) + " + r"datetime.interval.new({week=1})", + 'str': f'tarantool.Interval(year=0, month=0, week={int(MAX_WEEK_RANGE)}, day=0, ' + 'hour=0, minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'min_day_interval': { + 'python': tarantool.Interval(day=-int(MAX_DAY_RANGE)), + 'msgpack': (b'\x02\x03\xd3\xff\xff\xff\xfe\xff\xfe\xa7\x92\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{day=-{int(MAX_DAY_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{day=-{int(MAX_DAY_RANGE)} + 1}}) - " + r"datetime.interval.new({day=1})", + 'str': f'tarantool.Interval(year=0, month=0, week=0, day=-{int(MAX_DAY_RANGE)}, ' + 'hour=0, minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + 'tarantool_8887_issue': True, + }, + 'max_day_interval': { + 'python': tarantool.Interval(day=int(MAX_DAY_RANGE)), + 'msgpack': (b'\x02\x03\xcf\x00\x00\x00\x01\x00\x01\x58\x6e\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{day={int(MAX_DAY_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{day={int(MAX_DAY_RANGE)} - 1}}) + " + r"datetime.interval.new({day=1})", + 'str': f'tarantool.Interval(year=0, month=0, week=0, day={int(MAX_DAY_RANGE)}, hour=0, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + 'tarantool_8887_issue': True, + }, + 'min_int32_day_interval': { + 'python': tarantool.Interval(day=-2147483648), + 'msgpack': (b'\x02\x03\xd2\x80\x00\x00\x00\x08\x01'), + 'tarantool': r"datetime.interval.new({day=-2147483648})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=-2147483648, hour=0, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'max_int32_day_interval': { + 'python': tarantool.Interval(day=2147483647), + 'msgpack': (b'\x02\x03\xce\x7f\xff\xff\xff\x08\x01'), + 'tarantool': r"datetime.interval.new({day=2147483647})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=2147483647, hour=0, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'min_hour_interval': { + 'python': tarantool.Interval(hour=-int(MAX_HOUR_RANGE)), + 'msgpack': (b'\x02\x04\xd3\xff\xff\xff\xe7\xff\xdf\xb5\xaa\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{hour=-{int(MAX_HOUR_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{hour=-{int(MAX_HOUR_RANGE)} + 1}}) - " + r"datetime.interval.new({hour=1})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, ' + f'hour=-{int(MAX_HOUR_RANGE)}, minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + 'tarantool_8887_issue': True, + }, + 'max_hour_interval': { + 'python': tarantool.Interval(hour=int(MAX_HOUR_RANGE)), + 'msgpack': (b'\x02\x04\xcf\x00\x00\x00\x18\x00\x20\x4a\x56\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{hour={int(MAX_HOUR_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{hour={int(MAX_HOUR_RANGE)} - 1}}) + " + r"datetime.interval.new({hour=1})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, ' + f'hour={int(MAX_HOUR_RANGE)}, minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + 'tarantool_8887_issue': True, + }, + 'min_int32_hour_interval': { + 'python': tarantool.Interval(hour=-2147483648), + 'msgpack': (b'\x02\x04\xd2\x80\x00\x00\x00\x08\x01'), + 'tarantool': r"datetime.interval.new({hour=-2147483648})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=-2147483648, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'max_int32_hour_interval': { + 'python': tarantool.Interval(hour=2147483647), + 'msgpack': (b'\x02\x04\xce\x7f\xff\xff\xff\x08\x01'), + 'tarantool': r"datetime.interval.new({hour=2147483647})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=2147483647, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'min_minute_interval': { + 'python': tarantool.Interval(minute=-int(MAX_MIN_RANGE)), + 'msgpack': (b'\x02\x05\xd3\xff\xff\xfa\x5f\xf8\x6e\x93\xd8\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{min=-{int(MAX_MIN_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{min=-{int(MAX_MIN_RANGE)} + 1}}) - " + r"datetime.interval.new({min=1})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + f'minute=-{int(MAX_MIN_RANGE)}, sec=0, nsec=0, adjust=Adjust.NONE)', + 'tarantool_8887_issue': True, + }, + 'max_minute_interval': { + 'python': tarantool.Interval(minute=int(MAX_MIN_RANGE)), + 'msgpack': (b'\x02\x05\xcf\x00\x00\x05\xa0\x07\x91\x6c\x28\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{min={int(MAX_MIN_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{min={int(MAX_MIN_RANGE)} - 1}}) + " + r"datetime.interval.new({min=1})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + f'minute={int(MAX_MIN_RANGE)}, sec=0, nsec=0, adjust=Adjust.NONE)', + 'tarantool_8887_issue': True, + }, + 'min_int32_minute_interval': { + 'python': tarantool.Interval(minute=-2147483648), + 'msgpack': (b'\x02\x05\xd2\x80\x00\x00\x00\x08\x01'), + 'tarantool': r"datetime.interval.new({min=-2147483648})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + 'minute=-2147483648, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'max_int32_minute_interval': { + 'python': tarantool.Interval(minute=2147483647), + 'msgpack': (b'\x02\x05\xce\x7f\xff\xff\xff\x08\x01'), + 'tarantool': r"datetime.interval.new({min=2147483647})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + 'minute=2147483647, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'min_sec_interval': { + 'python': tarantool.Interval(sec=-int(MAX_SEC_RANGE)), + 'msgpack': (b'\x02\x06\xd3\xff\xfe\xae\x7e\x39\xea\xa6\xa0\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{sec=-{int(MAX_SEC_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{sec=-{int(MAX_SEC_RANGE)} + 1}}) - " + r"datetime.interval.new({sec=1})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + f'minute=0, sec=-{int(MAX_SEC_RANGE)}, nsec=0, adjust=Adjust.NONE)', + 'tarantool_8887_issue': True, + }, + 'max_sec_interval': { + 'python': tarantool.Interval(sec=int(MAX_SEC_RANGE)), + 'msgpack': (b'\x02\x06\xcf\x00\x01\x51\x81\xc6\x15\x59\x60\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{sec={int(MAX_SEC_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{sec={int(MAX_SEC_RANGE)} - 1}}) + " + r"datetime.interval.new({sec=1})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + f'minute=0, sec={int(MAX_SEC_RANGE)}, nsec=0, adjust=Adjust.NONE)', + 'tarantool_8887_issue': True, + }, + 'min_int32_sec_interval': { + 'python': tarantool.Interval(sec=-2147483648), + 'msgpack': (b'\x02\x06\xd2\x80\x00\x00\x00\x08\x01'), + 'tarantool': r"datetime.interval.new({sec=-2147483648})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + 'minute=0, sec=-2147483648, nsec=0, adjust=Adjust.NONE)', + }, + 'max_int32_sec_interval': { + 'python': tarantool.Interval(sec=2147483647), + 'msgpack': (b'\x02\x06\xce\x7f\xff\xff\xff\x08\x01'), + 'tarantool': r"datetime.interval.new({sec=2147483647})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + 'minute=0, sec=2147483647, nsec=0, adjust=Adjust.NONE)', + }, + 'min_nsec_interval': { + 'python': tarantool.Interval(nsec=-int(MAX_NSEC_RANGE)), + 'msgpack': (b'\x02\x07\xd2\x80\x00\x00\x01\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{nsec=-{int(MAX_NSEC_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{nsec=-{int(MAX_NSEC_RANGE)} + 1}}) - " + r"datetime.interval.new({nsec=1})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + f'minute=0, sec=0, nsec=-{int(MAX_NSEC_RANGE)}, adjust=Adjust.NONE)', + }, + 'max_nsec_interval': { + 'python': tarantool.Interval(nsec=int(MAX_NSEC_RANGE)), + 'msgpack': (b'\x02\x07\xce\x7f\xff\xff\xff\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{nsec={int(MAX_NSEC_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{nsec={int(MAX_NSEC_RANGE)} - 1}}) + " + r"datetime.interval.new({nsec=1})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + f'minute=0, sec=0, nsec={int(MAX_NSEC_RANGE)}, adjust=Adjust.NONE)', }, } @@ -165,6 +491,9 @@ def test_msgpack_decode(self): def test_tarantool_decode(self): for name, case in self.cases.items(): with self.subTest(msg=name): + if ('tarantool_8887_issue' in case) and (case['tarantool_8887_issue'] is True): + self.skipTest('See https://github.com/tarantool/tarantool/issues/8887') + self.adm(f"box.space['test']:replace{{'{name}', {case['tarantool']}, 'field'}}") self.assertSequenceEqual(self.con.select('test', name), @@ -180,6 +509,9 @@ def test_msgpack_encode(self): def test_tarantool_encode(self): for name, case in self.cases.items(): with self.subTest(msg=name): + if ('tarantool_8887_issue' in case) and (case['tarantool_8887_issue'] is True): + self.skipTest('See https://github.com/tarantool/tarantool/issues/8887') + self.con.insert('test', [name, case['python'], 'field']) lua_eval = f""" @@ -198,6 +530,12 @@ def test_tarantool_encode(self): self.assertSequenceEqual(self.adm(lua_eval), [True]) + def test_class_string(self): + for name, case in self.cases.items(): + with self.subTest(msg=name): + self.assertEqual(str(case['python']), case['str']) + self.assertEqual(repr(case['python']), case['str']) + def test_unknown_field_decode(self): case = b'\x01\x09\xce\x00\x98\x96\x80' self.assertRaisesRegex( @@ -210,6 +548,87 @@ def test_unknown_adjust_decode(self): MsgpackError, '3 is not a valid Adjust', lambda: unpacker_ext_hook(6, case, self.con._unpacker_factory())) + out_of_range_cases = { + 'year_too_small': { + 'kwargs': {'year': -int(MAX_YEAR_RANGE + 1)}, + 'range': MAX_YEAR_RANGE, + }, + 'year_too_large': { + 'kwargs': {'year': int(MAX_YEAR_RANGE + 1)}, + 'range': MAX_YEAR_RANGE, + }, + 'month_too_small': { + 'kwargs': {'month': -int(MAX_MONTH_RANGE + 1)}, + 'range': MAX_MONTH_RANGE, + }, + 'month_too_big': { + 'kwargs': {'month': int(MAX_MONTH_RANGE + 1)}, + 'range': MAX_MONTH_RANGE, + }, + 'week_too_small': { + 'kwargs': {'week': -int(MAX_WEEK_RANGE + 1)}, + 'range': MAX_WEEK_RANGE, + }, + 'week_too_big': { + 'kwargs': {'week': int(MAX_WEEK_RANGE + 1)}, + 'range': MAX_WEEK_RANGE, + }, + 'day_too_small': { + 'kwargs': {'day': -int(MAX_DAY_RANGE + 1)}, + 'range': MAX_DAY_RANGE, + }, + 'day_too_big': { + 'kwargs': {'day': int(MAX_DAY_RANGE + 1)}, + 'range': MAX_DAY_RANGE, + }, + 'hour_too_small': { + 'kwargs': {'hour': -int(MAX_HOUR_RANGE + 1)}, + 'range': MAX_HOUR_RANGE, + }, + 'hour_too_big': { + 'kwargs': {'hour': int(MAX_HOUR_RANGE + 1)}, + 'range': MAX_HOUR_RANGE, + }, + 'minute_too_small': { + 'kwargs': {'minute': -int(MAX_MIN_RANGE + 1)}, + 'range': MAX_MIN_RANGE, + }, + 'minute_too_big': { + 'kwargs': {'minute': int(MAX_MIN_RANGE + 1)}, + 'range': MAX_MIN_RANGE, + }, + 'sec_too_small': { + 'kwargs': {'sec': -int(MAX_SEC_RANGE + 1)}, + 'range': MAX_SEC_RANGE, + }, + 'sec_too_big': { + 'kwargs': {'sec': int(MAX_SEC_RANGE + 1)}, + 'range': MAX_SEC_RANGE, + }, + 'nsec_too_small': { + 'kwargs': {'nsec': -int(MAX_NSEC_RANGE + 1)}, + 'range': MAX_NSEC_RANGE, + }, + 'nsec_too_big': { + 'kwargs': {'nsec': int(MAX_NSEC_RANGE + 1)}, + 'range': MAX_NSEC_RANGE, + }, + } + + def test_out_of_range(self): + # pylint: disable=cell-var-from-loop,unnecessary-lambda + + for name, case in self.out_of_range_cases.items(): + with self.subTest(msg=name): + name = next(iter(case['kwargs'])) + val = case['kwargs'][name] + self.assertRaisesRegex( + ValueError, re.escape( + f"value {val} of {name} is out of " + f"allowed range [{-case['range']}, {case['range']}]" + ), + lambda: tarantool.Interval(**case['kwargs'])) + arithmetic_cases = { 'year': { 'arg_1': tarantool.Interval(year=2), @@ -265,6 +684,28 @@ def test_unknown_adjust_decode(self): 'res_add': tarantool.Interval(year=3, adjust=tarantool.IntervalAdjust.LAST), 'res_sub': tarantool.Interval(year=1, adjust=tarantool.IntervalAdjust.LAST), }, + 'weeks': { + 'arg_1': tarantool.Interval(week=2), + 'arg_2': tarantool.Interval(week=1), + 'res_add': tarantool.Interval(week=3), + 'res_sub': tarantool.Interval(week=1), + }, + 'date_with_week': { + 'arg_1': tarantool.Interval(year=1, month=2, week=3, day=4), + 'arg_2': tarantool.Interval(year=4, month=3, week=2, day=1), + 'res_add': tarantool.Interval(year=5, month=5, week=5, day=5), + 'res_sub': tarantool.Interval(year=-3, month=-1, week=1, day=3), + }, + 'datetime_with_week': { + 'arg_1': tarantool.Interval(year=1, month=2, week=3, day=4, hour=1, minute=2, + sec=3000, nsec=10000000), + 'arg_2': tarantool.Interval(year=2, month=1, week=-1, day=31, hour=-3, minute=0, + sec=1000, nsec=9876543), + 'res_add': tarantool.Interval(year=3, month=3, week=2, day=35, hour=-2, minute=2, + sec=4000, nsec=19876543), + 'res_sub': tarantool.Interval(year=-1, month=1, week=4, day=-27, hour=4, minute=2, + sec=2000, nsec=123457), + }, } def test_python_interval_addition(self): @@ -291,6 +732,22 @@ def test_tarantool_interval_subtraction(self): self.assertSequenceEqual(self.con.call('sub', case['arg_1'], case['arg_2']), [case['res_sub']]) + def test_addition_overflow(self): + self.assertRaisesRegex( + ValueError, re.escape( + f"value {int(MAX_YEAR_RANGE) + 1} of year is out of " + f"allowed range [{-MAX_YEAR_RANGE}, {MAX_YEAR_RANGE}]" + ), + lambda: tarantool.Interval(year=int(MAX_YEAR_RANGE)) + tarantool.Interval(year=1)) + + def test_subtraction_overflow(self): + self.assertRaisesRegex( + ValueError, re.escape( + f"value {-int(MAX_YEAR_RANGE) - 1} of year is out of " + f"allowed range [{-MAX_YEAR_RANGE}, {MAX_YEAR_RANGE}]" + ), + lambda: tarantool.Interval(year=-int(MAX_YEAR_RANGE)) - tarantool.Interval(year=1)) + @classmethod def tearDownClass(cls): cls.con.close() diff --git a/test/suites/test_mesh.py b/test/suites/test_mesh.py index b82ccc0e..a906597b 100644 --- a/test/suites/test_mesh.py +++ b/test/suites/test_mesh.py @@ -16,18 +16,23 @@ ClusterDiscoveryWarning, ) from .lib.tarantool_server import TarantoolServer +from .utils import assert_admin_success def create_server(_id): srv = TarantoolServer() srv.script = 'test/suites/box.lua' srv.start() - srv.admin("box.schema.user.create('test', {password = 'test', " - "if_not_exists = true})") - srv.admin("box.schema.user.grant('test', 'execute', 'universe')") + resp = srv.admin(f""" + box.schema.user.create('test', {{password = 'test', if_not_exists = true}}) + box.schema.user.grant('test', 'execute', 'universe', + nil, {{if_not_exists = true}}) - # Create srv_id function (for testing purposes). - srv.admin(f"function srv_id() return {_id} end") + function srv_id() return {_id} end + + return true + """) + assert_admin_success(resp) return srv @@ -43,18 +48,22 @@ def define_cluster_function(self, func_name, servers): function {func_name}() return {{{addresses_lua}}} end + return true """ for srv in self.servers: - srv.admin(func_body) + resp = srv.admin(func_body) + assert_admin_success(resp) def define_custom_cluster_function(self, func_name, retval): func_body = f""" function {func_name}() return {retval} end + return true """ for srv in self.servers: - srv.admin(func_body) + resp = srv.admin(func_body) + assert_admin_success(resp) @classmethod def setUpClass(cls): @@ -99,7 +108,11 @@ def assert_srv_id(con, srv_id): # Start instance#1, stop instance#2 -- response from # instance#1 again. self.srv.start() - self.srv.admin('function srv_id() return 1 end') + resp = self.srv.admin(""" + function srv_id() return 1 end + return true + """) + assert_admin_success(resp) self.srv2.stop() assert_srv_id(con, 1) @@ -122,7 +135,6 @@ def test_01_contructor(self): # Verify that a bad address given at initialization leads # to an error. bad_addrs = [ - {"port": 1234}, # no host {"host": "localhost"}, # no port {"host": "localhost", "port": "1234"}, # port is str ] diff --git a/test/suites/test_pool.py b/test/suites/test_pool.py index 7134f220..e15bcbbf 100644 --- a/test/suites/test_pool.py +++ b/test/suites/test_pool.py @@ -20,31 +20,46 @@ from .lib.skip import skip_or_run_sql_test from .lib.tarantool_server import TarantoolServer +from .utils import assert_admin_success def create_server(_id): srv = TarantoolServer() srv.script = 'test/suites/box.lua' srv.start() - srv.admin("box.schema.user.create('test', {password = 'test', " - "if_not_exists = true})") - srv.admin("box.schema.user.grant('test', 'execute', 'universe')") - srv.admin("box.schema.space.create('test')") - srv.admin(r"box.space.test:format({" - r" { name = 'pk', type = 'string' }," - r" { name = 'id', type = 'number', is_nullable = true }" - r"})") - srv.admin(r"box.space.test:create_index('pk'," - r"{ unique = true," - r" parts = {{field = 1, type = 'string'}}})") - srv.admin(r"box.space.test:create_index('id'," - r"{ unique = true," - r" parts = {{field = 2, type = 'number', is_nullable=true}}})") - srv.admin("box.schema.user.grant('test', 'read,write', 'space', 'test')") - srv.admin("json = require('json')") - - # Create srv_id function (for testing purposes). - srv.admin(f"function srv_id() return {_id} end") + resp = srv.admin(f""" + box.schema.user.create('test', {{password = 'test', if_not_exists = true}}) + box.schema.user.grant('test', 'execute', 'universe', + nil, {{if_not_exists = true}}) + box.schema.space.create('test', {{if_not_exists = true}}) + box.space.test:format({{ + {{ name = 'pk', type = 'string' }}, + {{ name = 'id', type = 'number', is_nullable = true }} + }}) + box.space.test:create_index('pk', + {{ + unique = true, + parts = {{ + {{field = 1, type = 'string'}} + }}, + if_not_exists = true, + }}) + box.space.test:create_index('id', + {{ + unique = true, + parts = {{ + {{field = 2, type = 'number', is_nullable=true}} + }}, + if_not_exists = true, + }}) + box.schema.user.grant('test', 'read,write', 'space', 'test', {{if_not_exists = true}}) + json = require('json') + + function srv_id() return {_id} end + + return true + """) + assert_admin_success(resp) return srv @@ -53,11 +68,12 @@ def create_server(_id): class TestSuitePool(unittest.TestCase): def set_ro(self, srv, read_only): if read_only: - req = r'box.cfg{read_only = true}' + req = r'box.cfg{read_only = true}; return true' else: - req = r'box.cfg{read_only = false}' + req = r'box.cfg{read_only = false}; return true' - srv.admin(req) + resp = srv.admin(req) + assert_admin_success(resp) def set_cluster_ro(self, read_only_list): assert len(self.servers) == len(read_only_list) diff --git a/test/suites/test_push.py b/test/suites/test_push.py index bd14455b..24a1ae7d 100644 --- a/test/suites/test_push.py +++ b/test/suites/test_push.py @@ -7,68 +7,58 @@ import unittest import tarantool from .lib.tarantool_server import TarantoolServer +from .utils import assert_admin_success def create_server(): srv = TarantoolServer() srv.script = 'test/suites/box.lua' srv.start() - srv.admin("box.schema.user.create('test', {password = 'test', " - "if_not_exists = true})") - srv.admin("box.schema.user.grant('test', 'read,write,execute', 'universe')") - - # Create server_function (for testing purposes). - srv.admin(""" - function server_function() - x = {0,0} - while x[1] < 3 do - x[1] = x[1] + 1 - box.session.push(x) + resp = srv.admin(""" + box.schema.user.create('test', {password = 'test', if_not_exists = true}) + box.schema.user.grant('test', 'read,write,execute', 'universe', + nil, {if_not_exists = true}) + + function server_function() + x = {0,0} + while x[1] < 3 do + x[1] = x[1] + 1 + box.session.push(x) + end + return x end - return x - end - """) - # Create tester space and on_replace trigger (for testing purposes). - srv.admin(""" - box.schema.create_space( - 'tester', { - format = { - {name = 'id', type = 'unsigned'}, - {name = 'name', type = 'string'}, - } - }) - """) - srv.admin(""" - box.space.tester:create_index( - 'primary_index', { - parts = { - {field = 1, type = 'unsigned'}, - } - }) - """) - srv.admin(""" - box.space.tester:create_index( - 'primary_index', { - parts = { - {field = 1, type = 'unsigned'}, - } - }) - """) - srv.admin(""" - function on_replace_callback() - x = {0,0} - while x[1] < 300 do - x[1] = x[1] + 100 - box.session.push(x) + box.schema.create_space( + 'tester', { + format = { + {name = 'id', type = 'unsigned'}, + {name = 'name', type = 'string'}, + } + }) + + box.space.tester:create_index( + 'primary_index', { + parts = { + {field = 1, type = 'unsigned'}, + }, + if_not_exists = true, + }) + + function on_replace_callback() + x = {0,0} + while x[1] < 300 do + x[1] = x[1] + 100 + box.session.push(x) + end end - end - """) - srv.admin(""" - box.space.tester:on_replace( - on_replace_callback - ) + + box.space.tester:on_replace( + on_replace_callback + ) + + return true """) + assert_admin_success(resp) return srv diff --git a/test/suites/test_schema.py b/test/suites/test_schema.py index 1402616d..c90ddbff 100644 --- a/test/suites/test_schema.py +++ b/test/suites/test_schema.py @@ -1,7 +1,7 @@ """ This module tests space and index schema fetch. """ -# pylint: disable=missing-class-docstring,missing-function-docstring,fixme,too-many-public-methods,too-many-branches,too-many-statements +# pylint: disable=missing-class-docstring,missing-function-docstring,fixme,too-many-public-methods,too-many-branches,too-many-statements,duplicate-code import sys import unittest @@ -12,6 +12,7 @@ from .lib.tarantool_server import TarantoolServer from .lib.skip import skip_or_run_constraints_test +from .utils import assert_admin_success # FIXME: I'm quite sure that there is a simpler way to count @@ -54,29 +55,35 @@ def setUpClass(cls): cls.srv = TarantoolServer() cls.srv.script = 'test/suites/box.lua' cls.srv.start() - cls.srv.admin("box.schema.user.create('test', {password = 'test', if_not_exists = true})") - cls.srv.admin("box.schema.user.grant('test', 'read,write,execute', 'universe')") - - # Create server_function and tester space (for fetch_schema opt testing purposes). - cls.srv.admin("function server_function() return 2+2 end") - cls.srv.admin(""" - box.schema.create_space( - 'tester', { - format = { - {name = 'id', type = 'unsigned'}, - {name = 'name', type = 'string', is_nullable = true}, - } - }) - """) - cls.srv.admin(""" - box.space.tester:create_index( - 'primary_index', { - parts = { - {field = 1, type = 'unsigned'}, - } - }) + resp = cls.srv.admin(""" + box.schema.user.create('test', {password = 'test', if_not_exists = true}) + box.schema.user.grant('test', 'read,write,execute', 'universe', + nil, {if_not_exists = true}) + + function server_function() return 2+2 end + + box.schema.create_space( + 'tester', { + format = { + {name = 'id', type = 'unsigned'}, + {name = 'name', type = 'string', is_nullable = true}, + }, + if_not_exists = true, + }) + + box.space.tester:create_index( + 'primary_index', { + parts = { + {field = 1, type = 'unsigned'}, + }, + if_not_exists = true, + }) + + box.space.tester:insert({1, nil}) + + return true """) - cls.srv.admin("box.space.tester:insert({1, null})") + assert_admin_success(resp) cls.con = tarantool.Connection(cls.srv.host, cls.srv.args['primary'], encoding=cls.encoding, user='test', password='test') @@ -112,31 +119,34 @@ def setUpClass(cls): """) if cls.srv.admin.tnt_version >= pkg_resources.parse_version('2.10.0'): - cls.srv.admin(""" - box.schema.create_space( - 'constr_tester_1', { - format = { - { name = 'id', type = 'unsigned' }, - { name = 'payload', type = 'number' }, - } - }) - box.space.constr_tester_1:create_index('I1', { parts = {'id'} }) - - box.space.constr_tester_1:replace({1, 999}) - - box.schema.create_space( - 'constr_tester_2', { - format = { - { name = 'id', type = 'unsigned' }, - { name = 'table1_id', type = 'unsigned', - foreign_key = { fk_video = { space = 'constr_tester_1', field = 'id' } }, - }, - { name = 'payload', type = 'number' }, - } - }) - box.space.constr_tester_2:create_index('I1', { parts = {'id'} }) - box.space.constr_tester_2:create_index('I2', { parts = {'table1_id'} }) + resp = cls.srv.admin(""" + box.schema.create_space( + 'constr_tester_1', { + format = { + { name = 'id', type = 'unsigned' }, + { name = 'payload', type = 'number' }, + } + }) + box.space.constr_tester_1:create_index('I1', { parts = {'id'} }) + + box.space.constr_tester_1:replace({1, 999}) + + box.schema.create_space( + 'constr_tester_2', { + format = { + { name = 'id', type = 'unsigned' }, + { name = 'table1_id', type = 'unsigned', + foreign_key = { fk_video = { space = 'constr_tester_1', field = 'id' } }, + }, + { name = 'payload', type = 'number' }, + } + }) + box.space.constr_tester_2:create_index('I1', { parts = {'id'} }) + box.space.constr_tester_2:create_index('I2', { parts = {'table1_id'} }) + + return true """) + assert_admin_success(resp) def setUp(self): # prevent a remote tarantool from clean our session @@ -603,10 +613,13 @@ def tearDownClass(cls): # We need to drop spaces with foreign keys with predetermined order, # otherwise remote server clean() will fail to clean up resources. if cls.srv.admin.tnt_version >= pkg_resources.parse_version('2.10.0'): - cls.srv.admin(""" - box.space.constr_tester_2:drop() - box.space.constr_tester_1:drop() + resp = cls.srv.admin(""" + box.space.constr_tester_2:drop() + box.space.constr_tester_1:drop() + + return true """) + assert_admin_success(resp) cls.con.close() cls.con_schema_disable.close() diff --git a/test/suites/test_socket_fd.py b/test/suites/test_socket_fd.py new file mode 100644 index 00000000..5cf94777 --- /dev/null +++ b/test/suites/test_socket_fd.py @@ -0,0 +1,200 @@ +""" +This module tests work with connection over socket fd. +""" +import os.path +# pylint: disable=missing-class-docstring,missing-function-docstring + +import socket +import sys +import unittest + +import tarantool +from .lib.skip import skip_or_run_box_session_new_tests +from .lib.tarantool_server import TarantoolServer, find_port +from .utils import assert_admin_success + + +def find_python(): + for _dir in os.environ["PATH"].split(os.pathsep): + exe = os.path.join(_dir, "python") + if os.access(exe, os.X_OK): + return os.path.abspath(exe) + raise RuntimeError("Can't find python executable in " + os.environ["PATH"]) + + +class TestSuiteSocketFD(unittest.TestCase): + EVAL_USER = "return box.session.user()" + + @classmethod + def setUpClass(cls): + print(' SOCKET FD '.center(70, '='), file=sys.stderr) + print('-' * 70, file=sys.stderr) + + cls.srv = TarantoolServer() + cls.srv.script = 'test/suites/box.lua' + cls.srv.start() + cls.tcp_port = find_port() + + # Start tcp server to test work with blocking sockets. + # pylint: disable=consider-using-f-string + resp = cls.srv.admin(""" + local socket = require('socket') + + box.cfg{} + box.schema.user.create('test', {password = 'test', if_not_exists = true}) + box.schema.user.grant('test', 'read,write,execute,create', 'universe', + nil, {if_not_exists = true}) + box.schema.user.grant('guest', 'execute', 'universe', + nil, {if_not_exists = true}) + + socket.tcp_server('0.0.0.0', %d, function(s) + if not s:nonblock(true) then + s:close() + return + end + box.session.new({ + type = 'binary', + fd = s:fd(), + user = 'test', + }) + s:detach() + end) + + box.schema.create_space('test', { + format = {{type='unsigned', name='id'}}, + if_not_exists = true, + }) + box.space.test:create_index('primary') + + return true + """ % cls.tcp_port) + assert_admin_success(resp) + + @skip_or_run_box_session_new_tests + def setUp(self): + # Prevent a remote tarantool from clean our session. + if self.srv.is_started(): + self.srv.touch_lock() + + def _get_tt_sock(self): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect((self.srv.host, self.tcp_port)) + return sock + + def test_01_incorrect_params(self): + cases = { + "host and socket_fd": { + "args": {"host": "123", "socket_fd": 3}, + "msg": "specifying both socket_fd and host/port is not allowed", + }, + "port and socket_fd": { + "args": {"port": 14, "socket_fd": 3}, + "msg": "specifying both socket_fd and host/port is not allowed", + }, + "empty": { + "args": {}, + "msg": r"host/port.* port.* or socket_fd", + }, + "only host": { + "args": {"host": "localhost"}, + "msg": "when specifying host, it is also necessary to specify port", + }, + } + + for name, case in cases.items(): + with self.subTest(msg=name): + with self.assertRaisesRegex(tarantool.Error, case["msg"]): + tarantool.Connection(**case["args"]) + + def test_02_socket_fd_connect(self): + sock = self._get_tt_sock() + conn = tarantool.connect(None, None, socket_fd=sock.fileno()) + sock.detach() + try: + self.assertSequenceEqual(conn.eval(self.EVAL_USER), ["test"]) + finally: + conn.close() + + def test_03_socket_fd_re_auth(self): + sock = self._get_tt_sock() + conn = tarantool.connect(None, None, socket_fd=sock.fileno(), user="guest") + sock.detach() + try: + self.assertSequenceEqual(conn.eval(self.EVAL_USER), ["guest"]) + finally: + conn.close() + + @unittest.skipIf(sys.platform.startswith("win"), + "Skip on Windows since it uses remote server") + def test_04_tarantool_made_socket(self): + python_exe = find_python() + cwd = os.getcwd() + side_script_path = os.path.join(cwd, "test", "suites", "sidecar.py") + + # pylint: disable=consider-using-f-string + ret_code, err = self.srv.admin(""" + local socket = require('socket') + local popen = require('popen') + local os = require('os') + local s1, s2 = socket.socketpair('AF_UNIX', 'SOCK_STREAM', 0) + + --[[ Tell sidecar which fd use to connect. --]] + os.setenv('SOCKET_FD', tostring(s2:fd())) + + --[[ Tell sidecar where find `tarantool` module. --]] + os.setenv('PYTHONPATH', (os.getenv('PYTHONPATH') or '') .. ':' .. '%s') + + box.session.new({ + type = 'binary', + fd = s1:fd(), + user = 'test', + }) + s1:detach() + + local ph, err = popen.new({'%s', '%s'}, { + stdout = popen.opts.PIPE, + stderr = popen.opts.PIPE, + inherit_fds = {s2:fd()}, + }) + + if err ~= nil then + return 1, err + end + + ph:wait() + + local status_code = ph:info().status.exit_code + local stderr = ph:read({stderr=true}):rstrip() + return status_code, stderr + """ % (cwd, python_exe, side_script_path)) + self.assertIsNone(err, err) + self.assertEqual(ret_code, 0) + + def test_05_socket_fd_pool(self): + sock = self._get_tt_sock() + pool = tarantool.ConnectionPool( + addrs=[{'host': None, 'port': None, 'socket_fd': sock.fileno()}] + ) + sock.detach() + try: + self.assertSequenceEqual(pool.eval(self.EVAL_USER, mode=tarantool.Mode.ANY), ["test"]) + finally: + pool.close() + + def test_06_socket_fd_mesh(self): + sock = self._get_tt_sock() + mesh = tarantool.MeshConnection( + host=None, + port=None, + socket_fd=sock.fileno() + ) + sock.detach() + try: + self.assertSequenceEqual(mesh.eval(self.EVAL_USER), ["test"]) + finally: + mesh.close() + + @classmethod + def tearDownClass(cls): + cls.srv.stop() + cls.srv.clean() diff --git a/test/suites/test_ssl.py b/test/suites/test_ssl.py index 9452e83b..b7d1da97 100644 --- a/test/suites/test_ssl.py +++ b/test/suites/test_ssl.py @@ -34,7 +34,7 @@ def is_test_ssl(): class SslTestCase: - # pylint: disable=too-few-public-methods,too-many-instance-attributes,too-many-arguments + # pylint: disable=too-few-public-methods,too-many-instance-attributes,too-many-arguments,too-many-positional-arguments def __init__(self, name="", diff --git a/test/suites/test_uuid.py b/test/suites/test_uuid.py index 0a9ef06b..90943fcb 100644 --- a/test/suites/test_uuid.py +++ b/test/suites/test_uuid.py @@ -7,6 +7,7 @@ import unittest import uuid +import pkg_resources import msgpack import tarantool @@ -15,6 +16,7 @@ from .lib.tarantool_server import TarantoolServer from .lib.skip import skip_or_run_uuid_test +from .utils import assert_admin_success class TestSuiteUUID(unittest.TestCase): @@ -27,26 +29,35 @@ def setUpClass(cls): cls.srv.start() cls.adm = cls.srv.admin - cls.adm(r""" + resp = cls.adm(""" _, uuid = pcall(require, 'uuid') - box.schema.space.create('test') + box.schema.space.create('test', {if_not_exists = true}) box.space['test']:create_index('primary', { type = 'tree', parts = {1, 'string'}, - unique = true}) + unique = true, + if_not_exists = true}) - pcall(function() - box.schema.space.create('test_pk') + box.schema.user.create('test', {password = 'test', if_not_exists = true}) + box.schema.user.grant('test', 'read,write,execute', 'universe', + nil, {if_not_exists = true}) + + return true + """) + assert_admin_success(resp) + + if cls.srv.admin.tnt_version >= pkg_resources.parse_version('2.4.1'): + resp = cls.adm(""" + box.schema.space.create('test_pk', {if_not_exists = true}) box.space['test_pk']:create_index('primary', { type = 'tree', parts = {1, 'uuid'}, - unique = true}) - end) - - box.schema.user.create('test', {password = 'test', if_not_exists = true}) - box.schema.user.grant('test', 'read,write,execute', 'universe') - """) + unique = true, + if_not_exists = true}) + return true + """) + assert_admin_success(resp) cls.con = tarantool.Connection(cls.srv.host, cls.srv.args['primary'], user='test', password='test') @@ -56,7 +67,11 @@ def setUp(self): if self.srv.is_started(): self.srv.touch_lock() - self.adm("box.space['test']:truncate()") + resp = self.adm(""" + box.space['test']:truncate() + return true + """) + assert_admin_success(resp) cases = { 'uuid_1': { diff --git a/test/suites/utils.py b/test/suites/utils.py new file mode 100644 index 00000000..65e68b4c --- /dev/null +++ b/test/suites/utils.py @@ -0,0 +1,16 @@ +""" +Various test utilities. +""" + + +def assert_admin_success(resp): + """ + Util to assert admin text request response. + It is expected that request last line is `return true`. + If something went wrong on executing, Tarantool throws an error + which would be a part of return values. + """ + + assert isinstance(resp, list), f'got unexpected resp type: {type(resp)}' + assert len(resp) > 0, 'got unexpected empty resp' + assert resp[0] is True, f'got unexpected resp: {resp}'