diff --git a/.codespellrc b/.codespellrc new file mode 100644 index 00000000..291bd5cf --- /dev/null +++ b/.codespellrc @@ -0,0 +1,3 @@ +[codespell] +skip = tarantool/msgpack_ext/types/timezones/timezones.py +ignore-words-list = ans,gost,ro,assertIn diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..4425b53d --- /dev/null +++ b/.flake8 @@ -0,0 +1,3 @@ +[flake8] +# Use pylint for lines length check +ignore=E501,W503 diff --git a/.github/workflows/packing.yml b/.github/workflows/packing.yml index 00ef82ab..70970387 100644 --- a/.github/workflows/packing.yml +++ b/.github/workflows/packing.yml @@ -6,6 +6,10 @@ on: pull_request_target: types: [labeled] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: pack_pip: # We want to run on external PRs, but not on our own internal @@ -16,14 +20,14 @@ jobs: if: (github.event_name == 'push') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false steps: - name: Clone the connector repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Checkout all tags for correct version computation. with: fetch-depth: 0 @@ -31,10 +35,12 @@ jobs: - name: Setup Python and basic packing tools uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.11' - name: Install tools for packing and verification - run: pip3 install wheel twine + run: | + pip3 install wheel twine + pip3 install --upgrade setuptools - name: Pack source and binary files run: make pip-dist @@ -43,7 +49,7 @@ jobs: run: make pip-dist-check - name: Archive pip artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4.4.0 with: name: pip_dist path: pip_dist @@ -61,30 +67,30 @@ jobs: if: (github.event_name == 'push') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false steps: - name: Clone the connector repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.11' - name: Remove connector source code run: python3 .github/scripts/remove_source_code.py - name: Install tarantool - uses: tarantool/setup-tarantool@v1 + uses: tarantool/setup-tarantool@v3 with: - tarantool-version: '2.10' + tarantool-version: '2.11' - name: Download pip package artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4.1.8 with: name: pip_dist path: pip_dist @@ -95,10 +101,13 @@ jobs: - name: Install test requirements run: pip3 install -r requirements-test.txt + # Installation of the specific CMake version is a hotfix for + # https://github.com/tarantool/checks/issues/64 - name: Install the crud module for testing purposes run: | curl -L https://tarantool.io/release/2/installer.sh | bash sudo apt install -y tt + pip3 install cmake==3.15.3 tt rocks install crud - name: Run tests @@ -123,18 +132,18 @@ jobs: steps: - name: Clone the connector repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.11' - name: Remove connector source code run: python3 .github/scripts/remove_source_code.py - name: Download pip package artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4.1.8 with: name: pip_dist path: pip_dist @@ -148,16 +157,16 @@ jobs: - name: Setup WSL for tarantool uses: Vampire/setup-wsl@v1 with: - distribution: Ubuntu-20.04 + distribution: Ubuntu-22.04 - name: Install tarantool - shell: wsl-bash_Ubuntu-20.04 {0} + shell: wsl-bash_Ubuntu-22.04 {0} run: | curl -L https://tarantool.io/release/2/installer.sh | bash -s sudo apt install -y tarantool tarantool-dev - name: Setup test tarantool instance - shell: wsl-bash_Ubuntu-20.04 {0} + shell: wsl-bash_Ubuntu-22.04 {0} run: | rm -f ./tarantool.pid ./tarantool.log TNT_PID=$(tarantool ./test/suites/lib/tarantool_python_ci.lua > tarantool.log 2>&1 & echo $!) @@ -172,7 +181,7 @@ jobs: - name: Stop test tarantool instance if: ${{ always() }} - shell: wsl-bash_Ubuntu-20.04 {0} + shell: wsl-bash_Ubuntu-22.04 {0} run: | cat tarantool.log || true kill $(cat tarantool.pid) || true @@ -184,25 +193,25 @@ jobs: - run_tests_pip_package_linux - run_tests_pip_package_windows - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false steps: - name: Clone the connector repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python and basic packing tools uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.11' - name: Install tools for package publishing run: pip3 install twine - name: Download pip package artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4.1.8 with: name: pip_dist path: pip_dist @@ -222,7 +231,7 @@ jobs: if: (github.event_name == 'push') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 container: image: ${{ matrix.target.os }}:${{ matrix.target.dist }} @@ -246,7 +255,7 @@ jobs: run: dnf install -y git - name: Clone the connector repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Checkout all tags for correct version computation. with: fetch-depth: 0 @@ -271,7 +280,7 @@ jobs: run: make rpm-dist-check - name: Archive RPM artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4.4.0 with: name: rpm_dist_${{ matrix.target.os }}_${{ matrix.target.dist }} path: rpm_dist @@ -289,7 +298,7 @@ jobs: if: (github.event_name == 'push') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 container: image: ${{ matrix.target.os }}:${{ matrix.target.dist }} @@ -308,7 +317,7 @@ jobs: steps: - name: Clone the connector repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python and test running tools # cmake rocks fail to install as expected without findutils: @@ -320,11 +329,11 @@ jobs: - name: Install tarantool run: | - curl -L https://tarantool.io/yeohchA/release/2/installer.sh | bash + curl -L https://tarantool.io/release/2/installer.sh | bash dnf install -y tarantool tarantool-devel - name: Download RPM artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4.1.8 with: name: rpm_dist_${{ matrix.target.os }}_${{ matrix.target.dist }} path: rpm_dist @@ -339,6 +348,7 @@ jobs: run: | curl -L https://tarantool.io/release/2/installer.sh | bash sudo dnf install -y tt + pip3 install cmake==3.15.3 tt rocks install crud - name: Run tests @@ -350,7 +360,7 @@ jobs: needs: - run_tests_rpm - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false @@ -366,13 +376,13 @@ jobs: steps: - name: Clone the connector repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install tools for package publishing run: sudo apt install -y curl make - name: Download RPM artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4.1.8 with: name: rpm_dist_${{ matrix.target.os }}_${{ matrix.target.dist }} path: rpm_dist @@ -400,21 +410,23 @@ jobs: if: (github.event_name == 'push') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false steps: - name: Clone the connector repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Checkout all tags for correct version computation with: fetch-depth: 0 - name: Install deb packing tools run: | + sudo apt update sudo apt install -y devscripts equivs + sudo apt install python3-setuptools python3-stdeb dh-python - name: Make changelog entry for non-release build if: startsWith(github.ref, 'refs/tags') != true @@ -432,7 +444,7 @@ jobs: run: make deb-dist-check - name: Archive deb artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4.4.0 with: name: deb_dist path: deb_dist @@ -448,7 +460,7 @@ jobs: if: (github.event_name == 'push') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 container: image: ${{ matrix.target.os }}:${{ matrix.target.dist }} @@ -458,18 +470,20 @@ jobs: matrix: target: + - os: debian + dist: bullseye # 11 + - os: debian + dist: bookworm # 12 - os: ubuntu dist: focal # 20.04 - os: ubuntu dist: jammy # 22.04 - - os: debian - dist: buster # 10 - - os: debian - dist: bullseye # 11 + - os: ubuntu + dist: noble # 24.04 steps: - name: Clone the connector repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Prepare apt run: apt update @@ -483,13 +497,13 @@ jobs: - name: Install tarantool ${{ matrix.tarantool }} run: | apt install -y curl - curl -L https://tarantool.io/yeohchA/release/2/installer.sh | bash + curl -L https://tarantool.io/release/2/installer.sh | bash apt install -y tarantool tarantool-dev env: DEBIAN_FRONTEND: noninteractive - name: Download deb artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4.1.8 with: name: deb_dist path: deb_dist @@ -499,17 +513,48 @@ jobs: env: DEBIAN_FRONTEND: noninteractive + # Usage of venv is mandatory starting with Debian 12 and Ubuntu 24.04. + - name: Create venv + run: | + apt install -y python3-venv + python3 -m venv .venv + - name: Install test requirements run: pip3 install -r requirements-test.txt + if: matrix.target.dist != 'bookworm' && matrix.target.dist != 'noble' + + - name: Install test requirements + run: | + . .venv/bin/activate + pip3 install -r requirements-test.txt + if: matrix.target.dist == 'bookworm' || matrix.target.dist == 'noble' - name: Install the crud module for testing purposes run: | curl -L https://tarantool.io/release/2/installer.sh | bash apt install -y tt tt rocks install crud + if: matrix.target.dist != 'bookworm' && matrix.target.dist != 'noble' + + - name: Install the crud module for testing purposes + run: | + . .venv/bin/activate + curl -L https://tarantool.io/release/3/installer.sh | bash + apt install -y tt + tt rocks install crud + if: matrix.target.dist == 'bookworm' || matrix.target.dist == 'noble' - name: Run tests run: make test-pure-install + if: matrix.target.dist != 'bookworm' && matrix.target.dist != 'noble' + + - name: Run tests + run: | + . .venv/bin/activate + export PYTHONPATH=$PYTHONPATH:/usr/lib/python3.11:/usr/lib/python3.12:/usr/bin:/usr/lib/python3/dist-packages + export PATH=$PATH:/usr/lib/python3/dist-packages + make test-pure-install + if: matrix.target.dist == 'bookworm' || matrix.target.dist == 'noble' publish_deb: if: startsWith(github.ref, 'refs/tags') @@ -517,7 +562,7 @@ jobs: needs: - run_tests_deb - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false @@ -528,20 +573,24 @@ jobs: dist: focal # 20.04 - os: ubuntu dist: jammy # 22.04 + - os: ubuntu + dist: noble # 24.04 - os: debian dist: buster # 10 - os: debian dist: bullseye # 11 + - os: debian + dist: bookworm # 12 steps: - name: Clone the connector repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install tools for package publishing run: sudo apt install -y curl make - name: Download deb artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4.1.8 with: name: deb_dist path: deb_dist diff --git a/.github/workflows/reusable_testing.yml b/.github/workflows/reusable_testing.yml index 1e66af13..79f9f2a4 100644 --- a/.github/workflows/reusable_testing.yml +++ b/.github/workflows/reusable_testing.yml @@ -11,15 +11,15 @@ on: jobs: run_tests: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - name: Clone the tarantool-python connector - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: repository: ${{ github.repository_owner }}/tarantool-python - name: Download the tarantool build artifact - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4.1.8 with: name: ${{ inputs.artifact_name }} @@ -29,9 +29,9 @@ jobs: run: sudo dpkg -i tarantool*.deb - name: Setup python3 for tests - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: - python-version: 3.7 + python-version: '3.11' - name: Install connector requirements run: pip3 install -r requirements.txt @@ -43,6 +43,7 @@ jobs: run: | curl -L https://tarantool.io/release/2/installer.sh | bash sudo apt install -y tt + pip3 install cmake==3.15.3 tt rocks install crud - run: make test diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 6c18acd0..8f94a4e6 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -6,6 +6,10 @@ on: pull_request_target: types: [labeled] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: run_tests_ce_linux: # We want to run on external PRs, but not on our own internal @@ -17,54 +21,88 @@ jobs: (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: tarantool: - - '1.10' - - '2.8' - - '2.10' + - '2.11' + - '3.3' + - 'master' python: - - '3.6' - - '3.7' - - '3.8' - '3.9' - '3.10' + - '3.11' msgpack-deps: # latest msgpack will be installed as a part of requirements.txt - '' # Adding too many elements to three-dimentional matrix results in - # too many test cases. It causes GitHub webpages to fail with + # too many test cases. It causes GitHub webpages to fail with # "This page is taking too long to load." error. Thus we use # pairwise testing. include: - - tarantool: '2.8' - python: '3.10' - msgpack-deps: 'msgpack-python==0.4.0' - - tarantool: '2.8' - python: '3.10' + - tarantool: '3.3' + python: '3.11' msgpack-deps: 'msgpack==0.5.0' - - tarantool: '2.8' - python: '3.10' + - tarantool: '3.3' + python: '3.11' msgpack-deps: 'msgpack==0.6.2' - - tarantool: '2.8' - python: '3.10' + - tarantool: '3.3' + python: '3.11' msgpack-deps: 'msgpack==1.0.4' steps: - name: Clone the connector - uses: actions/checkout@v2 + uses: actions/checkout@v4 + + - name: Setup tt + run: | + curl -L https://tarantool.io/release/3/installer.sh | sudo bash + sudo apt install -y tt + tt version + tt init + + # Installation of the specific CMake version is a hotfix for + # https://github.com/tarantool/checks/issues/64 + - name: Install old CMake + run: pip3 install cmake==3.15.3 - name: Install tarantool ${{ matrix.tarantool }} - uses: tarantool/setup-tarantool@v1 + if: matrix.tarantool != 'master' + uses: tarantool/setup-tarantool@v3 with: tarantool-version: ${{ matrix.tarantool }} + - name: Get Tarantool master latest commit + if: matrix.tarantool == 'master' + run: | + commit_hash=$(git ls-remote https://github.com/tarantool/tarantool.git --branch master | head -c 8) + echo "LATEST_COMMIT=${commit_hash}" >> $GITHUB_ENV + shell: bash + + - name: Cache Tarantool master + if: matrix.tarantool == 'master' + id: cache-latest + uses: actions/cache@v3 + with: + path: | + ${{ github.workspace }}/bin + ${{ github.workspace }}/include + key: cache-latest-${{ env.LATEST_COMMIT }} + + - name: Setup Tarantool master + if: matrix.tarantool == 'master' && steps.cache-latest.outputs.cache-hit != 'true' + run: | + tt install tarantool master + + - name: Add Tarantool master to PATH + if: matrix.tarantool == 'master' + run: echo "${GITHUB_WORKSPACE}/bin" >> $GITHUB_PATH + - name: Setup Python for tests - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} @@ -86,10 +124,7 @@ jobs: run: pip3 install -r requirements-test.txt - name: Install the crud module for testing purposes - run: | - curl -L https://tarantool.io/release/2/installer.sh | bash - sudo apt install -y tt - tt rocks install crud + run: tt rocks install crud - name: Run tests run: make test @@ -107,34 +142,30 @@ jobs: github.event.pull_request.head.repo.full_name != github.repository && github.event.label.name == 'full-ci') - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: tarantool: - - bundle: 'bundle-1.10.11-0-gf0b0e7ecf-r470' - path: '' - - bundle: 'bundle-2.8.3-21-g7d35cd2be-r470' - path: '' - - bundle: 'bundle-2.10.0-1-gfa775b383-r486-linux-x86_64' - path: '' - - bundle: 'sdk-gc64-2.11.0-entrypoint-113-g803baaffe-r529.linux.x86_64' - path: 'dev/linux/x86_64/master/' - python: ['3.6', '3.7', '3.8', '3.9', '3.10'] + - bundle: 'sdk-gc64-2.11.0-0-r563.linux.x86_64' + path: 'release/linux/x86_64/2.11/' + - bundle: 'sdk-gc64-3.3.1-0-r55.linux.x86_64' + path: 'release/linux/x86_64/3.3/' + python: ['3.9', '3.11'] steps: - name: Clone the connector # `ref` as merge request is needed for pull_request_target because this # target runs in the context of the base commit of the pull request. - uses: actions/checkout@v2 + uses: actions/checkout@v4 if: github.event_name == 'pull_request_target' with: ref: refs/pull/${{ github.event.pull_request.number }}/merge - name: Clone the connector if: github.event_name != 'pull_request_target' - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Install Tarantool EE SDK run: | @@ -144,7 +175,7 @@ jobs: rm -f ${ARCHIVE_NAME} - name: Setup Python for tests - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} @@ -159,8 +190,9 @@ jobs: # See more here: https://github.com/tarantool/tt/issues/282 run: | source tarantool-enterprise/env.sh - curl -L https://tarantool.io/release/2/installer.sh | bash + curl -L https://tarantool.io/release/3/installer.sh | bash sudo apt install -y tt + pip3 install cmake==3.15.3 tt rocks install crud TARANTOOL_DIR=$PWD/tarantool-enterprise - name: Run tests @@ -168,8 +200,8 @@ jobs: source tarantool-enterprise/env.sh make test env: - TEST_TNT_SSL: ${{ matrix.tarantool.bundle == 'bundle-2.10.0-1-gfa775b383-r486-linux-x86_64' || - matrix.tarantool.bundle == 'sdk-gc64-2.11.0-entrypoint-113-g803baaffe-r529.linux.x86_64'}} + TEST_TNT_SSL: ${{ matrix.tarantool.bundle == 'sdk-gc64-2.10.7-0-r563.linux.x86_64' || + matrix.tarantool.bundle == 'sdk-gc64-2.11.0-0-r563.linux.x86_64'}} run_tests_pip_branch_install_linux: # We want to run on external PRs, but not on our own internal @@ -180,31 +212,28 @@ jobs: if: (github.event_name == 'push') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: tarantool: - - '2.10' + - '2.11' + - '3.3' python: - - '3.6' - - '3.7' - - '3.8' - - '3.9' - - '3.10' + - '3.11' steps: - name: Clone the connector repo - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Install tarantool ${{ matrix.tarantool }} - uses: tarantool/setup-tarantool@v1 + uses: tarantool/setup-tarantool@v3 with: tarantool-version: ${{ matrix.tarantool }} - name: Setup Python for tests - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} @@ -212,15 +241,21 @@ jobs: run: python3 .github/scripts/remove_source_code.py - name: Install the package with pip - run: pip3 install git+$GITHUB_SERVER_URL/$GITHUB_REPOSITORY@$GITHUB_REF_NAME + run: pip3 install git+$GITHUB_SERVER_URL/$GITHUB_REPOSITORY@$GITHUB_REF + + # Installation of the specific CMake version is a hotfix for + # https://github.com/tarantool/checks/issues/64 + - name: Install old CMake + run: pip3 install cmake==3.15.3 - name: Install test requirements run: pip3 install -r requirements-test.txt - name: Install the crud module for testing purposes run: | - curl -L https://tarantool.io/release/2/installer.sh | bash + curl -L https://tarantool.io/release/3/installer.sh | bash sudo apt install -y tt + pip3 install cmake==3.15.3 tt rocks install crud - name: Run tests @@ -243,19 +278,17 @@ jobs: matrix: # Use reduced test matrix cause Windows pipelines are long. tarantool: - - '1.10' - - '2.8' - - '2.10.0.g0a5ce0b9c-1' + # https://github.com/tarantool/tarantool-python/issues/331 + - '2.11.0.g247a9a418-1' python: - - '3.6' - - '3.10' + - '3.11' steps: - name: Clone the connector - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Setup Python for tests - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} @@ -266,26 +299,18 @@ jobs: run: pip3 install -r requirements-test.txt - name: Setup WSL for tarantool - uses: Vampire/setup-wsl@v1 + uses: Vampire/setup-wsl@v2 with: - distribution: Ubuntu-20.04 - - - name: Install tarantool ${{ matrix.tarantool }} for WSL (2.8 and older) - if: (matrix.tarantool == '1.10') || (matrix.tarantool == '2.8') - shell: wsl-bash_Ubuntu-20.04 {0} - run: | - curl -L https://tarantool.io/installer.sh | VER=${{ matrix.tarantool }} bash -s -- --type "release" - sudo apt install -y tarantool tarantool-dev + distribution: Ubuntu-22.04 - name: Install tarantool ${{ matrix.tarantool }} for WSL (2.10 and newer) - if: (matrix.tarantool != '1.10') && (matrix.tarantool != '2.8') - shell: wsl-bash_Ubuntu-20.04 {0} + shell: wsl-bash_Ubuntu-22.04 {0} run: | curl -L https://tarantool.io/release/2/installer.sh | bash -s sudo apt install -y tarantool=${{ matrix.tarantool }} tarantool-dev=${{ matrix.tarantool }} - name: Setup test tarantool instance - shell: wsl-bash_Ubuntu-20.04 {0} + shell: wsl-bash_Ubuntu-22.04 {0} run: | rm -f ./tarantool.pid ./tarantool.log TNT_PID=$(tarantool ./test/suites/lib/tarantool_python_ci.lua > tarantool.log 2>&1 & echo $!) @@ -300,7 +325,7 @@ jobs: - name: Stop test tarantool instance if: ${{ always() }} - shell: wsl-bash_Ubuntu-20.04 {0} + shell: wsl-bash_Ubuntu-22.04 {0} run: | cat tarantool.log || true kill $(cat tarantool.pid) || true @@ -323,16 +348,16 @@ jobs: matrix: # Use reduced test matrix cause Windows pipelines are long. tarantool: - - '2.10.0.g0a5ce0b9c-1' + # https://github.com/tarantool/tarantool-python/issues/331 + - '2.11.0.g247a9a418-1' python: - - '3.6' - - '3.10' + - '3.11' steps: - name: Clone the connector repo - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Setup Python for tests - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} @@ -340,24 +365,24 @@ jobs: run: python3 .github/scripts/remove_source_code.py - name: Install the package with pip - run: pip3 install git+$env:GITHUB_SERVER_URL/$env:GITHUB_REPOSITORY@$env:GITHUB_REF_NAME + run: pip3 install git+$env:GITHUB_SERVER_URL/$env:GITHUB_REPOSITORY@$env:GITHUB_REF - name: Install test requirements run: pip3 install -r requirements-test.txt - name: Setup WSL for tarantool - uses: Vampire/setup-wsl@v1 + uses: Vampire/setup-wsl@v2 with: - distribution: Ubuntu-20.04 + distribution: Ubuntu-22.04 - name: Install tarantool ${{ matrix.tarantool }} for WSL - shell: wsl-bash_Ubuntu-20.04 {0} + shell: wsl-bash_Ubuntu-22.04 {0} run: | curl -L https://tarantool.io/release/2/installer.sh | bash -s sudo apt install -y tarantool=${{ matrix.tarantool }} tarantool-dev=${{ matrix.tarantool }} - name: Setup test tarantool instance - shell: wsl-bash_Ubuntu-20.04 {0} + shell: wsl-bash_Ubuntu-22.04 {0} run: | rm -f ./tarantool.pid ./tarantool.log TNT_PID=$(tarantool ./test/suites/lib/tarantool_python_ci.lua > tarantool.log 2>&1 & echo $!) @@ -372,7 +397,7 @@ jobs: - name: Stop test tarantool instance if: ${{ always() }} - shell: wsl-bash_Ubuntu-20.04 {0} + shell: wsl-bash_Ubuntu-22.04 {0} run: | cat tarantool.log || true kill $(cat tarantool.pid) || true diff --git a/.gitignore b/.gitignore index 9d3a8412..53a8daa2 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,5 @@ test/data/*.key !test/data/localhost.enc.key test/data/*.pem test/data/*.srl + +.rocks diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 00000000..385d67ad --- /dev/null +++ b/.pylintrc @@ -0,0 +1,13 @@ +[BASIC] + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_,ok,t,tz + +[FORMAT] +# Allow links in docstings, allow tables +ignore-long-lines=^(?:\s*(# )?(?:\.\.\s.+?:)?\s*?)|(\s\+.+\+)|(\s\|.+\|)$ + +[MESSAGES CONTROL] +# Ignore unknown options to support per-entity ignores for newest warnings/errors +# which are not supported for older versions. +disable=unknown-option-value diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 9375a9f6..bc990993 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -13,3 +13,6 @@ python: - method: pip path: . - requirements: docs/requirements.txt + +sphinx: + configuration: docs/source/conf.py diff --git a/AUTHORS b/AUTHORS index f77bd18d..92787ea1 100644 --- a/AUTHORS +++ b/AUTHORS @@ -35,3 +35,4 @@ Sergey Bronnikov Yaroslav Lobankov Georgy Moiseev Oleg Jukovec +Ilya Grishnov diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fb88ae8..8edd47e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,11 +6,77 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Changed +- Drop Python 3.6 support (PR #327). + +## 1.2.0 - 2024-03-27 + +### Added +- The ability to connect to the Tarantool using an existing socket fd (#304). + +## 1.1.2 - 2023-09-20 + +### Fixed +- Exception rethrow in crud API (PR #310). +- Work with timestamps larger than year 2038 for some platforms (like Windows) (PR #311). + It covers + - building new `tarantool.Datetime` objects from timestamp, + - parsing datetime objects received from Tarantool. + +## 1.1.1 - 2023-07-19 + +### Changed +- Validate `tarantool.Interval` limits with the same rules as in Tarantool (PR #302). + +### Fixed +- `tarantool.Interval` arithmetic with weeks (PR #302). +- `tarantool.Interval` weeks display in `str()` and `repr()` (PR #302). + +## 1.1.0 - 2023-06-30 + ### Added +- Allow to require specific server protocol version and features (#267). ### Changed +- Drop `msgpack-python` support. Use `msgpack` instead. + +### Fixed +- Parsing of E-notation Tarantool decimals with positive exponent (PR #298). + +## 1.0.0 - 2023-04-17 + +### Changed +- **Breaking**: Allow only named `on_push` and `on_push_ctx` for `insert` and `replace`. +- Migrate to built-in `Warning` instead of a custom one. +- Migrate to built-in `RecursionError` instead of a custom one. +- Collect full exception traceback. +- Rework `tarantool.Datetime` implementation to use built-in + `datetime.datetime`. External changes are as follows. Some of them + are **breaking**. + - Package no longer depends on `pandas` (#290). + - `__repr__` has been changed. + - Input arguments are validated with `datetime.datetime` rules. + - Class is no longer expected to throw `pandas.Timestamp` + exceptions. `datetime.datetime` exceptions will + be thrown instead of them. + - Drop the support of `__eq__` operator for `pandas.Timestamp`. +- **Breaking**: Remove `join` and `subscribe` connection methods. + +## 0.12.1 - 2023-02-28 + +### Changed +- Discovery iproto features only for Tarantools since version 2.10.0 (#283). + +### Fixed +- Schema fetch for spaces with foreign keys (#282). + +## 0.12.0 - 2023-02-13 + +### Added +- Support `fetch_schema` parameter for a connection (#219). ### Fixed +- Error code on socket error (#279). ## 0.11.0 - 2022-12-31 @@ -160,7 +226,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Support pandas way to build datetime from timestamp (PR #252). `timestamp_since_utc_epoch` is a parameter to set timestamp - convertion behavior for timezone-aware datetimes. + conversion behavior for timezone-aware datetimes. If ``False`` (default), behaves similar to Tarantool `datetime.new()`: @@ -180,7 +246,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Thus, if ``False``, datetime is computed from timestamp since epoch and then timezone is applied without any - convertion. In that case, `dt.timestamp` won't be equal to + conversion. In that case, `dt.timestamp` won't be equal to initialization `timestamp` for all timezones with non-zero offset. If ``True``, behaves similar to `pandas.Timestamp`: @@ -344,7 +410,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Various improvements and fixes in README (PR #210, PR #215). ### Fixed -- json.dumps compatibility with Python 2 (PR #186). +- json.dumps compatibility with Python 2 (PR #186). - Unix socket support in mesh_connection (PR #189, #111). - Various fixes in tests (PR #189, #111, PR #195, #194). diff --git a/Makefile b/Makefile index 1ab6441b..d3919795 100644 --- a/Makefile +++ b/Makefile @@ -3,8 +3,17 @@ install: pip3 install --editable . +PYTHON_FILES=tarantool test setup.py docs/source/conf.py +TEXT_FILES=README.rst CHANGELOG.md docs/source/*.rst +.PHONY: lint +lint: + python3 -m pylint --recursive=y $(PYTHON_FILES) + python3 -m flake8 $(PYTHON_FILES) + codespell $(PYTHON_FILES) $(TEXT_FILES) + + .PHONY: test -test: +test: lint python3 setup.py test .PHONY: test-pure-install diff --git a/README.rst b/README.rst index 41aa6a62..7e95ec3a 100644 --- a/README.rst +++ b/README.rst @@ -86,7 +86,7 @@ You can also install the development version of the package using ``pip``. What is Tarantool? ------------------ -`Tarantool`_ is an in-memory computing platform originally designed by +`Tarantool`_ is an in-memory computing platform originally designed by `VK`_ and released under the terms of `BSD license`_. Features diff --git a/debian/changelog b/debian/changelog index 0c13aaea..07127a11 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,140 @@ +python3-tarantool (1.2.0-0) unstable; urgency=medium + + ## Overview + + This release introduces socket fd connect support. + + ## Added + + - The ability to connect to the Tarantool using an existing socket fd (#304). + + -- Georgy Moiseev Wed, 27 Mar 2024 11:00:00 +0300 + +python3-tarantool (1.1.2-0) unstable; urgency=medium + + ## Overview + + This release introduce several bugfixes and behavior improvements. + + ## Fixed + + - Exception rethrow in crud API (PR #310). + - Work with timestamps larger than year 2038 for some platforms (like Windows) (PR #311). + It covers + - building new `tarantool.Datetime` objects from timestamp, + - parsing datetime objects received from Tarantool. + + -- Georgy Moiseev Wed, 20 Sep 2023 10:00:00 +0300 + +python3-tarantool (1.1.1-0) unstable; urgency=medium + + ## Overview + + This release introduces various datetime interval fixes and quality of life + improvements. + + ## Breaking changes + - Forbid to create datetime intervals out of Tarantool limits (PR #302). + + ## Changed + - Validate `tarantool.Interval` limits with the same rules as in Tarantool (PR #302). + + ## Fixed + - `tarantool.Interval` arithmetic with weeks (PR #302). + - `tarantool.Interval` weeks display in `str()` and `repr()` (PR #302). + + -- Georgy Moiseev Wed, 19 Jul 2023 18:00:00 +0300 + +python3-tarantool (1.1.0-0) unstable; urgency=medium + + ## Overview + + This release introduces API to request server protocol version and + feature, as well as introduce decimal bugfix. + + ## Breaking changes + + - Drop `msgpack-python` support. (Package not supported since 2019.) + Use `msgpack` instead. + + ## Added + - Allow to require specific server protocol version and features (#267). + + ## Fixed + - Parsing of E-notation Tarantool decimals with positive exponent (PR #298). + + -- Georgy Moiseev Fri, 30 Jun 2023 10:00:00 +0300 + +python3-tarantool (1.0.0-0) unstable; urgency=medium + + ## Overview + + This release introduces several minor behavior changes + to make API more consistent. + + Starting from this release, connector no longer depends on `pandas`. + + ## Breaking changes + + - Allow only named `on_push` and `on_push_ctx` for `insert` and `replace`. + - `tarantool.Datetime` `__repr__` has been changed. + - `tarantool.Datetime` input arguments are validated with `datetime.datetime` rules. + - `tarantool.Datetime` is no longer expected to throw `pandas.Timestamp` + exceptions. `datetime.datetime` exceptions will be thrown instead of them. + - Drop the support of `__eq__` operator of `tarantool.Datetime` for `pandas.Timestamp`. + - Remove `join` and `subscribe` connection methods. + + ## Changes + + - Migrate to built-in `Warning` instead of a custom one. + - Migrate to built-in `RecursionError` instead of a custom one. + - Collect full exception traceback. + - Package no longer depends on `pandas` (#290). + + ## Infrastructure + + - Lint the code with `pylint`, `flake8` and `codespell`. + + -- Georgy Moiseev Mon, 17 Apr 2023 13:00:00 +0300 + +python3-tarantool (0.12.1-0) unstable; urgency=medium + + ## Overview + + This release introduces several bugfixes and behavior improvements. + + ## Breaking changes + + This release should not break any existing behavior. + + ## Bugfixes + - Discovery iproto features only for Tarantools since version 2.10.0 (#283). + - Schema fetch for spaces with foreign keys (#282). + + -- Georgy Moiseev Tue, 28 Feb 2023 10:20:48 +0300 + +python3-tarantool (0.12.0-0) unstable; urgency=medium + + ## Overview + + This release introduces the support of `fetch_schema` connection + option to disable schema fetch and various fixes. + + ## Breaking changes + + This release should not break any existing behavior. + + ## New features + - `fetch_schema` parameter for a connection (#219). + + ## Bugfixes + - Error code on socket error (#279). + + ## Thanks + We want to thank @bekhzod91 for a bugfix contribution. + + -- Georgy Moiseev Mon, 13 Feb 2023 11:43:30 +0300 + python3-tarantool (0.11.0-0) unstable; urgency=medium ## Overview @@ -18,7 +155,7 @@ python3-tarantool (0.11.0-0) unstable; urgency=medium - Support specifying authentication method with `auth_type` and Tarantool EE `pap-sha256` authentication method (#269). - -- Georgy.moiseev Sat, 31 Dec 2022 02:09:03 +0300 + -- Georgy Moiseev Sat, 31 Dec 2022 02:09:03 +0300 python3-tarantool (0.10.0-0) unstable; urgency=medium @@ -223,7 +360,7 @@ python3-tarantool (0.10.0-0) unstable; urgency=medium - Pack and publish pip, RPM and deb packages with GitHub Actions (#164, #198). - Publish on readthedocs with CI/CD (including PRs) (#67). - -- Georgy.moiseev Wed, 09 Nov 2022 13:14:20 +0300 + -- Georgy Moiseev Wed, 09 Nov 2022 13:14:20 +0300 tarantool-python (0.9.0-0) unstable; urgency=medium ## Overview @@ -252,7 +389,7 @@ tarantool-python (0.9.0-0) unstable; urgency=medium transport="ssl", ssl_ca_file=client_ca_file) ``` - + If the server authenticates clients using certificates issued by given CA, you must provide private SSL key file with `ssl_key_file` parameter and SSL certificate file with `ssl_cert_file` parameter. @@ -313,7 +450,7 @@ tarantool-python (0.9.0-0) unstable; urgency=medium ``` See [Tarantool Enterprise Edition manual](https://www.tarantool.io/en/enterprise_doc/security/#enterprise-iproto-encryption) - for details. + for details. ## Breaking changes diff --git a/debian/control b/debian/control index dc7869fc..a5cfe113 100644 --- a/debian/control +++ b/debian/control @@ -4,7 +4,7 @@ Section: python Priority: optional # See https://github.com/astraw/stdeb/issues/175 for dependencies Build-Depends: python3, python3-dev, python3-pip, python3-setuptools, - python3-distutils, python3-wheel, python3-stdeb, dh-python, + python3-wheel, python3-stdeb, dh-python, debhelper (>= 10) Standards-Version: 3.9.1 Homepage: https://github.com/tarantool/tarantool-python diff --git a/debian/rules b/debian/rules index 658c3ceb..df1413c3 100755 --- a/debian/rules +++ b/debian/rules @@ -14,7 +14,11 @@ override_dh_auto_build: python3 setup.py build --force override_dh_auto_install: - python3 setup.py install --force --root=debian/python3-tarantool --no-compile -O0 --install-layout=deb --prefix=/usr + python3 setup.py install --force --root=debian/python3-tarantool --no-compile -O0 --install-layout=deb --prefix=/usr override_dh_python2: dh_python2 --no-guessing-versions + +# Force `xz` compression for older system with dpkg version < 1.15.6 +override_dh_builddeb: + dh_builddeb -- -Zxz diff --git a/docs/requirements.txt b/docs/requirements.txt index 964c9f18..a5b9de39 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,3 @@ sphinx==5.2.1 sphinx-paramlinks==0.5.4 -sphinx-favicon==0.2 +sphinx_favicon==1.0.1 diff --git a/docs/source/conf.py b/docs/source/conf.py index 4738e61d..953c2034 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,15 +1,19 @@ -# Tarantool python client library documentation build configuration file, created by -# sphinx-quickstart on Tue Nov 29 06:29:57 2011. -# -# This file is execfile()d with the current directory set to its containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -import sys, os +""" +Tarantool python client library documentation build configuration file, created by +sphinx-quickstart on Tue Nov 29 06:29:57 2011. + +This file is execfile()d with the current directory set to its containing dir. + +Note that not all possible configuration values are present in this +autogenerated file. + +All configuration values have a default; values that are commented out +serve to show the default. +""" +# pylint: disable=invalid-name,redefined-builtin,duplicate-code +# flake8: noqa: E265 + +import tarantool # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -25,7 +29,7 @@ # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.ifconfig', 'sphinx_paramlinks', - 'sphinx-favicon',] + 'sphinx_favicon',] autodoc_default_options = { 'members': True, @@ -45,15 +49,13 @@ master_doc = 'index' # General information about the project. -project = u'Tarantool python client library' -copyright = u'2011-2022, tarantool-python AUTHORS' +project = 'Tarantool python client library' +copyright = '2011-2022, tarantool-python AUTHORS' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. -import tarantool - # The short X.Y version. version = tarantool.__version__ # The full version, including alpha/beta/rc tags. @@ -126,7 +128,7 @@ # pixels large. #html_favicon = None -# Set up favicons with sphinx-favicon. +# Set up favicons with sphinx_favicon. favicons = [ { "rel": "icon", @@ -279,8 +281,8 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'Tarantoolpythonclientlibrary.tex', u'Tarantool python client library Documentation', - u'tarantool-python AUTHORS', 'manual'), + ('index', 'Tarantoolpythonclientlibrary.tex', 'Tarantool python client library Documentation', + 'tarantool-python AUTHORS', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -309,8 +311,8 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('index', 'tarantoolpythonclientlibrary', u'Tarantool python client library Documentation', - [u'tarantool-python AUTHORS'], 1) + ('index', 'tarantoolpythonclientlibrary', 'Tarantool python client library Documentation', + ['tarantool-python AUTHORS'], 1) ] # If true, show URL addresses after external links. @@ -323,8 +325,8 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'Tarantoolpythonclientlibrary', u'Tarantool python client library Documentation', - u'tarantool-python AUTHORS', 'Tarantoolpythonclientlibrary', 'One line description of project.', + ('index', 'Tarantoolpythonclientlibrary', 'Tarantool python client library Documentation', + 'tarantool-python AUTHORS', 'Tarantoolpythonclientlibrary', 'One line description of project.', 'Miscellaneous'), ] @@ -342,7 +344,6 @@ intersphinx_mapping = { 'python': ('http://docs.python.org/', None), 'msgpack': ('https://msgpack-python.readthedocs.io/en/latest/', None), - 'pandas': ('https://pandas.pydata.org/docs/', None), 'pytz': ('https://pytz.sourceforge.net/', None), } diff --git a/docs/source/index.rst b/docs/source/index.rst index b2d31cd7..60a264fe 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -5,7 +5,7 @@ Python client library for Tarantool :Version: |version| -`Tarantool`_ is an in-memory computing platform originally designed by +`Tarantool`_ is an in-memory computing platform originally designed by `VK`_ and released under the terms of `BSD license`_. Install Tarantool Python connector with ``pip`` (`PyPI`_ page): @@ -78,7 +78,7 @@ API Reference .. Indices and tables .. ================== -.. +.. .. * :ref:`genindex` .. * :ref:`modindex` .. * :ref:`search` diff --git a/docs/source/quick-start.rst b/docs/source/quick-start.rst index 27633012..20dec4db 100644 --- a/docs/source/quick-start.rst +++ b/docs/source/quick-start.rst @@ -56,7 +56,7 @@ Throws an error if there is already a tuple with the same primary key. ... conn.insert('demo', ('BBBB', 'Bravo')) ... except Exception as exc: ... print(exc) - ... + ... (3, 'Duplicate key exists in unique index "pk" in space "demo" with old tuple - ["BBBB", "Bravo"] and new tuple - ["BBBB", "Bravo"]') Replace @@ -125,7 +125,7 @@ Creating a space instance An instance of :class:`~tarantool.space.Space` is a named object to access the key space. -Create a ``demo`` object that will be used to access the space +Create a ``demo`` object that will be used to access the space with id ``'demo'``: .. code-block:: python @@ -205,26 +205,26 @@ read-write and read-only pool instances: Receiving out-of-band messages ---------------------------------- -Receiving out-of-band messages from a server that uses box.session.push -call is supported for methods: :meth:`~tarantool.Connection.call`, -:meth:`~tarantool.Connection.eval`, :meth:`~tarantool.Connection.select`, -:meth:`~tarantool.Connection.insert`, :meth:`~tarantool.Connection.replace`, -:meth:`~tarantool.Connection.update`, :meth:`~tarantool.Connection.upsert`, +Receiving out-of-band messages from a server that uses box.session.push +call is supported for methods: :meth:`~tarantool.Connection.call`, +:meth:`~tarantool.Connection.eval`, :meth:`~tarantool.Connection.select`, +:meth:`~tarantool.Connection.insert`, :meth:`~tarantool.Connection.replace`, +:meth:`~tarantool.Connection.update`, :meth:`~tarantool.Connection.upsert`, :meth:`~tarantool.Connection.delete`. -To work with out-of-band messages, 2 optional arguments are used in +To work with out-of-band messages, 2 optional arguments are used in the methods listed above: * `on_push` - callback, launched with the received data for each out-of-band message. Two arguments for this callback are expected: - + * the first is the received from an out-of-band message data. * the second is `on_push_ctx`, variable for working with callback context (for example, recording the result or pass data to callback). * `on_push_ctx` - result of the `on_push` work can be written to this variable, or through this variable you can pass data to `on_push` callback. -Below is an example of the proposed API with method :meth:`~tarantool.Connection.call` -and :meth:`~tarantool.Connection.insert`. In the described example, before the end -of the :meth:`~tarantool.Connection.call` and :meth:`~tarantool.Connection.insert`, +Below is an example of the proposed API with method :meth:`~tarantool.Connection.call` +and :meth:`~tarantool.Connection.insert`. In the described example, before the end +of the :meth:`~tarantool.Connection.call` and :meth:`~tarantool.Connection.insert`, out-of-band messages are processed via specified callback. In the example below, two shells are used, in the first we will configure the server: @@ -249,7 +249,7 @@ In the example below, two shells are used, in the first we will configure the se return x end -In the second shell, we will execute a :meth:`~tarantool.Connection.call` +In the second shell, we will execute a :meth:`~tarantool.Connection.call` with receiving out-of-band messages from the server: .. code-block:: python @@ -266,11 +266,11 @@ with receiving out-of-band messages from the server: conn = tarantool.Connection(port=3301) res = conn.call( 'server_function', - on_push=callback, + on_push=callback, on_push_ctx=callback_res ) - # receiving out-of-band messages, + # receiving out-of-band messages, # the conn.call is not finished yet. >>> run callback with data: [[1, 0]] @@ -285,7 +285,7 @@ with receiving out-of-band messages from the server: print(callback_res) >>> [[[1, 1]], [[2, 1]], [[3, 1]]] -Let's go back to the first shell with the server and +Let's go back to the first shell with the server and create a space and a trigger for it: .. code-block:: lua @@ -315,7 +315,7 @@ create a space and a trigger for it: on_replace_callback ) -Now, in the second shell, we will execute an :meth:`~tarantool.ConnectionPool.insert` +Now, in the second shell, we will execute an :meth:`~tarantool.ConnectionPool.insert` with out-of-band message processing: .. code-block:: python @@ -333,7 +333,7 @@ with out-of-band message processing: on_push_ctx=callback_res, ) - # receiving out-of-band messages, + # receiving out-of-band messages, # the conn_pool.insert is not finished yet. >>> run callback with data: [[100, 0]] @@ -352,26 +352,26 @@ with out-of-band message processing: Interaction with the crud module ---------------------------------- -Through the :class:`~tarantool.Connection` object, you can access +Through the :class:`~tarantool.Connection` object, you can access `crud module `_ methods: .. code-block:: python >>> import tarantool >>> from tarantool.error import CrudModuleError, CrudModuleManyError, DatabaseError - >>> conn = tarantool.Connection(host='localhost',port=3301) + >>> conn = tarantool.Connection(host='localhost',port=3301,fetch_schema=False) >>> conn.crud_ - conn.crud_count( conn.crud_insert( conn.crud_insert_object_many( - conn.crud_min( conn.crud_replace_object( conn.crud_stats( - conn.crud_unflatten_rows( conn.crud_upsert_many( conn.crud_delete( - conn.crud_insert_many( conn.crud_len( conn.crud_replace( - conn.crud_replace_object_many( conn.crud_storage_info( conn.crud_update( - conn.crud_upsert_object( conn.crud_get( conn.crud_insert_object( - conn.crud_max( conn.crud_replace_many( conn.crud_select( + conn.crud_count( conn.crud_insert( conn.crud_insert_object_many( + conn.crud_min( conn.crud_replace_object( conn.crud_stats( + conn.crud_unflatten_rows( conn.crud_upsert_many( conn.crud_delete( + conn.crud_insert_many( conn.crud_len( conn.crud_replace( + conn.crud_replace_object_many( conn.crud_storage_info( conn.crud_update( + conn.crud_upsert_object( conn.crud_get( conn.crud_insert_object( + conn.crud_max( conn.crud_replace_many( conn.crud_select( conn.crud_truncate( conn.crud_upsert( conn.crud_upsert_object_many( -As an example, consider :meth:`~tarantool.Connection.crud_insert` and :meth:`~tarantool.Connection.crud_insert_object_many`. +As an example, consider :meth:`~tarantool.Connection.crud_insert` and :meth:`~tarantool.Connection.crud_insert_object_many`. It is recommended to enclose calls in the try-except construction as follows: .. code-block:: python @@ -392,13 +392,13 @@ It is recommended to enclose calls in the try-except construction as follows: ... res = conn.crud_insert('tester', (3500,300,'Rob')) ... except CrudModuleError as e: ... exc_crud = e - ... + ... >>> exc_crud CrudModuleError(0, 'Failed to insert: Duplicate key exists in unique index "primary_index" in space "tester" with old tuple - [3500, 300, "Rob"] and new tuple - [3500, 300, "Rob"]') >>> exc_crud.extra_info_error >>> exc_crud.extra_info_error. - exc_crud.extra_info_error.class_name exc_crud.extra_info_error.err exc_crud.extra_info_error.file exc_crud.extra_info_error.line exc_crud.extra_info_error.str + exc_crud.extra_info_error.class_name exc_crud.extra_info_error.err exc_crud.extra_info_error.file exc_crud.extra_info_error.line exc_crud.extra_info_error.str >>> exc_crud.extra_info_error.class_name 'InsertError' >>> exc_crud.extra_info_error.str @@ -409,7 +409,7 @@ It is recommended to enclose calls in the try-except construction as follows: ... res = conn.crud_insert_object_many('tester', ({'id':3,'bucket_id':100,'name':'Ann'}, {'id':4,'bucket_id':100,'name':'Sam'}), {'timeout':100, 'rollback_on_error':False}) ... except CrudModuleManyError as e: ... exc_crud = e - ... + ... >>> exc_crud CrudModuleManyError(0, 'Got multiple errors, see errors_list') >>> exc_crud.success_list # some of the rows were inserted. @@ -422,7 +422,7 @@ It is recommended to enclose calls in the try-except construction as follows: 'CallError: Failed for 037adb3a-b9e3-4f78-a6d1-9f0cdb6cbefc: Function returned an error: Duplicate key exists in unique index "primary_index" in space "tester" with old tuple - [3500, 300, "Rob"] and new tuple - [3500, 100, "Mike"]' >>> exc_crud.errors_list[1].str 'InsertManyError: Failed to flatten object: FlattenError: Object is specified in bad format: FlattenError: Unknown field "second_name" is specified' - + # If there are no problems with any rows, the entire response will be contained in the res variable. >>> res = conn.crud_insert_object_many('tester', ({'id':3,'bucket_id':100,'name':'Ann'}, {'id':4,'bucket_id':100,'name':'Sam'}), {'timeout':100, 'rollback_on_error':False}) >>> res.rows @@ -436,7 +436,7 @@ If module crud not found on the router or user has not sufficient grants: ... res = conn.crud_insert('tester', (22221,300,'Rob')) ... except DatabaseError as e: ... exc_db = e - ... + ... >>> exc_db DatabaseError(33, "Procedure 'crud.insert' is not defined. Ensure that you're calling crud.router and user has sufficient grants") >>> exc_db.extra_info diff --git a/requirements-test.txt b/requirements-test.txt index ae4091d2..0f2ad987 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,3 +1,11 @@ git+https://github.com/baztian/dbapi-compliance.git@ea7cb1b4#egg=dbapi-compliance -pyyaml==6.0 +pyyaml >= 6.0.2 importlib-metadata >= 1.0 ; python_version < '3.8' +pylint == 3.3.0 ; python_version >= '3.9' +pylint == 3.2.7 ; python_version == '3.8' +pylint == 2.17.7 ; python_version == '3.7' +flake8 == 6.1.0 ; python_version >= '3.8' +flake8 == 5.0.4 ; python_version < '3.8' +codespell == 2.3.0 ; python_version >= '3.8' +codespell == 2.2.5 ; python_version < '3.8' +setuptools >= 75.3.2 diff --git a/requirements.txt b/requirements.txt index aa4406ce..afcf7b25 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,2 @@ msgpack -pandas pytz -dataclasses; python_version <= '3.6' diff --git a/rpm/SPECS/python-tarantool.spec b/rpm/SPECS/python-tarantool.spec index b1dae43d..b1e5727e 100644 --- a/rpm/SPECS/python-tarantool.spec +++ b/rpm/SPECS/python-tarantool.spec @@ -29,7 +29,6 @@ Python client library for Tarantool.} %package -n python3-%{srcname} Requires: python3-msgpack -Requires: python3-pandas Requires: python3-pytz Summary: %{summary} diff --git a/setup.py b/setup.py index 1c565e79..3fa65c9c 100755 --- a/setup.py +++ b/setup.py @@ -1,37 +1,46 @@ #!/usr/bin/env python +""" +Package setup commands. +""" +# pylint: disable=bad-option-value,too-many-ancestors import codecs import os -import re -try: - from setuptools import setup, find_packages - from setuptools.command.build_py import build_py -except ImportError: - from distutils.core import setup, find_packages - from distutils.command.build_py import build_py +from setuptools import find_packages, setup +from setuptools.command.build_py import build_py # Extra commands for documentation management cmdclass = {} command_options = {} -# Build the package -# python setup.py build_py -# builds the package with generating correspondent VERSION file + class BuildPyCommand(build_py): + """ + Build the package + python setup.py build_py + builds the package with generating correspondent VERSION file + """ + def run(self): + """ + Run the command. + """ + # Import here to allow to run commands # like `python setup.py test` without setuptools_scm. + # pylint: disable=import-outside-toplevel,import-error from setuptools_scm import get_version version = get_version() package_dir = self.get_package_dir('tarantool') version_file = os.path.join(package_dir, 'version.py') - with open(version_file, 'w') as file: + with open(version_file, 'w', encoding='utf-8') as file: file.write(f"__version__ = '{version}'") return super().run() + cmdclass["build_py"] = BuildPyCommand # Build Sphinx documentation (html) @@ -47,33 +56,34 @@ def run(self): # Test runner # python setup.py test try: - from test.setup_command import test - cmdclass["test"] = test + from test.setup_command import Test + cmdclass["test"] = Test except ImportError: pass def read(*parts): + """ + Read the file. + """ + filename = os.path.join(os.path.dirname(__file__), *parts) - with codecs.open(filename, encoding='utf-8') as fp: - return fp.read() + with codecs.open(filename, encoding='utf-8') as file: + return file.read() + + +def get_dependencies(filename): + """ + Get package dependencies from the `requirements.txt`. + """ -def get_dependencies(file): root = os.path.dirname(os.path.realpath(__file__)) - requirements = os.path.join(root, file) - result = [] + requirements = os.path.join(root, filename) if os.path.isfile(requirements): - with open(requirements) as f: - return f.read().splitlines() - raise RuntimeError("Unable to get dependencies from file " + file) - -def find_version(*file_paths): - version_file = read(*file_paths) - version_match = re.search(r"""^__version__\s*=\s*(['"])(.+)\1""", - version_file, re.M) - if version_match: - return version_match.group(2) - raise RuntimeError("Unable to find version string.") + with open(requirements, encoding='utf-8') as file: + return file.read().splitlines() + raise RuntimeError("Unable to get dependencies from file " + filename) + packages = [item for item in find_packages('.') if item.startswith('tarantool')] @@ -102,7 +112,7 @@ def find_version(*file_paths): command_options=command_options, install_requires=get_dependencies('requirements.txt'), setup_requires=[ - 'setuptools_scm==6.4.2', + 'setuptools_scm==7.1.0', ], - python_requires='>=3.6', + python_requires='>=3.7', ) diff --git a/tarantool/__init__.py b/tarantool/__init__.py index 915961fd..97a1e860 100644 --- a/tarantool/__init__.py +++ b/tarantool/__init__.py @@ -1,6 +1,7 @@ -# pylint: disable=C0301,W0105,W0401,W0614 - -import sys +""" +This package provides API for interaction with a Tarantool server. +""" +# pylint: disable=too-many-arguments,too-many-positional-arguments from tarantool.connection import Connection from tarantool.mesh_connection import MeshConnection @@ -50,7 +51,7 @@ __version__ = '0.0.0-dev' -def connect(host="localhost", port=33013, user=None, password=None, +def connect(host="localhost", port=33013, socket_fd=None, user=None, password=None, encoding=ENCODING_DEFAULT, transport=DEFAULT_TRANSPORT, ssl_key_file=DEFAULT_SSL_KEY_FILE, ssl_cert_file=DEFAULT_SSL_CERT_FILE, @@ -63,6 +64,8 @@ def connect(host="localhost", port=33013, user=None, password=None, :param port: Refer to :paramref:`~tarantool.Connection.params.port`. + :param socket_fd: Refer to :paramref:`~tarantool.Connection.params.socket_fd`. + :param user: Refer to :paramref:`~tarantool.Connection.params.user`. :param password: Refer to @@ -92,6 +95,7 @@ def connect(host="localhost", port=33013, user=None, password=None, """ return Connection(host, port, + socket_fd=socket_fd, user=user, password=password, socket_timeout=SOCKET_TIMEOUT, @@ -141,4 +145,4 @@ def connectmesh(addrs=({'host': 'localhost', 'port': 3301},), user=None, __all__ = ['connect', 'Connection', 'connectmesh', 'MeshConnection', 'Schema', 'Error', 'DatabaseError', 'NetworkError', 'NetworkWarning', 'SchemaError', 'dbapi', 'Datetime', 'Interval', 'IntervalAdjust', - 'ConnectionPool', 'Mode', 'BoxError',] + 'ConnectionPool', 'Mode', 'BoxError'] diff --git a/tarantool/connection.py b/tarantool/connection.py index 7fd43b9e..682b0700 100644 --- a/tarantool/connection.py +++ b/tarantool/connection.py @@ -1,43 +1,43 @@ -# pylint: disable=C0301,W0105,W0401,W0614 """ This module provides API for interaction with a Tarantool server. """ +# pylint: disable=too-many-lines,duplicate-code import os +import select import time import errno +from enum import Enum import socket try: import ssl - is_ssl_supported = True + IS_SSL_SUPPORTED = True except ImportError: - is_ssl_supported = False + IS_SSL_SUPPORTED = False import sys import abc import ctypes import ctypes.util from ctypes import c_ssize_t +from typing import Optional, Union +from copy import copy import msgpack from tarantool.response import ( unpacker_factory as default_unpacker_factory, - Response, ) from tarantool.request import ( packer_factory as default_packer_factory, Request, - # RequestOK, RequestCall, RequestDelete, RequestEval, RequestInsert, - RequestJoin, RequestReplace, RequestPing, RequestSelect, - RequestSubscribe, RequestUpdate, RequestUpsert, RequestAuthenticate, @@ -52,14 +52,15 @@ RECONNECT_DELAY, DEFAULT_TRANSPORT, SSL_TRANSPORT, + DEFAULT_HOST, + DEFAULT_PORT, + DEFAULT_SOCKET_FD, DEFAULT_SSL_KEY_FILE, DEFAULT_SSL_CERT_FILE, DEFAULT_SSL_CA_FILE, DEFAULT_SSL_CIPHERS, DEFAULT_SSL_PASSWORD, DEFAULT_SSL_PASSWORD_FILE, - REQUEST_TYPE_OK, - REQUEST_TYPE_ERROR, IPROTO_GREETING_SIZE, ITERATOR_EQ, ITERATOR_ALL, @@ -69,7 +70,9 @@ IPROTO_FEATURE_TRANSACTIONS, IPROTO_FEATURE_ERROR_EXTENSION, IPROTO_FEATURE_WATCHERS, - IPROTO_AUTH_TYPE, + IPROTO_FEATURE_PAGINATION, + IPROTO_FEATURE_SPACE_AND_INDEX_NAMES, + IPROTO_FEATURE_WATCH_ONCE, IPROTO_CHUNK, AUTH_TYPE_CHAP_SHA1, AUTH_TYPE_PAP_SHA256, @@ -93,7 +96,6 @@ CrudModuleError, CrudModuleManyError, SchemaReloadException, - Warning, warn ) from tarantool.schema import Schema @@ -108,7 +110,10 @@ CrudError, call_crud, ) -from typing import Union + +WWSAEWOULDBLOCK = 10035 +ER_UNKNOWN_REQUEST_TYPE = 48 + # Based on https://realpython.com/python-interface/ class ConnectionInterface(metaclass=abc.ABCMeta): @@ -120,36 +125,37 @@ class ConnectionInterface(metaclass=abc.ABCMeta): Lua code on server, make simple data manipulations and execute SQL queries. """ + # pylint: disable=too-many-public-methods @classmethod def __subclasshook__(cls, subclass): - return (hasattr(subclass, 'close') and - callable(subclass.close) and - hasattr(subclass, 'is_closed') and - callable(subclass.is_closed) and - hasattr(subclass, 'connect') and - callable(subclass.connect) and - hasattr(subclass, 'call') and - callable(subclass.call) and - hasattr(subclass, 'eval') and - callable(subclass.eval) and - hasattr(subclass, 'replace') and - callable(subclass.replace) and - hasattr(subclass, 'insert') and - callable(subclass.insert) and - hasattr(subclass, 'delete') and - callable(subclass.delete) and - hasattr(subclass, 'upsert') and - callable(subclass.upsert) and - hasattr(subclass, 'update') and - callable(subclass.update) and - hasattr(subclass, 'ping') and - callable(subclass.ping) and - hasattr(subclass, 'select') and - callable(subclass.select) and - hasattr(subclass, 'execute') and - callable(subclass.execute) or - NotImplemented) + return (hasattr(subclass, 'close') + and callable(subclass.close) + and hasattr(subclass, 'is_closed') + and callable(subclass.is_closed) + and hasattr(subclass, 'connect') + and callable(subclass.connect) + and hasattr(subclass, 'call') + and callable(subclass.call) + and hasattr(subclass, 'eval') + and callable(subclass.eval) + and hasattr(subclass, 'replace') + and callable(subclass.replace) + and hasattr(subclass, 'insert') + and callable(subclass.insert) + and hasattr(subclass, 'delete') + and callable(subclass.delete) + and hasattr(subclass, 'upsert') + and callable(subclass.upsert) + and hasattr(subclass, 'update') + and callable(subclass.update) + and hasattr(subclass, 'ping') + and callable(subclass.ping) + and hasattr(subclass, 'select') + and callable(subclass.select) + and hasattr(subclass, 'execute') + and callable(subclass.execute) + or NotImplemented) @abc.abstractmethod def close(self): @@ -193,7 +199,7 @@ def eval(self, expr, *args, on_push=None, on_push_ctx=None): raise NotImplementedError @abc.abstractmethod - def replace(self, space_name, values, on_push=None, on_push_ctx=None): + def replace(self, space_name, values, *, on_push=None, on_push_ctx=None): """ Reference implementation: :meth:`~tarantool.Connection.replace`. """ @@ -201,7 +207,7 @@ def replace(self, space_name, values, on_push=None, on_push_ctx=None): raise NotImplementedError @abc.abstractmethod - def insert(self, space_name, values, on_push=None, on_push_ctx=None): + def insert(self, space_name, values, *, on_push=None, on_push_ctx=None): """ Reference implementation: :meth:`~tarantool.Connection.insert`. """ @@ -213,15 +219,17 @@ def delete(self, space_name, key, *, index=None, on_push=None, on_push_ctx=None) """ Reference implementation: :meth:`~tarantool.Connection.delete`. """ + # pylint: disable=too-many-arguments raise NotImplementedError @abc.abstractmethod - def upsert(self, space_name, tuple_value, op_list, *, index=None, + def upsert(self, space_name, tuple_value, op_list, *, index=None, on_push=None, on_push_ctx=None): """ Reference implementation: :meth:`~tarantool.Connection.upsert`. """ + # pylint: disable=too-many-arguments raise NotImplementedError @@ -230,6 +238,7 @@ def update(self, space_name, key, op_list, *, index=None, on_push=None, on_push_ """ Reference implementation: :meth:`~tarantool.Connection.update`. """ + # pylint: disable=too-many-arguments raise NotImplementedError @@ -247,6 +256,7 @@ def select(self, space_name, key, *, offset=None, limit=None, """ Reference implementation: :meth:`~tarantool.Connection.select`. """ + # pylint: disable=too-many-arguments raise NotImplementedError @@ -259,7 +269,7 @@ def execute(self, query, params): raise NotImplementedError @abc.abstractmethod - def crud_insert(self, space_name, values, opts={}): + def crud_insert(self, space_name, values, opts=None): """ Reference implementation: :meth:`~tarantool.Connection.crud_insert`. """ @@ -267,7 +277,7 @@ def crud_insert(self, space_name, values, opts={}): raise NotImplementedError @abc.abstractmethod - def crud_insert_object(self, space_name, values, opts={}): + def crud_insert_object(self, space_name, values, opts=None): """ Reference implementation: :meth:`~tarantool.Connection.crud_insert_object`. """ @@ -275,7 +285,7 @@ def crud_insert_object(self, space_name, values, opts={}): raise NotImplementedError @abc.abstractmethod - def crud_insert_many(self, space_name, values, opts={}): + def crud_insert_many(self, space_name, values, opts=None): """ Reference implementation: :meth:`~tarantool.Connection.crud_insert_many`. """ @@ -283,7 +293,7 @@ def crud_insert_many(self, space_name, values, opts={}): raise NotImplementedError @abc.abstractmethod - def crud_insert_object_many(self, space_name, values, opts={}): + def crud_insert_object_many(self, space_name, values, opts=None): """ Reference implementation: :meth:`~tarantool.Connection.crud_insert_object_many`. """ @@ -291,7 +301,7 @@ def crud_insert_object_many(self, space_name, values, opts={}): raise NotImplementedError @abc.abstractmethod - def crud_get(self, space_name, key, opts={}): + def crud_get(self, space_name, key, opts=None): """ Reference implementation: :meth:`~tarantool.Connection.crud_get`. """ @@ -299,7 +309,7 @@ def crud_get(self, space_name, key, opts={}): raise NotImplementedError @abc.abstractmethod - def crud_update(self, space_name, key, operations=[], opts={}): + def crud_update(self, space_name, key, operations=None, opts=None): """ Reference implementation: :meth:`~tarantool.Connection.crud_update`. """ @@ -307,7 +317,7 @@ def crud_update(self, space_name, key, operations=[], opts={}): raise NotImplementedError @abc.abstractmethod - def crud_delete(self, space_name, key, opts={}): + def crud_delete(self, space_name, key, opts=None): """ Reference implementation: :meth:`~tarantool.Connection.crud_delete`. """ @@ -315,7 +325,7 @@ def crud_delete(self, space_name, key, opts={}): raise NotImplementedError @abc.abstractmethod - def crud_replace(self, space_name, values, opts={}): + def crud_replace(self, space_name, values, opts=None): """ Reference implementation: :meth:`~tarantool.Connection.crud_replace`. """ @@ -323,7 +333,7 @@ def crud_replace(self, space_name, values, opts={}): raise NotImplementedError @abc.abstractmethod - def crud_replace_object(self, space_name, values, opts={}): + def crud_replace_object(self, space_name, values, opts=None): """ Reference implementation: :meth:`~tarantool.Connection.crud_replace_object`. """ @@ -331,7 +341,7 @@ def crud_replace_object(self, space_name, values, opts={}): raise NotImplementedError @abc.abstractmethod - def crud_replace_many(self, space_name, values, opts={}): + def crud_replace_many(self, space_name, values, opts=None): """ Reference implementation: :meth:`~tarantool.Connection.crud_replace_many`. """ @@ -339,7 +349,7 @@ def crud_replace_many(self, space_name, values, opts={}): raise NotImplementedError @abc.abstractmethod - def crud_replace_object_many(self, space_name, values, opts={}): + def crud_replace_object_many(self, space_name, values, opts=None): """ Reference implementation: :meth:`~tarantool.Connection.crud_replace_object_many`. """ @@ -347,7 +357,7 @@ def crud_replace_object_many(self, space_name, values, opts={}): raise NotImplementedError @abc.abstractmethod - def crud_upsert(self, space_name, values, operations=[], opts={}): + def crud_upsert(self, space_name, values, operations=None, opts=None): """ Reference implementation: :meth:`~tarantool.Connection.crud_upsert`. """ @@ -355,7 +365,7 @@ def crud_upsert(self, space_name, values, operations=[], opts={}): raise NotImplementedError @abc.abstractmethod - def crud_upsert_object(self, space_name, values, operations=[], opts={}): + def crud_upsert_object(self, space_name, values, operations=None, opts=None): """ Reference implementation: :meth:`~tarantool.Connection.crud_upsert_object`. """ @@ -363,7 +373,7 @@ def crud_upsert_object(self, space_name, values, operations=[], opts={}): raise NotImplementedError @abc.abstractmethod - def crud_upsert_many(self, space_name, values_operation, opts={}): + def crud_upsert_many(self, space_name, values_operation, opts=None): """ Reference implementation: :meth:`~tarantool.Connection.crud_upsert_many`. """ @@ -371,7 +381,7 @@ def crud_upsert_many(self, space_name, values_operation, opts={}): raise NotImplementedError @abc.abstractmethod - def crud_upsert_object_many(self, space_name, values_operation, opts={}): + def crud_upsert_object_many(self, space_name, values_operation, opts=None): """ Reference implementation: :meth:`~tarantool.Connection.crud_upsert_object_many`. """ @@ -379,7 +389,7 @@ def crud_upsert_object_many(self, space_name, values_operation, opts={}): raise NotImplementedError @abc.abstractmethod - def crud_select(self, space_name: str, conditions=[], opts={}): + def crud_select(self, space_name: str, conditions=None, opts=None): """ Reference implementation: :meth:`~tarantool.Connection.crud_select`. """ @@ -387,7 +397,7 @@ def crud_select(self, space_name: str, conditions=[], opts={}): raise NotImplementedError @abc.abstractmethod - def crud_min(self, space_name, index_name, opts={}): + def crud_min(self, space_name, index_name, opts=None): """ Reference implementation: :meth:`~tarantool.Connection.crud_min`. """ @@ -395,7 +405,7 @@ def crud_min(self, space_name, index_name, opts={}): raise NotImplementedError @abc.abstractmethod - def crud_max(self, space_name, index_name, opts={}): + def crud_max(self, space_name, index_name, opts=None): """ Reference implementation: :meth:`~tarantool.Connection.crud_max`. """ @@ -403,7 +413,7 @@ def crud_max(self, space_name, index_name, opts={}): raise NotImplementedError @abc.abstractmethod - def crud_truncate(self, space_name, opts={}): + def crud_truncate(self, space_name, opts=None): """ Reference implementation: :meth:`~tarantool.Connection.crud_truncate`. """ @@ -411,7 +421,7 @@ def crud_truncate(self, space_name, opts={}): raise NotImplementedError @abc.abstractmethod - def crud_len(self, space_name, opts={}): + def crud_len(self, space_name, opts=None): """ Reference implementation: :meth:`~tarantool.Connection.crud_len`. """ @@ -419,7 +429,7 @@ def crud_len(self, space_name, opts={}): raise NotImplementedError @abc.abstractmethod - def crud_storage_info(self, opts={}): + def crud_storage_info(self, opts=None): """ Reference implementation: :meth:`~tarantool.Connection.crud_storage_info`. """ @@ -427,7 +437,7 @@ def crud_storage_info(self, opts={}): raise NotImplementedError @abc.abstractmethod - def crud_count(self, space_name, conditions=[], opts={}): + def crud_count(self, space_name, conditions=None, opts=None): """ Reference implementation: :meth:`~tarantool.Connection.crud_count`. """ @@ -451,6 +461,34 @@ def crud_unflatten_rows(self, rows, metadata): raise NotImplementedError +class JoinState(Enum): + """ + Current replication join state. See `join protocol`_ for more info. + + .. _join protocol: https://www.tarantool.io/en/doc/latest/dev_guide/internals/iproto/replication/#box-protocol-join + """ + + HANDSHAKE = 1 + """ + Sent the join request. + """ + + INITIAL = 2 + """ + Received initial vclock. + """ + + FINAL = 3 + """ + Received current vclock. + """ + + DONE = 4 + """ + No more messages expected. + """ + + class Connection(ConnectionInterface): """ Represents a connection to the Tarantool server. @@ -459,6 +497,7 @@ class Connection(ConnectionInterface): check its status, call procedures and evaluate Lua code on server, make simple data manipulations and execute SQL queries. """ + # pylint: disable=too-many-instance-attributes,too-many-public-methods,bad-option-value,no-self-use # DBAPI Extension: supply exceptions as attributes on the connection Error = Error @@ -559,7 +598,10 @@ class Connection(ConnectionInterface): :value: :exc:`~tarantool.error.CrudModuleError` """ - def __init__(self, host, port, + def __init__(self, + host=DEFAULT_HOST, + port=DEFAULT_PORT, + socket_fd=DEFAULT_SOCKET_FD, user=None, password=None, socket_timeout=SOCKET_TIMEOUT, @@ -579,14 +621,20 @@ def __init__(self, host, port, ssl_password_file=DEFAULT_SSL_PASSWORD_FILE, packer_factory=default_packer_factory, unpacker_factory=default_unpacker_factory, - auth_type=None): + auth_type=None, + fetch_schema=True, + required_protocol_version=None, + required_features=None): """ :param host: Server hostname or IP address. Use ``None`` for Unix sockets. :type host: :obj:`str` or :obj:`None` - :param port: Server port or Unix socket path. - :type port: :obj:`int` or :obj:`str` + :param port: Server port, or Unix socket path. + :type port: :obj:`int` or :obj:`str` or :obj:`None` + + :param socket_fd: socket fd number. + :type socket_fd: :obj:`int` or :obj:`None` :param user: User name for authentication on the Tarantool server. @@ -736,6 +784,26 @@ def __init__(self, host, port, ``"chap-sha1"``. :type auth_type: :obj:`None` or :obj:`str`, optional + :param bool fetch_schema: If ``False``, schema is not loaded on connect + and schema updates are not automatically loaded. + As a result, these methods become unavailable: + :meth:`~tarantool.Connection.replace`, + :meth:`~tarantool.Connection.insert`, + :meth:`~tarantool.Connection.delete`, + :meth:`~tarantool.Connection.upsert`, + :meth:`~tarantool.Connection.update`, + :meth:`~tarantool.Connection.select`, + :meth:`~tarantool.Connection.space`. + :type fetch_schema: :obj:`bool`, optional + + :param required_protocol_version: Minimal protocol version that + should be supported by Tarantool server. + :type required_protocol_version: :obj:`int` or :obj:`None`, optional + + :param required_features: List of protocol features that + should be supported by Tarantool server. + :type required_features: :obj:`list` or :obj:`None`, optional + :raise: :exc:`~tarantool.error.ConfigurationError`, :meth:`~tarantool.Connection.connect` exceptions @@ -744,10 +812,23 @@ def __init__(self, host, port, .. _mp_bin: https://github.com/msgpack/msgpack/blob/master/spec.md#bin-format-family .. _mp_array: https://github.com/msgpack/msgpack/blob/master/spec.md#array-format-family """ + # pylint: disable=too-many-arguments,too-many-locals,too-many-statements,too-many-positional-arguments + + if host is None and port is None and socket_fd is None: + raise ConfigurationError("need to specify host/port, " + "port (in case of Unix sockets) " + "or socket_fd") + + if socket_fd is not None and (host is not None or port is not None): + raise ConfigurationError("specifying both socket_fd and host/port is not allowed") + + if host is not None and port is None: + raise ConfigurationError("when specifying host, " + "it is also necessary to specify port") if msgpack.version >= (1, 0, 0) and encoding not in (None, 'utf-8'): - raise ConfigurationError("msgpack>=1.0.0 only supports None and " + - "'utf-8' encoding option values") + raise ConfigurationError("msgpack>=1.0.0 only supports None and " + + "'utf-8' encoding option values") if os.name == 'nt': libc = ctypes.WinDLL( @@ -761,13 +842,15 @@ def __init__(self, host, port, recv.restype = ctypes.c_int self.host = host self.port = port + self.socket_fd = socket_fd self.user = user self.password = password self.socket_timeout = socket_timeout self.reconnect_delay = reconnect_delay self.reconnect_max_attempts = reconnect_max_attempts - self.schema = Schema(self) - self.schema_version = 1 + self.fetch_schema = fetch_schema + self.schema = None + self.schema_version = 0 self._socket = None self.connected = False self.error = True @@ -788,11 +871,23 @@ def __init__(self, host, port, IPROTO_FEATURE_TRANSACTIONS: False, IPROTO_FEATURE_ERROR_EXTENSION: False, IPROTO_FEATURE_WATCHERS: False, + IPROTO_FEATURE_PAGINATION: False, + IPROTO_FEATURE_SPACE_AND_INDEX_NAMES: False, + IPROTO_FEATURE_WATCH_ONCE: False, } self._packer_factory_impl = packer_factory self._unpacker_factory_impl = unpacker_factory self._client_auth_type = auth_type self._server_auth_type = None + self.version_id = None + self.uuid = None + self._salt = None + self._client_protocol_version = CONNECTOR_IPROTO_VERSION + self._client_features = copy(CONNECTOR_FEATURES) + self._server_protocol_version = None + self._server_features = None + self.required_protocol_version = required_protocol_version + self.required_features = copy(required_features) if connect_now: self.connect() @@ -825,10 +920,37 @@ def connect_basic(self): :meta private: """ - if self.host is None: - self.connect_unix() - else: + if self.socket_fd is not None: + self.connect_socket_fd() + elif self.host is not None: self.connect_tcp() + else: + self.connect_unix() + + def connect_socket_fd(self): + """ + Establish a connection using an existing socket fd. + + +---------------------+--------------------------+-------------------------+ + | socket_fd / timeout | >= 0 | `None` | + +=====================+==========================+=========================+ + | blocking | Set non-blocking socket | Don't change, `select` | + | | lib call `select` | isn't needed | + +---------------------+--------------------------+-------------------------+ + | non-blocking | Don't change, socket lib | Don't change, call | + | | call `select` | `select` ourselves | + +---------------------+--------------------------+-------------------------+ + + :meta private: + """ + + self.connected = True + if self._socket: + self._socket.close() + + self._socket = socket.socket(fileno=self.socket_fd) + if self.socket_timeout is not None: + self._socket.settimeout(self.socket_timeout) def connect_tcp(self): """ @@ -849,9 +971,9 @@ def connect_tcp(self): (self.host, self.port), timeout=self.connection_timeout) self._socket.settimeout(self.socket_timeout) self._socket.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) - except socket.error as e: + except socket.error as exc: self.connected = False - raise NetworkError(e) + raise NetworkError(exc) from exc def connect_unix(self): """ @@ -862,6 +984,7 @@ def connect_unix(self): :meta private: """ + # pylint: disable=no-member try: # If old socket already exists - close it and re-create @@ -872,9 +995,9 @@ def connect_unix(self): self._socket.settimeout(self.connection_timeout) self._socket.connect(self.port) self._socket.settimeout(self.socket_timeout) - except socket.error as e: + except socket.error as exc: self.connected = False - raise NetworkError(e) + raise NetworkError(exc) from exc def wrap_socket_ssl(self): """ @@ -885,17 +1008,17 @@ def wrap_socket_ssl(self): :meta private: """ - if not is_ssl_supported: + if not IS_SSL_SUPPORTED: raise SslError("Your version of Python doesn't support SSL") ver = sys.version_info if ver[0] < 3 or (ver[0] == 3 and ver[1] < 5): - raise SslError("SSL transport is supported only since " + + raise SslError("SSL transport is supported only since " "python 3.5") if ((self.ssl_cert_file is None and self.ssl_key_file is not None) or (self.ssl_cert_file is not None and self.ssl_key_file is None)): - raise SslError("Both ssl_cert_file and ssl_key_file should be " + + raise SslError("Both ssl_cert_file and ssl_key_file should be " "configured or unconfigured") try: @@ -926,10 +1049,10 @@ def wrap_socket_ssl(self): context.set_ciphers(self.ssl_ciphers) self._socket = context.wrap_socket(self._socket) - except SslError as e: - raise e - except Exception as e: - raise SslError(e) + except SslError as exc: + raise exc + except Exception as exc: + raise SslError(exc) from exc def _ssl_load_cert_chain(self, context): """ @@ -954,34 +1077,32 @@ def _ssl_load_cert_chain(self, context): keyfile=self.ssl_key_file, password=self.ssl_password) return - except Exception as e: - exc_list.append(e) - + except Exception as exc: # pylint: disable=bad-option-value,broad-exception-caught,broad-except + exc_list.append(exc) if self.ssl_password_file is not None: - with open(self.ssl_password_file) as file: + with open(self.ssl_password_file, encoding=self.encoding) as file: for line in file: try: context.load_cert_chain(certfile=self.ssl_cert_file, keyfile=self.ssl_key_file, password=line.rstrip()) return - except Exception as e: - exc_list.append(e) - + except Exception as exc: # pylint: disable=bad-option-value,broad-exception-caught,broad-except + exc_list.append(exc) try: def password_raise_error(): - raise SslError("Password prompt for decrypting the private " + - "key is unsupported, use ssl_password or " + + raise SslError("Password prompt for decrypting the private " + "key is unsupported, use ssl_password or " "ssl_password_file") context.load_cert_chain(certfile=self.ssl_cert_file, keyfile=self.ssl_key_file, password=password_raise_error) return - except Exception as e: - exc_list.append(e) + except Exception as exc: # pylint: disable=bad-option-value,broad-exception-caught,broad-except + exc_list.append(exc) raise SslError(exc_list) @@ -1000,9 +1121,11 @@ def handshake(self): if greeting.protocol != "Binary": raise NetworkError("Unsupported protocol: " + greeting.protocol) self.version_id = greeting.version_id - self._check_features() self.uuid = greeting.uuid self._salt = greeting.salt + + self._check_features() + if self.user: self.authenticate(self.user, self.password) @@ -1023,12 +1146,16 @@ def connect(self): if self.transport == SSL_TRANSPORT: self.wrap_socket_ssl() self.handshake() - self.load_schema() - except SslError as e: - raise e - except Exception as e: + if self.fetch_schema: + self.schema = Schema(self) + self.load_schema() + else: + self.schema = None + except SslError as exc: + raise exc + except Exception as exc: self.connected = False - raise NetworkError(e) + raise NetworkError(exc) from exc def _recv(self, to_read): """ @@ -1047,28 +1174,33 @@ def _recv(self, to_read): while to_read > 0: try: tmp = self._socket.recv(to_read) - except OverflowError: + except BlockingIOError: + ready, _, _ = select.select([self._socket.fileno()], [], [], self.socket_timeout) + if not ready: + raise NetworkError(TimeoutError()) # pylint: disable=raise-missing-from + continue + except OverflowError as exc: self._socket.close() err = socket.error( errno.ECONNRESET, "Packet too large. Closing connection to server" ) - raise NetworkError(err) - except socket.error: + raise NetworkError(err) from exc + except socket.error as exc: + err = socket.error( + errno.ECONNRESET, + "Lost connection to server during query" + ) + raise NetworkError(err) from exc + + if len(tmp) == 0: err = socket.error( errno.ECONNRESET, "Lost connection to server during query" ) raise NetworkError(err) - else: - if len(tmp) == 0: - err = socket.error( - errno.ECONNRESET, - "Lost connection to server during query" - ) - raise NetworkError(err) - to_read -= len(tmp) - buf += tmp + to_read -= len(tmp) + buf += tmp return buf def _read_response(self): @@ -1086,6 +1218,41 @@ def _read_response(self): # Read the packet return self._recv(length) + def _sendall(self, bytes_to_send): + """ + Sends bytes to the transport (socket). + + :param bytes_to_send: Message to send. + :type bytes_to_send: :obj:`bytes` + + :raise: :exc:`~tarantool.error.NetworkError` + + :meta private: + """ + + total_sent = 0 + while total_sent < len(bytes_to_send): + try: + sent = self._socket.send(bytes_to_send[total_sent:]) + if sent == 0: + err = socket.error( + errno.ECONNRESET, + "Lost connection to server during query" + ) + raise NetworkError(err) + total_sent += sent + except BlockingIOError as exc: + total_sent += exc.characters_written + _, ready, _ = select.select([], [self._socket.fileno()], [], self.socket_timeout) + if not ready: + raise NetworkError(TimeoutError()) # pylint: disable=raise-missing-from + except socket.error as exc: + err = socket.error( + errno.ECONNRESET, + "Lost connection to server during query" + ) + raise NetworkError(err) from exc + def _send_request_wo_reconnect(self, request, on_push=None, on_push_ctx=None): """ Send request without trying to reconnect. @@ -1114,16 +1281,17 @@ def _send_request_wo_reconnect(self, request, on_push=None, on_push_ctx=None): response = None while True: try: - self._socket.sendall(bytes(request)) + self._sendall(bytes(request)) response = request.response_class(self, self._read_response()) break - except SchemaReloadException as e: - self.update_schema(e.schema_version) + except SchemaReloadException as exc: + if self.schema is not None: + self.update_schema(exc.schema_version) continue - while response._code == IPROTO_CHUNK: - if on_push is not None: - on_push(response._data, on_push_ctx) + while response.code == IPROTO_CHUNK: + if on_push is not None: + on_push(response.data, on_push_ctx) response = request.response_class(self, self._read_response()) return response @@ -1138,42 +1306,41 @@ def _opt_reconnect(self): :meta private: """ + # pylint: disable=no-member # **Bug in Python: timeout is an internal Python construction (???). if not self._socket: - return self.connect() + self.connect() + return def check(): # Check that connection is alive buf = ctypes.create_string_buffer(2) try: sock_fd = self._socket.fileno() - except socket.error as e: - if e.errno == errno.EBADF: - return errno.ECONNRESETtuple_value - else: - if os.name == 'nt': - flag = socket.MSG_PEEK - self._socket.setblocking(False) - else: - flag = socket.MSG_DONTWAIT | socket.MSG_PEEK - retbytes = self._sys_recv(sock_fd, buf, 1, flag) - - err = 0 - if os.name!= 'nt': - err = ctypes.get_errno() - else: - err = ctypes.get_last_error() - self._socket.setblocking(True) - - - WWSAEWOULDBLOCK = 10035 - if (retbytes < 0) and (err == errno.EAGAIN or - err == errno.EWOULDBLOCK or - err == WWSAEWOULDBLOCK): - ctypes.set_errno(0) - return errno.EAGAIN - else: + except socket.error as exc: + if exc.errno == errno.EBADF: return errno.ECONNRESET + return exc.errno + + if os.name == 'nt': + flag = socket.MSG_PEEK + self._socket.setblocking(False) + else: + flag = socket.MSG_DONTWAIT | socket.MSG_PEEK + retbytes = self._sys_recv(sock_fd, buf, 1, flag) + + err = 0 + if os.name != 'nt': + err = ctypes.get_errno() + else: + err = ctypes.get_last_error() + self._socket.setblocking(True) + + if (retbytes < 0) and err in (errno.EAGAIN, errno.EWOULDBLOCK, WWSAEWOULDBLOCK): + ctypes.set_errno(0) + return errno.EAGAIN + + return errno.ECONNRESET last_errno = check() if self.connected and last_errno == errno.EAGAIN: @@ -1190,8 +1357,8 @@ def check(): # Check that connection is alive else: if self.connected: break - warn("Reconnecting, attempt %d of %d" % - (attempt, self.reconnect_max_attempts), NetworkWarning) + warn(f"Reconnecting, attempt {attempt} of {self.reconnect_max_attempts}", + NetworkWarning) if attempt == self.reconnect_max_attempts: raise NetworkError( socket.error(last_errno, errno.errorcode[last_errno])) @@ -1255,6 +1422,9 @@ def update_schema(self, schema_version): :meta private: """ + if self.schema is None: + self.schema = Schema(self) + self.schema_version = schema_version self.flush_schema() @@ -1269,6 +1439,19 @@ def flush_schema(self): self.schema.flush() self.load_schema() + def _schemaful_connection_check(self): + """ + Checks whether the connection is schemaful. + If the connection is schemaless, an exception will be thrown + about unsupporting the method in connection opened + with fetch_schema=False. + + :raise: :exc:`~tarantool.error.NotSupportedError` + """ + if self.schema is None: + raise NotSupportedError('This method is not available in ' + 'connection opened with fetch_schema=False') + def call(self, func_name, *args, on_push=None, on_push_ctx=None): """ Execute a CALL request: call a stored Lua function. @@ -1342,7 +1525,7 @@ def eval(self, expr, *args, on_push=None, on_push_ctx=None): response = self._send_request(request, on_push, on_push_ctx) return response - def replace(self, space_name, values, on_push=None, on_push_ctx=None): + def replace(self, space_name, values, *, on_push=None, on_push_ctx=None): """ Execute a REPLACE request: `replace`_ a tuple in the space. Doesn't throw an error if there is no tuple with the specified @@ -1366,11 +1549,14 @@ def replace(self, space_name, values, on_push=None, on_push_ctx=None): :exc:`~tarantool.error.DatabaseError`, :exc:`~tarantool.error.SchemaError`, :exc:`~tarantool.error.NetworkError`, - :exc:`~tarantool.error.SslError` + :exc:`~tarantool.error.SslError`, + :exc:`~tarantool.error.NotSupportedError` .. _replace: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/replace/ """ + self._schemaful_connection_check() + if isinstance(space_name, str): space_name = self.schema.get_space(space_name).sid if on_push is not None and not callable(on_push): @@ -1405,13 +1591,14 @@ def authenticate(self, user, password): if not self._socket: return self._opt_reconnect() - request = RequestAuthenticate(self, - salt=self._salt, - user=self.user, - password=self.password, - auth_type=self._get_auth_type()) + request = RequestAuthenticate( + self, + salt=self._salt, + user=self.user, + password=self.password, + auth_type=self._get_auth_type()) auth_response = self._send_request_wo_reconnect(request) - if auth_response.return_code == 0: + if auth_response.return_code == 0 and self.schema is not None: self.flush_schema() return auth_response @@ -1431,11 +1618,13 @@ def _get_auth_type(self): auth_type = AUTH_TYPE_CHAP_SHA1 else: if self._server_auth_type not in AUTH_TYPES: - raise ConfigurationError(f'Unknown server authentication type {self._server_auth_type}') + raise ConfigurationError('Unknown server authentication type ' + + str(self._server_auth_type)) auth_type = self._server_auth_type else: if self._client_auth_type not in AUTH_TYPES: - raise ConfigurationError(f'Unknown client authentication type {self._client_auth_type}') + raise ConfigurationError('Unknown client authentication type ' + + str(self._client_auth_type)) auth_type = self._client_auth_type if auth_type == AUTH_TYPE_PAP_SHA256 and self.transport != SSL_TRANSPORT: @@ -1443,124 +1632,16 @@ def _get_auth_type(self): return auth_type - def _join_v16(self, server_uuid): - """ - Execute a JOIN request for Tarantool 1.6 and older. - - :param server_uuid: UUID of Tarantool server to join. - :type server_uuid: :obj:`str` - - :raise: :exc:`~AssertionError`, - :exc:`~tarantool.error.DatabaseError`, - :exc:`~tarantool.error.SchemaError`, - :exc:`~tarantool.error.NetworkError`, - :exc:`~tarantool.error.SslError` - """ - - request = RequestJoin(self, server_uuid) - self._socket.sendall(bytes(request)) - - while True: - resp = Response(self, self._read_response()) - yield resp - if resp.code == REQUEST_TYPE_OK or resp.code >= REQUEST_TYPE_ERROR: - return - self.close() # close connection after JOIN - - def _join_v17(self, server_uuid): - """ - Execute a JOIN request for Tarantool 1.7 and newer. - - :param server_uuid: UUID of Tarantool server to join. - :type server_uuid: :obj:`str` - - :raise: :exc:`~AssertionError`, - :exc:`~tarantool.error.DatabaseError`, - :exc:`~tarantool.error.SchemaError`, - :exc:`~tarantool.error.NetworkError`, - :exc:`~tarantool.error.SslError` - """ - - class JoinState: - Handshake, Initial, Final, Done = range(4) - - request = RequestJoin(self, server_uuid) - self._socket.sendall(bytes(request)) - state = JoinState.Handshake - while True: - resp = Response(self, self._read_response()) - yield resp - if resp.code >= REQUEST_TYPE_ERROR: - return - elif resp.code == REQUEST_TYPE_OK: - state = state + 1 - if state == JoinState.Done: - return - def _ops_process(self, space, update_ops): new_ops = [] - for op in update_ops: - if isinstance(op[1], str): - op = list(op) - op[1] = self.schema.get_field(space, op[1])['id'] - new_ops.append(op) + for operation in update_ops: + if isinstance(operation[1], str): + operation = list(operation) + operation[1] = self.schema.get_field(space, operation[1])['id'] + new_ops.append(operation) return new_ops - def join(self, server_uuid): - """ - Execute a JOIN request: `join`_ a replicaset. - - :param server_uuid: UUID of connector "server". - :type server_uuid: :obj:`str` - - :raise: :exc:`~AssertionError`, - :exc:`~tarantool.error.DatabaseError`, - :exc:`~tarantool.error.SchemaError`, - :exc:`~tarantool.error.NetworkError`, - :exc:`~tarantool.error.SslError` - - .. _join: https://www.tarantool.io/en/doc/latest/dev_guide/internals/box_protocol/#iproto-join-0x41 - """ - - self._opt_reconnect() - if self.version_id < version_id(1, 7, 0): - return self._join_v16(server_uuid) - return self._join_v17(server_uuid) - - def subscribe(self, cluster_uuid, server_uuid, vclock=None): - """ - Execute a SUBSCRIBE request: `subscribe`_ to a replicaset - updates. Connection is closed after subscribing. - - :param cluster_uuid: UUID of replicaset cluster. - :type cluster_uuid: :obj:`str` - - :param server_uuid: UUID of connector "server". - :type server_uuid: :obj:`str` - - :param vclock: Connector "server" vclock. - :type vclock: :obj:`dict` or :obj:`None`, optional - - :raise: :exc:`~AssertionError`, - :exc:`~tarantool.error.DatabaseError`, - :exc:`~tarantool.error.SchemaError`, - :exc:`~tarantool.error.NetworkError`, - :exc:`~tarantool.error.SslError` - - .. _subscribe: https://www.tarantool.io/en/doc/latest/dev_guide/internals/box_protocol/#iproto-subscribe-0x42 - """ - - vclock = vclock or {} - request = RequestSubscribe(self, cluster_uuid, server_uuid, vclock) - self._socket.sendall(bytes(request)) - while True: - resp = Response(self, self._read_response()) - yield resp - if resp.code >= REQUEST_TYPE_ERROR: - return - self.close() # close connection after SUBSCRIBE - - def insert(self, space_name, values, on_push=None, on_push_ctx=None): + def insert(self, space_name, values, *, on_push=None, on_push_ctx=None): """ Execute an INSERT request: `insert`_ a tuple to the space. Throws an error if there is already a tuple with the same @@ -1584,11 +1665,14 @@ def insert(self, space_name, values, on_push=None, on_push_ctx=None): :exc:`~tarantool.error.DatabaseError`, :exc:`~tarantool.error.SchemaError`, :exc:`~tarantool.error.NetworkError`, - :exc:`~tarantool.error.SslError` + :exc:`~tarantool.error.SslError`, + :exc:`~tarantool.error.NotSupportedError` .. _insert: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/insert/ """ + self._schemaful_connection_check() + if isinstance(space_name, str): space_name = self.schema.get_space(space_name).sid if on_push is not None and not callable(on_push): @@ -1623,10 +1707,14 @@ def delete(self, space_name, key, *, index=0, on_push=None, on_push_ctx=None): :exc:`~tarantool.error.DatabaseError`, :exc:`~tarantool.error.SchemaError`, :exc:`~tarantool.error.NetworkError`, - :exc:`~tarantool.error.SslError` + :exc:`~tarantool.error.SslError`, + :exc:`~tarantool.error.NotSupportedError` .. _delete: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/delete/ """ + # pylint: disable=too-many-arguments + + self._schemaful_connection_check() key = wrap_key(key) if isinstance(space_name, str): @@ -1682,10 +1770,14 @@ def upsert(self, space_name, tuple_value, op_list, *, index=0, on_push=None, on_ :exc:`~tarantool.error.DatabaseError`, :exc:`~tarantool.error.SchemaError`, :exc:`~tarantool.error.NetworkError`, - :exc:`~tarantool.error.SslError` + :exc:`~tarantool.error.SslError`, + :exc:`~tarantool.error.NotSupportedError` .. _upsert: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/upsert/ """ + # pylint: disable=too-many-arguments + + self._schemaful_connection_check() if isinstance(space_name, str): space_name = self.schema.get_space(space_name).sid @@ -1770,10 +1862,14 @@ def update(self, space_name, key, op_list, *, index=0, on_push=None, on_push_ctx :exc:`~tarantool.error.DatabaseError`, :exc:`~tarantool.error.SchemaError`, :exc:`~tarantool.error.NetworkError`, - :exc:`~tarantool.error.SslError` + :exc:`~tarantool.error.SslError`, + :exc:`~tarantool.error.NotSupportedError` .. _update: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/update/ """ + # pylint: disable=too-many-arguments + + self._schemaful_connection_check() key = wrap_key(key) if isinstance(space_name, str): @@ -1807,15 +1903,16 @@ def ping(self, notime=False): """ request = RequestPing(self) - t0 = time.time() + start_time = time.time() self._send_request(request) - t1 = time.time() + finish_time = time.time() if notime: return "Success" - return t1 - t0 + return finish_time - start_time - def select(self, space_name, key=None, *, offset=0, limit=0xffffffff, index=0, iterator=None, on_push=None, on_push_ctx=None): + def select(self, space_name, key=None, *, offset=0, limit=0xffffffff, index=0, iterator=None, + on_push=None, on_push_ctx=None): """ Execute a SELECT request: `select`_ a tuple from the space. @@ -1949,22 +2046,26 @@ def select(self, space_name, key=None, *, offset=0, limit=0xffffffff, index=0, i :param on_push_ctx: Сontext for working with on_push callback. :type on_push_ctx: optional - + :rtype: :class:`~tarantool.response.Response` :raise: :exc:`~AssertionError`, :exc:`~tarantool.error.DatabaseError`, :exc:`~tarantool.error.SchemaError`, :exc:`~tarantool.error.NetworkError`, - :exc:`~tarantool.error.SslError` + :exc:`~tarantool.error.SslError`, + :exc:`~tarantool.error.NotSupportedError` .. _select: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/select/ """ + # pylint: disable=too-many-arguments + + self._schemaful_connection_check() if iterator is None: iterator = ITERATOR_EQ - if key is None or (isinstance(key, (list, tuple)) and - len(key) == 0): + if key is None or (isinstance(key, (list, tuple)) + and len(key) == 0): iterator = ITERATOR_ALL # Perform smart type checking (scalar / list of scalars / list of @@ -1996,6 +2097,8 @@ def space(self, space_name): :raise: :exc:`~tarantool.error.SchemaError` """ + self._schemaful_connection_check() + return Space(self, space_name) def generate_sync(self): @@ -2076,33 +2179,47 @@ def _check_features(self): :exc:`~tarantool.error.SslError` """ - try: - request = RequestProtocolVersion(self, - CONNECTOR_IPROTO_VERSION, - CONNECTOR_FEATURES) - response = self._send_request(request) - server_protocol_version = response.protocol_version - server_features = response.features - server_auth_type = response.auth_type - except DatabaseError as exc: - ER_UNKNOWN_REQUEST_TYPE = 48 - if exc.code == ER_UNKNOWN_REQUEST_TYPE: - server_protocol_version = None - server_features = [] - server_auth_type = None + if self.version_id >= version_id(2, 10, 0): + try: + request = RequestProtocolVersion(self, + self._client_protocol_version, + self._client_features) + response = self._send_request(request) + self._server_protocol_version = response.protocol_version + self._server_features = response.features + self._server_auth_type = response.auth_type + except DatabaseError as exc: + if exc.code != ER_UNKNOWN_REQUEST_TYPE: + raise exc + + if self.required_protocol_version is not None: + if self._server_protocol_version is None or \ + self._server_protocol_version < self.required_protocol_version: + raise ConfigurationError('Server protocol version is ' + f'{self._server_protocol_version}, ' + f'protocol version {self.required_protocol_version} ' + 'is required') + + if self.required_features is not None: + if self._server_features is None: + failed_features = self.required_features else: - raise exc + failed_features = [val for val in self.required_features + if val not in self._server_features] - if server_protocol_version is not None: - self._protocol_version = min(server_protocol_version, - CONNECTOR_IPROTO_VERSION) + if len(failed_features) > 0: + str_features = ', '.join([str(v) for v in failed_features]) + raise ConfigurationError(f'Server missing protocol features with id {str_features}') - # Intercept lists of features - features_list = [val for val in CONNECTOR_FEATURES if val in server_features] - for val in features_list: - self._features[val] = True + if self._server_protocol_version is not None: + self._protocol_version = min(self._server_protocol_version, + self._client_protocol_version) - self._server_auth_type = server_auth_type + # Intercept lists of features + if self._server_features is not None: + features_list = [val for val in self._client_features if val in self._server_features] + for val in features_list: + self._features[val] = True def _packer_factory(self): return self._packer_factory_impl(self) @@ -2110,9 +2227,10 @@ def _packer_factory(self): def _unpacker_factory(self): return self._unpacker_factory_impl(self) - def crud_insert(self, space_name: str, values: Union[tuple, list], opts: dict={}) -> CrudResult: + def crud_insert(self, space_name: str, values: Union[tuple, list], + opts: Optional[dict] = None) -> CrudResult: """ - Inserts row through the + Inserts row through the `crud `__. :param space_name: The name of the target space. @@ -2132,18 +2250,21 @@ def crud_insert(self, space_name: str, values: Union[tuple, list], opts: dict={} assert isinstance(space_name, str) assert isinstance(values, (tuple, list)) + if opts is None: + opts = {} assert isinstance(opts, dict) crud_resp = call_crud(self, "crud.insert", space_name, values, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) - def crud_insert_object(self, space_name: str, values: dict, opts: dict={}) -> CrudResult: + def crud_insert_object(self, space_name: str, values: dict, + opts: Optional[dict] = None) -> CrudResult: """ - Inserts object row through the + Inserts object row through the `crud `__. :param space_name: The name of the target space. @@ -2163,18 +2284,21 @@ def crud_insert_object(self, space_name: str, values: dict, opts: dict={}) -> Cr assert isinstance(space_name, str) assert isinstance(values, dict) + if opts is None: + opts = {} assert isinstance(opts, dict) crud_resp = call_crud(self, "crud.insert_object", space_name, values, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) - def crud_insert_many(self, space_name: str, values: Union[tuple, list], opts: dict={}) -> CrudResult: + def crud_insert_many(self, space_name: str, values: Union[tuple, list], + opts: Optional[dict] = None) -> CrudResult: """ - Inserts batch rows through the + Inserts batch rows through the `crud `__. :param space_name: The name of the target space. @@ -2194,6 +2318,8 @@ def crud_insert_many(self, space_name: str, values: Union[tuple, list], opts: di assert isinstance(space_name, str) assert isinstance(values, (tuple, list)) + if opts is None: + opts = {} assert isinstance(opts, dict) crud_resp = call_crud(self, "crud.insert_many", space_name, values, opts) @@ -2202,17 +2328,18 @@ def crud_insert_many(self, space_name: str, values: Union[tuple, list], opts: di if crud_resp[0] is not None: res = CrudResult(crud_resp[0]) - if crud_resp[1] is not None: - errs = list() + if len(crud_resp) > 1 and crud_resp[1] is not None: + errs = [] for err in crud_resp[1]: errs.append(CrudError(err)) raise CrudModuleManyError(res, errs) return res - def crud_insert_object_many(self, space_name: str, values: Union[tuple, list], opts: dict={}) -> CrudResult: + def crud_insert_object_many(self, space_name: str, values: Union[tuple, list], + opts: Optional[dict] = None) -> CrudResult: """ - Inserts batch object rows through the + Inserts batch object rows through the `crud `__. :param space_name: The name of the target space. @@ -2232,6 +2359,8 @@ def crud_insert_object_many(self, space_name: str, values: Union[tuple, list], o assert isinstance(space_name, str) assert isinstance(values, (tuple, list)) + if opts is None: + opts = {} assert isinstance(opts, dict) crud_resp = call_crud(self, "crud.insert_object_many", space_name, values, opts) @@ -2240,17 +2369,17 @@ def crud_insert_object_many(self, space_name: str, values: Union[tuple, list], o if crud_resp[0] is not None: res = CrudResult(crud_resp[0]) - if crud_resp[1] is not None: - errs = list() + if len(crud_resp) > 1 and crud_resp[1] is not None: + errs = [] for err in crud_resp[1]: errs.append(CrudError(err)) raise CrudModuleManyError(res, errs) return res - def crud_get(self, space_name: str, key: int, opts: dict={}) -> CrudResult: + def crud_get(self, space_name: str, key: int, opts: Optional[dict] = None) -> CrudResult: """ - Gets row through the + Gets row through the `crud `__. :param space_name: The name of the target space. @@ -2269,18 +2398,21 @@ def crud_get(self, space_name: str, key: int, opts: dict={}) -> CrudResult: """ assert isinstance(space_name, str) + if opts is None: + opts = {} assert isinstance(opts, dict) crud_resp = call_crud(self, "crud.get", space_name, key, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) - def crud_update(self, space_name: str, key: int, operations: list=[], opts: dict={}) -> CrudResult: + def crud_update(self, space_name: str, key: int, operations: Optional[list] = None, + opts: Optional[dict] = None) -> CrudResult: """ - Updates row through the + Updates row through the `crud `__. :param space_name: The name of the target space. @@ -2302,19 +2434,23 @@ def crud_update(self, space_name: str, key: int, operations: list=[], opts: dict """ assert isinstance(space_name, str) + if operations is None: + operations = [] assert isinstance(operations, list) + if opts is None: + opts = {} assert isinstance(opts, dict) crud_resp = call_crud(self, "crud.update", space_name, key, operations, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) - def crud_delete(self, space_name: str, key: int, opts: dict={}) -> CrudResult: + def crud_delete(self, space_name: str, key: int, opts: Optional[dict] = None) -> CrudResult: """ - Deletes row through the + Deletes row through the `crud `__. :param space_name: The name of the target space. @@ -2333,18 +2469,21 @@ def crud_delete(self, space_name: str, key: int, opts: dict={}) -> CrudResult: """ assert isinstance(space_name, str) + if opts is None: + opts = {} assert isinstance(opts, dict) crud_resp = call_crud(self, "crud.delete", space_name, key, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) - def crud_replace(self, space_name: str, values: Union[tuple, list], opts: dict={}) -> CrudResult: + def crud_replace(self, space_name: str, values: Union[tuple, list], + opts: Optional[dict] = None) -> CrudResult: """ - Replaces row through the + Replaces row through the `crud `__. :param space_name: The name of the target space. @@ -2364,18 +2503,21 @@ def crud_replace(self, space_name: str, values: Union[tuple, list], opts: dict={ assert isinstance(space_name, str) assert isinstance(values, (tuple, list)) + if opts is None: + opts = {} assert isinstance(opts, dict) crud_resp = call_crud(self, "crud.replace", space_name, values, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) - def crud_replace_object(self, space_name: str, values: dict, opts: dict={}) -> CrudResult: + def crud_replace_object(self, space_name: str, values: dict, + opts: Optional[dict] = None) -> CrudResult: """ - Replaces object row through the + Replaces object row through the `crud `__. :param space_name: The name of the target space. @@ -2395,18 +2537,21 @@ def crud_replace_object(self, space_name: str, values: dict, opts: dict={}) -> C assert isinstance(space_name, str) assert isinstance(values, dict) + if opts is None: + opts = {} assert isinstance(opts, dict) crud_resp = call_crud(self, "crud.replace_object", space_name, values, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) - def crud_replace_many(self, space_name: str, values: Union[tuple, list], opts: dict={}) -> CrudResult: + def crud_replace_many(self, space_name: str, values: Union[tuple, list], + opts: Optional[dict] = None) -> CrudResult: """ - Replaces batch rows through the + Replaces batch rows through the `crud `__. :param space_name: The name of the target space. @@ -2426,6 +2571,8 @@ def crud_replace_many(self, space_name: str, values: Union[tuple, list], opts: d assert isinstance(space_name, str) assert isinstance(values, (tuple, list)) + if opts is None: + opts = {} assert isinstance(opts, dict) crud_resp = call_crud(self, "crud.replace_many", space_name, values, opts) @@ -2434,17 +2581,18 @@ def crud_replace_many(self, space_name: str, values: Union[tuple, list], opts: d if crud_resp[0] is not None: res = CrudResult(crud_resp[0]) - if crud_resp[1] is not None: - errs = list() + if len(crud_resp) > 1 and crud_resp[1] is not None: + errs = [] for err in crud_resp[1]: errs.append(CrudError(err)) raise CrudModuleManyError(res, errs) return res - def crud_replace_object_many(self, space_name: str, values: Union[tuple, list], opts: dict={}) -> CrudResult: + def crud_replace_object_many(self, space_name: str, values: Union[tuple, list], + opts: Optional[dict] = None) -> CrudResult: """ - Replaces batch object rows through the + Replaces batch object rows through the `crud `__. :param space_name: The name of the target space. @@ -2464,6 +2612,8 @@ def crud_replace_object_many(self, space_name: str, values: Union[tuple, list], assert isinstance(space_name, str) assert isinstance(values, (tuple, list)) + if opts is None: + opts = {} assert isinstance(opts, dict) crud_resp = call_crud(self, "crud.replace_object_many", space_name, values, opts) @@ -2472,17 +2622,18 @@ def crud_replace_object_many(self, space_name: str, values: Union[tuple, list], if crud_resp[0] is not None: res = CrudResult(crud_resp[0]) - if crud_resp[1] is not None: - errs = list() + if len(crud_resp) > 1 and crud_resp[1] is not None: + errs = [] for err in crud_resp[1]: errs.append(CrudError(err)) raise CrudModuleManyError(res, errs) return res - def crud_upsert(self, space_name: str, values: Union[tuple, list], operations: list=[], opts: dict={}) -> CrudResult: + def crud_upsert(self, space_name: str, values: Union[tuple, list], + operations: Optional[list] = None, opts: Optional[dict] = None) -> CrudResult: """ - Upserts row through the + Upserts row through the `crud `__. :param space_name: The name of the target space. @@ -2505,19 +2656,25 @@ def crud_upsert(self, space_name: str, values: Union[tuple, list], operations: l assert isinstance(space_name, str) assert isinstance(values, (tuple, list)) + if operations is None: + operations = [] assert isinstance(operations, list) + if opts is None: + opts = {} assert isinstance(opts, dict) crud_resp = call_crud(self, "crud.upsert", space_name, values, operations, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) - def crud_upsert_object(self, space_name: str, values: dict, operations: list=[], opts: dict={}) -> CrudResult: + def crud_upsert_object(self, space_name: str, values: dict, + operations: Optional[list] = None, + opts: Optional[dict] = None) -> CrudResult: """ - Upserts object row through the + Upserts object row through the `crud `__. :param space_name: The name of the target space. @@ -2540,19 +2697,24 @@ def crud_upsert_object(self, space_name: str, values: dict, operations: list=[], assert isinstance(space_name, str) assert isinstance(values, dict) + if operations is None: + operations = [] assert isinstance(operations, list) + if opts is None: + opts = {} assert isinstance(opts, dict) crud_resp = call_crud(self, "crud.upsert_object", space_name, values, operations, opts) - - if crud_resp[1] is not None: + + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) - def crud_upsert_many(self, space_name: str, values_operation: Union[tuple, list], opts: dict={}) -> CrudResult: + def crud_upsert_many(self, space_name: str, values_operation: Union[tuple, list], + opts: Optional[dict] = None) -> CrudResult: """ - Upserts batch rows through the + Upserts batch rows through the `crud `__. :param space_name: The name of the target space. @@ -2572,6 +2734,8 @@ def crud_upsert_many(self, space_name: str, values_operation: Union[tuple, list] assert isinstance(space_name, str) assert isinstance(values_operation, (tuple, list)) + if opts is None: + opts = {} assert isinstance(opts, dict) crud_resp = call_crud(self, "crud.upsert_many", space_name, values_operation, opts) @@ -2580,17 +2744,18 @@ def crud_upsert_many(self, space_name: str, values_operation: Union[tuple, list] if crud_resp[0] is not None: res = CrudResult(crud_resp[0]) - if crud_resp[1] is not None: - errs = list() + if len(crud_resp) > 1 and crud_resp[1] is not None: + errs = [] for err in crud_resp[1]: errs.append(CrudError(err)) raise CrudModuleManyError(res, errs) return res - def crud_upsert_object_many(self, space_name: str, values_operation: Union[tuple, list], opts: dict={}) -> CrudResult: + def crud_upsert_object_many(self, space_name: str, values_operation: Union[tuple, list], + opts: Optional[dict] = None) -> CrudResult: """ - Upserts batch object rows through the + Upserts batch object rows through the `crud `__. :param space_name: The name of the target space. @@ -2610,6 +2775,8 @@ def crud_upsert_object_many(self, space_name: str, values_operation: Union[tuple assert isinstance(space_name, str) assert isinstance(values_operation, (tuple, list)) + if opts is None: + opts = {} assert isinstance(opts, dict) crud_resp = call_crud(self, "crud.upsert_object_many", space_name, values_operation, opts) @@ -2618,17 +2785,18 @@ def crud_upsert_object_many(self, space_name: str, values_operation: Union[tuple if crud_resp[0] is not None: res = CrudResult(crud_resp[0]) - if crud_resp[1] is not None: - errs = list() + if len(crud_resp) > 1 and crud_resp[1] is not None: + errs = [] for err in crud_resp[1]: errs.append(CrudError(err)) raise CrudModuleManyError(res, errs) return res - def crud_select(self, space_name: str, conditions: list=[], opts: dict={}) -> CrudResult: + def crud_select(self, space_name: str, conditions: Optional[list] = None, + opts: Optional[dict] = None) -> CrudResult: """ - Selects rows through the + Selects rows through the `crud `__. :param space_name: The name of the target space. @@ -2647,19 +2815,23 @@ def crud_select(self, space_name: str, conditions: list=[], opts: dict={}) -> Cr """ assert isinstance(space_name, str) - assert isinstance(conditions, (tuple, list)) + if conditions is None: + conditions = [] + assert isinstance(conditions, list) + if opts is None: + opts = {} assert isinstance(opts, dict) crud_resp = call_crud(self, "crud.select", space_name, conditions, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) - def crud_min(self, space_name: str, index_name: str, opts: dict={}) -> CrudResult: + def crud_min(self, space_name: str, index_name: str, opts: Optional[dict] = None) -> CrudResult: """ - Gets rows with minimum value in the specified index through + Gets rows with minimum value in the specified index through the `crud `__. :param space_name: The name of the target space. @@ -2678,18 +2850,20 @@ def crud_min(self, space_name: str, index_name: str, opts: dict={}) -> CrudResul """ assert isinstance(space_name, str) + if opts is None: + opts = {} assert isinstance(opts, dict) crud_resp = call_crud(self, "crud.min", space_name, index_name, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) - def crud_max(self, space_name: str, index_name: str, opts: dict={}) -> CrudResult: + def crud_max(self, space_name: str, index_name: str, opts: Optional[dict] = None) -> CrudResult: """ - Gets rows with maximum value in the specified index through + Gets rows with maximum value in the specified index through the `crud `__. :param space_name: The name of the target space. @@ -2708,18 +2882,20 @@ def crud_max(self, space_name: str, index_name: str, opts: dict={}) -> CrudResul """ assert isinstance(space_name, str) + if opts is None: + opts = {} assert isinstance(opts, dict) crud_resp = call_crud(self, "crud.max", space_name, index_name, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return CrudResult(crud_resp[0]) - def crud_truncate(self, space_name: str, opts: dict={}) -> bool: + def crud_truncate(self, space_name: str, opts: Optional[dict] = None) -> bool: """ - Truncate rows through + Truncate rows through the `crud `__. :param space_name: The name of the target space. @@ -2735,20 +2911,20 @@ def crud_truncate(self, space_name: str, opts: dict={}) -> bool: """ assert isinstance(space_name, str) + if opts is None: + opts = {} assert isinstance(opts, dict) crud_resp = call_crud(self, "crud.truncate", space_name, opts) - # In absence of an error, crud does not give - # variable err as nil (as in most cases). - if len(crud_resp) != 1: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return crud_resp[0] - - def crud_len(self, space_name: str, opts: dict={}) -> int: + + def crud_len(self, space_name: str, opts: Optional[dict] = None) -> int: """ - Gets the number of tuples in the space through + Gets the number of tuples in the space through the `crud `__. :param space_name: The name of the target space. @@ -2764,20 +2940,20 @@ def crud_len(self, space_name: str, opts: dict={}) -> int: """ assert isinstance(space_name, str) + if opts is None: + opts = {} assert isinstance(opts, dict) crud_resp = call_crud(self, "crud.len", space_name, opts) - # In absence of an error, crud does not give - # variable err as nil (as in most cases). - if len(crud_resp) != 1: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return crud_resp[0] - def crud_storage_info(self, opts: dict={}) -> dict: + def crud_storage_info(self, opts: Optional[dict] = None) -> dict: """ - Gets storages status through the + Gets storages status through the `crud `__. :param opts: The opts for the crud module. @@ -2789,20 +2965,21 @@ def crud_storage_info(self, opts: dict={}) -> dict: :exc:`~tarantool.error.DatabaseError` """ + if opts is None: + opts = {} assert isinstance(opts, dict) crud_resp = call_crud(self, "crud.storage_info", opts) - # In absence of an error, crud does not give - # variable err as nil (as in most cases). - if len(crud_resp) != 1: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return crud_resp[0] - def crud_count(self, space_name: str, conditions: list=[], opts: dict={}) -> int: + def crud_count(self, space_name: str, conditions: Optional[list] = None, + opts: Optional[dict] = None) -> int: """ - Gets rows count through the + Gets rows count through the `crud `__. :param space_name: The name of the target space. @@ -2821,19 +2998,23 @@ def crud_count(self, space_name: str, conditions: list=[], opts: dict={}) -> int """ assert isinstance(space_name, str) - assert isinstance(conditions, (tuple, list)) + if conditions is None: + conditions = [] + assert isinstance(conditions, list) + if opts is None: + opts = {} assert isinstance(opts, dict) crud_resp = call_crud(self, "crud.count", space_name, conditions, opts) - if crud_resp[1] is not None: + if len(crud_resp) > 1 and crud_resp[1] is not None: raise CrudModuleError(None, CrudError(crud_resp[1])) return crud_resp[0] - def crud_stats(self, space_name: str=None) -> CrudResult: + def crud_stats(self, space_name: str = None) -> CrudResult: """ - Gets statistics from the + Gets statistics from the `crud `__. :param space_name: The name of the target space. @@ -2849,6 +3030,7 @@ def crud_stats(self, space_name: str=None) -> CrudResult: crud_resp = call_crud(self, "crud.stats", space_name) + # There are no errors in `crud.stats`. res = None if len(crud_resp.data[0]) > 0: res = CrudResult(crud_resp.data[0]) @@ -2857,7 +3039,7 @@ def crud_stats(self, space_name: str=None) -> CrudResult: def crud_unflatten_rows(self, rows: list, metadata: list) -> list: """ - Makes rows unflatten through the + Makes rows unflatten through the `crud `__. :param rows: The rows to unflatten. diff --git a/tarantool/connection_pool.py b/tarantool/connection_pool.py index 362b7618..f1e675b3 100644 --- a/tarantool/connection_pool.py +++ b/tarantool/connection_pool.py @@ -1,6 +1,7 @@ """ This module provides API for interaction with Tarantool servers cluster. """ +# pylint: disable=too-many-lines,duplicate-code import abc import itertools @@ -18,8 +19,6 @@ POOL_INSTANCE_RECONNECT_MAX_ATTEMPTS, POOL_REFRESH_DELAY, SOCKET_TIMEOUT, - DEFAULT_SSL_PASSWORD, - DEFAULT_SSL_PASSWORD_FILE, ) from tarantool.error import ( ClusterConnectWarning, @@ -93,13 +92,13 @@ class InstanceState(): """ :type: :class:`~tarantool.connection_pool.Status` """ - ro: typing.Optional[bool] = None + read_only: typing.Optional[bool] = None """ :type: :obj:`bool`, optional """ -def QueueFactory(): +def queue_factory(): """ Build a queue-based channel. """ @@ -116,7 +115,7 @@ class PoolUnit(): addr: dict """ - ``{"host": host, "port": port}`` info. + ``{"host": host, "port": port, "socket_fd": socket_fd}`` info. :type: :obj:`dict` """ @@ -126,14 +125,14 @@ class PoolUnit(): :type: :class:`~tarantool.Connection` """ - input_queue: queue.Queue = field(default_factory=QueueFactory) + input_queue: queue.Queue = field(default_factory=queue_factory) """ Channel to pass requests for the server thread. :type: :obj:`queue.Queue` """ - output_queue: queue.Queue = field(default_factory=QueueFactory) + output_queue: queue.Queue = field(default_factory=queue_factory) """ Channel to receive responses from the server thread. @@ -162,21 +161,31 @@ class PoolUnit(): :type: :obj:`bool` """ + def get_address(self): + """ + Get an address string representation. + """ + if self.addr['socket_fd'] is not None: + return f'fd://{self.addr["socket_fd"]}' + return f'{self.addr["host"]}:{self.addr["port"]}' + + # Based on https://realpython.com/python-interface/ class StrategyInterface(metaclass=abc.ABCMeta): """ Defines strategy to choose a pool server based on a request mode. """ + # pylint: disable=bad-option-value,super-init-not-called @classmethod def __subclasshook__(cls, subclass): - return (hasattr(subclass, '__init__') and - callable(subclass.__init__) and - hasattr(subclass, 'update') and - callable(subclass.update) and - hasattr(subclass, 'getnext') and - callable(subclass.getnext) or - NotImplemented) + return (hasattr(subclass, '__init__') + and callable(subclass.__init__) + and hasattr(subclass, 'update') + and callable(subclass.update) + and hasattr(subclass, 'getnext') + and callable(subclass.getnext) + or NotImplemented) @abc.abstractmethod def __init__(self, pool): @@ -205,10 +214,12 @@ def getnext(self, mode): raise NotImplementedError + class RoundRobinStrategy(StrategyInterface): """ Simple round-robin pool servers rotation. """ + # pylint: disable=bad-option-value,no-self-use,super-init-not-called def __init__(self, pool): """ @@ -216,9 +227,9 @@ def __init__(self, pool): :class:`~tarantool.connection_pool.PoolUnit` objects """ - self.ANY_iter = None - self.RW_iter = None - self.RO_iter = None + self.any_iter = None + self.rw_iter = None + self.ro_iter = None self.pool = pool self.rebuild_needed = True @@ -228,9 +239,9 @@ def build(self): based on `box.info.ro`_ state. """ - ANY_pool = [] - RW_pool = [] - RO_pool = [] + any_pool = [] + rw_pool = [] + ro_pool = [] for key in self.pool: state = self.pool[key].state @@ -238,27 +249,27 @@ def build(self): if state.status == Status.UNHEALTHY: continue - ANY_pool.append(key) + any_pool.append(key) - if state.ro == False: - RW_pool.append(key) + if state.read_only is False: + rw_pool.append(key) else: - RO_pool.append(key) + ro_pool.append(key) - if len(ANY_pool) > 0: - self.ANY_iter = itertools.cycle(ANY_pool) + if len(any_pool) > 0: + self.any_iter = itertools.cycle(any_pool) else: - self.ANY_iter = None + self.any_iter = None - if len(RW_pool) > 0: - self.RW_iter = itertools.cycle(RW_pool) + if len(rw_pool) > 0: + self.rw_iter = itertools.cycle(rw_pool) else: - self.RW_iter = None + self.rw_iter = None - if len(RO_pool) > 0: - self.RO_iter = itertools.cycle(RO_pool) + if len(ro_pool) > 0: + self.ro_iter = itertools.cycle(ro_pool) else: - self.RO_iter = None + self.ro_iter = None self.rebuild_needed = False @@ -271,6 +282,27 @@ def update(self): self.rebuild_needed = True + def _getnext_by_mode(self, *iters, err_msg="Can't find healthy instance in pool"): + """ + Get server from prioritized list of iterators. + + :param iters: list of iterators + :type iters: :obj:`list` + + :param err_msg: Error message to raise in case of error. + :type err_msg: :obj:`str` + + :rtype: :class:`~tarantool.connection_pool.PoolUnit` + + :raise: :exc:`~tarantool.error.PoolTolopogyError` + + :meta private: + """ + for itr in iters: + if itr is not None: + return next(itr) + raise PoolTolopogyError(err_msg) + def getnext(self, mode): """ Get server based on the request mode. @@ -287,34 +319,19 @@ def getnext(self, mode): self.build() if mode == Mode.ANY: - if self.ANY_iter is not None: - return next(self.ANY_iter) - else: - raise PoolTolopogyError("Can't find healthy instance in pool") - elif mode == Mode.RW: - if self.RW_iter is not None: - return next(self.RW_iter) - else: - raise PoolTolopogyError("Can't find healthy rw instance in pool") - elif mode == Mode.RO: - if self.RO_iter is not None: - return next(self.RO_iter) - else: - raise PoolTolopogyError("Can't find healthy ro instance in pool") - elif mode == Mode.PREFER_RO: - if self.RO_iter is not None: - return next(self.RO_iter) - elif self.RW_iter is not None: - return next(self.RW_iter) - else: - raise PoolTolopogyError("Can't find healthy instance in pool") - elif mode == Mode.PREFER_RW: - if self.RW_iter is not None: - return next(self.RW_iter) - elif self.RO_iter is not None: - return next(self.RO_iter) - else: - raise PoolTolopogyError("Can't find healthy instance in pool") + return self._getnext_by_mode(self.any_iter) + if mode == Mode.RW: + return self._getnext_by_mode(self.rw_iter, + err_msg="Can't find healthy rw instance in pool") + if mode == Mode.RO: + return self._getnext_by_mode(self.ro_iter, + err_msg="Can't find healthy ro instance in pool") + if mode == Mode.PREFER_RO: + return self._getnext_by_mode(self.ro_iter, self.rw_iter) + if mode == Mode.PREFER_RW: + return self._getnext_by_mode(self.rw_iter, self.ro_iter) + + raise ValueError(f"Unexpected mode {mode}") @dataclass @@ -365,6 +382,7 @@ class ConnectionPool(ConnectionInterface): >>> resp - ['AAAA', 'Alpha'] """ + # pylint: disable=too-many-public-methods,duplicate-code,bad-option-value,no-self-use def __init__(self, addrs, @@ -378,7 +396,8 @@ def __init__(self, call_16=False, connection_timeout=CONNECTION_TIMEOUT, strategy_class=RoundRobinStrategy, - refresh_delay=POOL_REFRESH_DELAY): + refresh_delay=POOL_REFRESH_DELAY, + fetch_schema=True): """ :param addrs: List of dictionaries describing server addresses: @@ -387,6 +406,7 @@ def __init__(self, { "host': "str" or None, # mandatory "port": int or "str", # mandatory + "socket_fd": int, # optional "transport": "str", # optional "ssl_key_file": "str", # optional "ssl_cert_file": "str", # optional @@ -452,6 +472,9 @@ def __init__(self, `box.info.ro`_ status background refreshes, in seconds. :type connection_timeout: :obj:`float`, optional + :param fetch_schema: Refer to + :paramref:`~tarantool.Connection.params.fetch_schema`. + :raise: :exc:`~tarantool.error.ConfigurationError`, :class:`~tarantool.Connection` exceptions @@ -459,6 +482,7 @@ def __init__(self, .. _box.info.status: .. _box.info: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_info/ """ + # pylint: disable=too-many-arguments,too-many-locals,too-many-positional-arguments if not isinstance(addrs, list) or len(addrs) == 0: raise ConfigurationError("addrs must be non-empty list") @@ -484,12 +508,13 @@ def __init__(self, conn=Connection( host=addr['host'], port=addr['port'], + socket_fd=addr['socket_fd'], user=user, password=password, socket_timeout=socket_timeout, reconnect_max_attempts=reconnect_max_attempts, reconnect_delay=reconnect_delay, - connect_now=False, # Connect in ConnectionPool.connect() + connect_now=False, # Connect in ConnectionPool.connect() encoding=encoding, call_16=call_16, connection_timeout=connection_timeout, @@ -500,7 +525,8 @@ def __init__(self, ssl_ciphers=addr['ssl_ciphers'], ssl_password=addr['ssl_password'], ssl_password_file=addr['ssl_password_file'], - auth_type=addr['auth_type']) + auth_type=addr['auth_type'], + fetch_schema=fetch_schema) ) if connect_now: @@ -513,15 +539,16 @@ def _make_key(self, addr): """ Make a unique key for a server based on its address. - :param addr: `{"host": host, "port": port}` dictionary. + :param addr: `{"host": host, "port": port, "socket_fd": socket_fd}` dictionary. :type addr: :obj:`dict` :rtype: :obj:`str` :meta private: """ - - return '{0}:{1}'.format(addr['host'], addr['port']) + if addr['socket_fd'] is None: + return f"{addr['host']}:{addr['port']}" + return addr['socket_fd'] def _get_new_state(self, unit): """ @@ -540,25 +567,25 @@ def _get_new_state(self, unit): if conn.is_closed(): try: conn.connect() - except NetworkError as e: - msg = "Failed to connect to {0}:{1}".format( - unit.addr['host'], unit.addr['port']) + except NetworkError as exc: + msg = (f"Failed to connect to {unit.get_address()}, " + f"reason: {repr(exc)}") warn(msg, ClusterConnectWarning) return InstanceState(Status.UNHEALTHY) try: resp = conn.call('box.info') - except NetworkError as e: - msg = "Failed to get box.info for {0}:{1}, reason: {2}".format( - unit.addr['host'], unit.addr['port'], repr(e)) + except NetworkError as exc: + msg = (f"Failed to get box.info for {unit.get_address()}, " + f"reason: {repr(exc)}") warn(msg, PoolTolopogyWarning) return InstanceState(Status.UNHEALTHY) try: - ro = resp.data[0]['ro'] - except (IndexError, KeyError) as e: - msg = "Incorrect box.info response from {0}:{1}".format( - unit.addr['host'], unit.addr['port']) + read_only = resp.data[0]['ro'] + except (IndexError, KeyError) as exc: + msg = (f"Incorrect box.info response from {unit.get_address()}" + f"reason: {repr(exc)}") warn(msg, PoolTolopogyWarning) return InstanceState(Status.UNHEALTHY) @@ -566,17 +593,16 @@ def _get_new_state(self, unit): status = resp.data[0]['status'] if status != 'running': - msg = "{0}:{1} instance status is not 'running'".format( - unit.addr['host'], unit.addr['port']) + msg = f"{unit.get_address()} instance status is not 'running'" warn(msg, PoolTolopogyWarning) return InstanceState(Status.UNHEALTHY) - except (IndexError, KeyError) as e: - msg = "Incorrect box.info response from {0}:{1}".format( - unit.addr['host'], unit.addr['port']) + except (IndexError, KeyError) as exc: + msg = (f"Incorrect box.info response from {unit.get_address()}" + f"reason: {repr(exc)}") warn(msg, PoolTolopogyWarning) return InstanceState(Status.UNHEALTHY) - return InstanceState(Status.HEALTHY, ro) + return InstanceState(Status.HEALTHY, read_only) def _refresh_state(self, key): """ @@ -615,7 +641,7 @@ def is_closed(self): :rtype: :obj:`bool` """ - return all(unit.request_processing_enabled == False for unit in self.pool.values()) + return all(unit.request_processing_enabled is False for unit in self.pool.values()) def _request_process_loop(self, key, unit, last_refresh): """ @@ -639,8 +665,8 @@ def _request_process_loop(self, key, unit, last_refresh): method = getattr(Connection, task.method_name) try: resp = method(unit.conn, *task.args, **task.kwargs) - except Exception as e: - unit.output_queue.put(e) + except Exception as exc: # pylint: disable=bad-option-value,broad-exception-caught,broad-except + unit.output_queue.put(exc) else: unit.output_queue.put(resp) @@ -663,9 +689,7 @@ def connect(self): and refresh the info would be processed in the background. """ - for key in self.pool: - unit = self.pool[key] - + for key, unit in self.pool.items(): self._refresh_state(key) last_refresh = time.time() @@ -807,8 +831,10 @@ def replace(self, space_name, values, *, mode=Mode.RW, on_push=None, on_push_ctx .. _replace: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/replace/ """ + # pylint: disable=too-many-arguments - return self._send(mode, 'replace', space_name, values, on_push=on_push, on_push_ctx=on_push_ctx) + return self._send(mode, 'replace', space_name, values, + on_push=on_push, on_push_ctx=on_push_ctx) def insert(self, space_name, values, *, mode=Mode.RW, on_push=None, on_push_ctx=None): """ @@ -836,8 +862,10 @@ def insert(self, space_name, values, *, mode=Mode.RW, on_push=None, on_push_ctx= .. _insert: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/insert/ """ + # pylint: disable=too-many-arguments - return self._send(mode, 'insert', space_name, values, on_push=on_push, on_push_ctx=on_push_ctx) + return self._send(mode, 'insert', space_name, values, + on_push=on_push, on_push_ctx=on_push_ctx) def delete(self, space_name, key, *, index=0, mode=Mode.RW, on_push=None, on_push_ctx=None): """ @@ -868,10 +896,13 @@ def delete(self, space_name, key, *, index=0, mode=Mode.RW, on_push=None, on_pus .. _delete: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/delete/ """ + # pylint: disable=too-many-arguments - return self._send(mode, 'delete', space_name, key, index=index, on_push=on_push, on_push_ctx=on_push_ctx) + return self._send(mode, 'delete', space_name, key, index=index, + on_push=on_push, on_push_ctx=on_push_ctx) - def upsert(self, space_name, tuple_value, op_list, *, index=0, mode=Mode.RW, on_push=None, on_push_ctx=None): + def upsert(self, space_name, tuple_value, op_list, *, index=0, mode=Mode.RW, + on_push=None, on_push_ctx=None): """ Execute an UPSERT request on the pool server: `upsert`_ a tuple to the space. Refer to :meth:`~tarantool.Connection.upsert`. @@ -903,11 +934,13 @@ def upsert(self, space_name, tuple_value, op_list, *, index=0, mode=Mode.RW, on_ .. _upsert: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/upsert/ """ + # pylint: disable=too-many-arguments return self._send(mode, 'upsert', space_name, tuple_value, - op_list, index=index, on_push=on_push, on_push_ctx=on_push_ctx) + op_list, index=index, on_push=on_push, on_push_ctx=on_push_ctx) - def update(self, space_name, key, op_list, *, index=0, mode=Mode.RW, on_push=None, on_push_ctx=None): + def update(self, space_name, key, op_list, *, index=0, mode=Mode.RW, + on_push=None, on_push_ctx=None): """ Execute an UPDATE request on the pool server: `update`_ a tuple in the space. Refer to :meth:`~tarantool.Connection.update`. @@ -939,9 +972,10 @@ def update(self, space_name, key, op_list, *, index=0, mode=Mode.RW, on_push=Non .. _update: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/update/ """ + # pylint: disable=too-many-arguments - return self._send(mode, 'update', space_name, key, - op_list, index=index, on_push=on_push, on_push_ctx=on_push_ctx) + return self._send(mode, 'update', space_name, key, + op_list, index=index, on_push=on_push, on_push_ctx=on_push_ctx) def ping(self, notime=False, *, mode=None): """ @@ -1005,6 +1039,7 @@ def select(self, space_name, key, *, offset=0, limit=0xffffffff, .. _select: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/select/ """ + # pylint: disable=too-many-arguments return self._send(mode, 'select', space_name, key, offset=offset, limit=limit, index=index, iterator=iterator, on_push=on_push, on_push_ctx=on_push_ctx) @@ -1034,10 +1069,10 @@ def execute(self, query, params=None, *, mode=None): return self._send(mode, 'execute', query, params) - def crud_insert(self, space_name, values, opts={}, *, mode=Mode.ANY): + def crud_insert(self, space_name, values, opts=None, *, mode=Mode.ANY): """ - Execute an crud_insert request on the pool server: - inserts row through the + Execute an crud_insert request on the pool server: + inserts row through the `crud `__. Refer to :meth:`~tarantool.Connection.crud_insert`. @@ -1061,10 +1096,10 @@ def crud_insert(self, space_name, values, opts={}, *, mode=Mode.ANY): return self._send(mode, 'crud_insert', space_name, values, opts) - def crud_insert_object(self, space_name, values, opts={}, *, mode=Mode.ANY): + def crud_insert_object(self, space_name, values, opts=None, *, mode=Mode.ANY): """ - Execute an crud_insert_object request on the pool server: - inserts object row through the + Execute an crud_insert_object request on the pool server: + inserts object row through the `crud `__. Refer to :meth:`~tarantool.Connection.crud_insert_object`. @@ -1088,10 +1123,10 @@ def crud_insert_object(self, space_name, values, opts={}, *, mode=Mode.ANY): return self._send(mode, 'crud_insert_object', space_name, values, opts) - def crud_insert_many(self, space_name, values, opts={}, *, mode=Mode.ANY): + def crud_insert_many(self, space_name, values, opts=None, *, mode=Mode.ANY): """ - Execute an crud_insert_many request on the pool server: - inserts batch rows through the + Execute an crud_insert_many request on the pool server: + inserts batch rows through the `crud `__. Refer to :meth:`~tarantool.Connection.crud_insert_many`. @@ -1115,9 +1150,9 @@ def crud_insert_many(self, space_name, values, opts={}, *, mode=Mode.ANY): return self._send(mode, 'crud_insert_many', space_name, values, opts) - def crud_insert_object_many(self, space_name, values, opts={}, *, mode=Mode.ANY): + def crud_insert_object_many(self, space_name, values, opts=None, *, mode=Mode.ANY): """ - Execute an crud_insert_object_many request on the pool server: + Execute an crud_insert_object_many request on the pool server: inserts batch object rows through the `crud `__. Refer to :meth:`~tarantool.Connection.crud_insert_object_many`. @@ -1142,10 +1177,10 @@ def crud_insert_object_many(self, space_name, values, opts={}, *, mode=Mode.ANY) return self._send(mode, 'crud_insert_object_many', space_name, values, opts) - def crud_get(self, space_name, key, opts={}, *, mode=Mode.ANY): + def crud_get(self, space_name, key, opts=None, *, mode=Mode.ANY): """ - Execute an crud_get request on the pool server: - gets row through the + Execute an crud_get request on the pool server: + gets row through the `crud `__. Refer to :meth:`~tarantool.Connection.crud_get`. @@ -1169,10 +1204,10 @@ def crud_get(self, space_name, key, opts={}, *, mode=Mode.ANY): return self._send(mode, 'crud_get', space_name, key, opts) - def crud_update(self, space_name, key, operations=[], opts={}, *, mode=Mode.ANY): + def crud_update(self, space_name, key, operations=None, opts=None, *, mode=Mode.ANY): """ - Execute an crud_update request on the pool server: - updates row through the + Execute an crud_update request on the pool server: + updates row through the `crud `__. Refer to :meth:`~tarantool.Connection.crud_update`. @@ -1196,13 +1231,14 @@ def crud_update(self, space_name, key, operations=[], opts={}, *, mode=Mode.ANY) :raise: :exc:`~tarantool.error.CrudModuleError`, :exc:`~tarantool.error.DatabaseError` """ + # pylint: disable=too-many-arguments return self._send(mode, 'crud_update', space_name, key, operations, opts) - def crud_delete(self, space_name, key, opts={}, *, mode=Mode.ANY): + def crud_delete(self, space_name, key, opts=None, *, mode=Mode.ANY): """ - Execute an crud_delete request on the pool server: - deletes row through the + Execute an crud_delete request on the pool server: + deletes row through the `crud `__. Refer to :meth:`~tarantool.Connection.crud_delete`. @@ -1226,10 +1262,10 @@ def crud_delete(self, space_name, key, opts={}, *, mode=Mode.ANY): return self._send(mode, 'crud_delete', space_name, key, opts) - def crud_replace(self, space_name, values, opts={}, *, mode=Mode.ANY): + def crud_replace(self, space_name, values, opts=None, *, mode=Mode.ANY): """ - Execute an crud_replace request on the pool server: - replaces row through the + Execute an crud_replace request on the pool server: + replaces row through the `crud `__. Refer to :meth:`~tarantool.Connection.crud_replace`. @@ -1253,10 +1289,10 @@ def crud_replace(self, space_name, values, opts={}, *, mode=Mode.ANY): return self._send(mode, 'crud_replace', space_name, values, opts) - def crud_replace_object(self, space_name, values, opts={}, *, mode=Mode.ANY): + def crud_replace_object(self, space_name, values, opts=None, *, mode=Mode.ANY): """ - Execute an crud_replace_object request on the pool server: - replaces object row through the + Execute an crud_replace_object request on the pool server: + replaces object row through the `crud `__. Refer to :meth:`~tarantool.Connection.crud_replace_object`. @@ -1280,10 +1316,10 @@ def crud_replace_object(self, space_name, values, opts={}, *, mode=Mode.ANY): return self._send(mode, 'crud_replace_object', space_name, values, opts) - def crud_replace_many(self, space_name, values, opts={}, *, mode=Mode.ANY): + def crud_replace_many(self, space_name, values, opts=None, *, mode=Mode.ANY): """ - Execute an crud_replace_many request on the pool server: - replaces batch rows through the + Execute an crud_replace_many request on the pool server: + replaces batch rows through the `crud `__. Refer to :meth:`~tarantool.Connection.crud_replace_many`. @@ -1307,10 +1343,10 @@ def crud_replace_many(self, space_name, values, opts={}, *, mode=Mode.ANY): return self._send(mode, 'crud_replace_many', space_name, values, opts) - def crud_replace_object_many(self, space_name, values, opts={}, *, mode=Mode.ANY): + def crud_replace_object_many(self, space_name, values, opts=None, *, mode=Mode.ANY): """ - Execute an crud_replace_object_many request on the pool server: - replaces batch object rows through the + Execute an crud_replace_object_many request on the pool server: + replaces batch object rows through the `crud `__. Refer to :meth:`~tarantool.Connection.crud_replace_object_many`. @@ -1334,10 +1370,10 @@ def crud_replace_object_many(self, space_name, values, opts={}, *, mode=Mode.ANY return self._send(mode, 'crud_replace_object_many', space_name, values, opts) - def crud_upsert(self, space_name, values, operations=[], opts={}, *, mode=Mode.ANY): + def crud_upsert(self, space_name, values, operations=None, opts=None, *, mode=Mode.ANY): """ - Execute an crud_upsert request on the pool server: - upserts row through the + Execute an crud_upsert request on the pool server: + upserts row through the `crud `__. Refer to :meth:`~tarantool.Connection.crud_upsert`. @@ -1361,13 +1397,14 @@ def crud_upsert(self, space_name, values, operations=[], opts={}, *, mode=Mode.A :raise: :exc:`~tarantool.error.CrudModuleError`, :exc:`~tarantool.error.DatabaseError` """ + # pylint: disable=too-many-arguments return self._send(mode, 'crud_upsert', space_name, values, operations, opts) - def crud_upsert_object(self, space_name, values, operations=[], opts={}, *, mode=Mode.ANY): + def crud_upsert_object(self, space_name, values, operations=None, opts=None, *, mode=Mode.ANY): """ - Execute an crud_upsert_object request on the pool server: - upserts object row through the + Execute an crud_upsert_object request on the pool server: + upserts object row through the `crud `__. Refer to :meth:`~tarantool.Connection.crud_upsert_object`. @@ -1391,13 +1428,14 @@ def crud_upsert_object(self, space_name, values, operations=[], opts={}, *, mode :raise: :exc:`~tarantool.error.CrudModuleError`, :exc:`~tarantool.error.DatabaseError` """ + # pylint: disable=too-many-arguments return self._send(mode, 'crud_upsert_object', space_name, values, operations, opts) - def crud_upsert_many(self, space_name, values_operation, opts={}, *, mode=Mode.ANY): + def crud_upsert_many(self, space_name, values_operation, opts=None, *, mode=Mode.ANY): """ - Execute an crud_upsert_many request on the pool server: - upserts batch rows through the + Execute an crud_upsert_many request on the pool server: + upserts batch rows through the `crud `__. Refer to :meth:`~tarantool.Connection.crud_upsert_many`. @@ -1421,10 +1459,10 @@ def crud_upsert_many(self, space_name, values_operation, opts={}, *, mode=Mode.A return self._send(mode, 'crud_upsert_many', space_name, values_operation, opts) - def crud_upsert_object_many(self, space_name, values_operation, opts={}, *, mode=Mode.ANY): + def crud_upsert_object_many(self, space_name, values_operation, opts=None, *, mode=Mode.ANY): """ - Execute an crud_upsert_object_many request on the pool server: - upserts batch object rows through the + Execute an crud_upsert_object_many request on the pool server: + upserts batch object rows through the `crud `__. Refer to :meth:`~tarantool.Connection.crud_upsert_object_many`. @@ -1448,10 +1486,10 @@ def crud_upsert_object_many(self, space_name, values_operation, opts={}, *, mode return self._send(mode, 'crud_upsert_object_many', space_name, values_operation, opts) - def crud_select(self, space_name, conditions=[], opts={}, *, mode=Mode.ANY): + def crud_select(self, space_name, conditions=None, opts=None, *, mode=Mode.ANY): """ - Execute an crud_select request on the pool server: - selects rows through the + Execute an crud_select request on the pool server: + selects rows through the `crud `__. Refer to :meth:`~tarantool.Connection.crud_select`. @@ -1475,10 +1513,10 @@ def crud_select(self, space_name, conditions=[], opts={}, *, mode=Mode.ANY): return self._send(mode, 'crud_select', space_name, conditions, opts) - def crud_min(self, space_name, index_name, opts={}, *, mode=Mode.ANY): + def crud_min(self, space_name, index_name, opts=None, *, mode=Mode.ANY): """ - Execute an crud_min request on the pool server: - gets rows with minimum value in the specified index through + Execute an crud_min request on the pool server: + gets rows with minimum value in the specified index through `crud `__. Refer to :meth:`~tarantool.Connection.crud_min`. @@ -1502,10 +1540,10 @@ def crud_min(self, space_name, index_name, opts={}, *, mode=Mode.ANY): return self._send(mode, 'crud_min', space_name, index_name, opts) - def crud_max(self, space_name, index_name, opts={}, *, mode=Mode.ANY): + def crud_max(self, space_name, index_name, opts=None, *, mode=Mode.ANY): """ - Execute an crud_max request on the pool server: - gets rows with maximum value in the specified index through + Execute an crud_max request on the pool server: + gets rows with maximum value in the specified index through `crud `__. Refer to :meth:`~tarantool.Connection.crud_max`. @@ -1529,10 +1567,10 @@ def crud_max(self, space_name, index_name, opts={}, *, mode=Mode.ANY): return self._send(mode, 'crud_max', space_name, index_name, opts) - def crud_len(self, space_name, opts={}, *, mode=Mode.ANY): + def crud_len(self, space_name, opts=None, *, mode=Mode.ANY): """ - Execute an crud_len request on the pool server: - gets the number of tuples in the space through + Execute an crud_len request on the pool server: + gets the number of tuples in the space through `crud `__. Refer to :meth:`~tarantool.Connection.crud_len`. @@ -1553,10 +1591,10 @@ def crud_len(self, space_name, opts={}, *, mode=Mode.ANY): return self._send(mode, 'crud_len', space_name, opts) - def crud_storage_info(self, opts={}, *, mode=Mode.ANY): + def crud_storage_info(self, opts=None, *, mode=Mode.ANY): """ - Execute an crud_storage_info request on the pool server: - gets storages status through the + Execute an crud_storage_info request on the pool server: + gets storages status through the `crud `__. Refer to :meth:`~tarantool.Connection.crud_storage_info`. @@ -1574,10 +1612,10 @@ def crud_storage_info(self, opts={}, *, mode=Mode.ANY): return self._send(mode, 'crud_storage_info', opts) - def crud_count(self, space_name, conditions=[], opts={}, *, mode=Mode.ANY): + def crud_count(self, space_name, conditions=None, opts=None, *, mode=Mode.ANY): """ - Execute an crud_count request on the pool server: - gets rows count through the + Execute an crud_count request on the pool server: + gets rows count through the `crud `__. Refer to :meth:`~tarantool.Connection.crud_count`. @@ -1603,8 +1641,8 @@ def crud_count(self, space_name, conditions=[], opts={}, *, mode=Mode.ANY): def crud_stats(self, space_name=None, *, mode=Mode.ANY): """ - Execute an crud_stats request on the pool server: - gets statistics through the + Execute an crud_stats request on the pool server: + gets statistics through the `crud `__. Refer to :meth:`~tarantool.Connection.crud_stats`. @@ -1624,7 +1662,7 @@ def crud_stats(self, space_name=None, *, mode=Mode.ANY): def crud_unflatten_rows(self, rows, metadata, *, mode=Mode.ANY): """ - Makes rows unflatten through the + Makes rows unflatten through the `crud `__. Refer to :meth:`~tarantool.Connection.crud_unflatten_rows`. @@ -1645,10 +1683,10 @@ def crud_unflatten_rows(self, rows, metadata, *, mode=Mode.ANY): return self._send(mode, 'crud_unflatten_rows', rows, metadata) - def crud_truncate(self, space_name, opts={}, *, mode=Mode.ANY): + def crud_truncate(self, space_name, opts=None, *, mode=Mode.ANY): """ - Execute an crud_truncate request on the pool server: - truncates rows through + Execute an crud_truncate request on the pool server: + truncates rows through `crud `__. Refer to :meth:`~tarantool.Connection.crud_truncate`. diff --git a/tarantool/const.py b/tarantool/const.py index 6b12598d..53749cec 100644 --- a/tarantool/const.py +++ b/tarantool/const.py @@ -1,4 +1,6 @@ -# pylint: disable=C0301,W0105,W0401,W0614 +""" +This module a set of constants for the package. +""" IPROTO_REQUEST_TYPE = 0x00 IPROTO_SYNC = 0x01 @@ -97,7 +99,16 @@ IPROTO_FEATURE_TRANSACTIONS = 1 IPROTO_FEATURE_ERROR_EXTENSION = 2 IPROTO_FEATURE_WATCHERS = 3 +IPROTO_FEATURE_PAGINATION = 4 +IPROTO_FEATURE_SPACE_AND_INDEX_NAMES = 5 +IPROTO_FEATURE_WATCH_ONCE = 6 +# Default value for host. +DEFAULT_HOST = None +# Default value for port. +DEFAULT_PORT = None +# Default value for socket_fd. +DEFAULT_SOCKET_FD = None # Default value for connection timeout (seconds) CONNECTION_TIMEOUT = None # Default value for socket timeout (seconds) @@ -131,10 +142,10 @@ # Default delay between attempts to reconnect (seconds) POOL_INSTANCE_RECONNECT_DELAY = 0 -# Tarantool 2.10 protocol version is 3 -CONNECTOR_IPROTO_VERSION = 3 +# Tarantool master 970ea48 protocol version is 6 +CONNECTOR_IPROTO_VERSION = 6 # List of connector-supported features -CONNECTOR_FEATURES = [IPROTO_FEATURE_ERROR_EXTENSION,] +CONNECTOR_FEATURES = [IPROTO_FEATURE_ERROR_EXTENSION] # Authenticate with CHAP-SHA1 (Tarantool CE and EE) AUTH_TYPE_CHAP_SHA1 = "chap-sha1" diff --git a/tarantool/crud.py b/tarantool/crud.py index dee1be36..3a642564 100644 --- a/tarantool/crud.py +++ b/tarantool/crud.py @@ -7,13 +7,14 @@ from tarantool.error import DatabaseError, ER_NO_SUCH_PROC, ER_ACCESS_DENIED -class CrudResponse(object): +class CrudResponse(): """ - Contains response fields from the `crud`_ module that correspond + Contains response fields from the `crud`_ module that correspond to the Lua implementation. .. _crud: https://github.com/tarantool/crud/ """ + # pylint: disable=too-few-public-methods def __init__(self, response): """ @@ -33,16 +34,16 @@ def __init__(self, response): raise RuntimeError('Unable to decode response to object due to unknown type') -class CrudResult(CrudResponse): +class CrudResult(CrudResponse): # pylint: disable=too-few-public-methods """ - Contains result's fields from result variable + Contains result's fields from result variable of crud module operation. """ -class CrudError(CrudResponse): +class CrudError(CrudResponse): # pylint: disable=too-few-public-methods """ - Contains error's fields from error variable + Contains error's fields from error variable of crud module operation. """ @@ -64,9 +65,10 @@ def call_crud(conn, *args): try: crud_resp = conn.call(*args) - except DatabaseError as e: - if e.code == ER_NO_SUCH_PROC or e.code == ER_ACCESS_DENIED: + except DatabaseError as exc: + if exc.code in (ER_NO_SUCH_PROC, ER_ACCESS_DENIED): exc_msg = ". Ensure that you're calling crud.router and user has sufficient grants" - raise DatabaseError(e.code, e.message + exc_msg, extra_info=e.extra_info) from e + raise DatabaseError(exc.code, exc.message + exc_msg, extra_info=exc.extra_info) from exc + raise exc return crud_resp diff --git a/tarantool/dbapi.py b/tarantool/dbapi.py index f6f5d2ab..3b4d1013 100644 --- a/tarantool/dbapi.py +++ b/tarantool/dbapi.py @@ -3,11 +3,24 @@ .. _PEP-249: http://www.python.org/dev/peps/pep-0249/ """ +# pylint: disable=fixme,unused-import,bad-option-value,no-self-use +# flake8: noqa: F401 from tarantool.connection import Connection as BaseConnection -from tarantool.error import * - - +from tarantool.error import ( + Error, + InterfaceError, + DatabaseError, + OperationalError, + IntegrityError, + InternalError, + ProgrammingError, + NotSupportedError, +) + +Warning = Warning # pylint: disable=redefined-builtin,self-assigning-variable + +# pylint: disable=invalid-name paramstyle = 'named' apilevel = "2.0" threadsafety = 1 @@ -50,6 +63,11 @@ def callproc(self, procname, *params): @property def rows(self): + """ + Returns the current count of rows in cursor. + + :type: :obj:`int` + """ return self._rows @property @@ -278,7 +296,7 @@ def __init__(self, *args, **kwargs): :raise: :class:`~tarantool.Connection` exceptions """ - super(Connection, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self._set_autocommit(kwargs.get('autocommit', True)) def _set_autocommit(self, autocommit): @@ -291,7 +309,7 @@ def _set_autocommit(self, autocommit): if not isinstance(autocommit, bool): raise InterfaceError("autocommit parameter must be boolean, " - "not %s" % autocommit.__class__.__name__) + f"not {autocommit.__class__.__name__}") if autocommit is False: raise NotSupportedError("The connector supports " "only autocommit mode") @@ -337,7 +355,7 @@ def close(self): """ self._check_not_closed("The closed connector can not be closed again.") - super(Connection, self).close() + super().close() def commit(self): """ diff --git a/tarantool/error.py b/tarantool/error.py index 29c2430f..85e30c03 100644 --- a/tarantool/error.py +++ b/tarantool/error.py @@ -1,4 +1,3 @@ -# pylint: disable=C0301,W0105,W0401,W0614 """ Python DB API compatible exceptions, see `PEP-249`_. @@ -7,22 +6,10 @@ import os import socket -try: - import ssl - is_ssl_supported = True -except ImportError: - is_ssl_supported = False import sys import warnings - -class Warning(Exception): - """ - Exception raised for important warnings - like data truncations while inserting, etc. - """ - class Error(Exception): """ Base class for error exceptions. @@ -124,6 +111,7 @@ class ConfigurationError(Error): Error of initialization with a user-provided configuration. """ + class MsgpackError(Error): """ Error with encoding or decoding of `MP_EXT`_ types. @@ -131,11 +119,13 @@ class MsgpackError(Error): .. _MP_EXT: https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/ """ + class MsgpackWarning(UserWarning): """ Warning with encoding or decoding of `MP_EXT`_ types. """ + # Monkey patch os.strerror for win32 if sys.platform == "win32": # Windows Sockets Error Codes (not all, but related on network errors) @@ -194,8 +184,8 @@ def os_strerror_patched(code): message = os_strerror_orig(code) if not message.startswith("Unknown"): return message - else: - return _code2str.get(code, "Unknown error %s" % code) + + return _code2str.get(code, f"Unknown error {code}") os.strerror = os_strerror_patched del os_strerror_patched @@ -211,7 +201,7 @@ def __init__(self, value): :param value: Error value. """ - super(SchemaError, self).__init__(0, value) + super().__init__(0, value) self.value = value def __str__(self): @@ -232,7 +222,7 @@ def __init__(self, message, schema_version): :type schema_version: :obj:`int` """ - super(SchemaReloadException, self).__init__(109, message) + super().__init__(109, message) self.schema_version = schema_version def __str__(self): @@ -244,72 +234,79 @@ class NetworkError(DatabaseError): Error related to network. """ - def __init__(self, orig_exception=None, *args, **kwargs): + def __init__(self, *args, **kwargs): """ - :param orig_exception: Exception to wrap. - :type orig_exception: optional + :param args: Exception arguments. If the first argument is + a socket exception, it is wrapped. + :type args: :obj:`tuple`, optional - :param args: Wrapped exception arguments. - :type args: :obj:`tuple` + :param kwargs: Exception to wrap. + :type args: :obj:`dict`, optional """ self.errno = 0 - if hasattr(orig_exception, 'errno'): - self.errno = orig_exception.errno - if orig_exception: + + if len(args) > 0: + orig_exception = args[0] + + if hasattr(orig_exception, 'errno'): + self.errno = orig_exception.errno + if isinstance(orig_exception, socket.timeout): self.message = "Socket timeout" - super(NetworkError, self).__init__(0, self.message) - elif isinstance(orig_exception, socket.error): + super().__init__(0, self.message) + return + + if isinstance(orig_exception, socket.error): self.message = os.strerror(orig_exception.errno) - super(NetworkError, self).__init__( - orig_exception.errno, self.message) - else: - super(NetworkError, self).__init__(orig_exception, *args, **kwargs) + super().__init__(orig_exception.errno, self.message) + return + + super().__init__(*args, **kwargs) class NetworkWarning(UserWarning): """ Warning related to network. """ - pass + class SslError(DatabaseError): """ Error related to SSL. """ - def __init__(self, orig_exception=None, *args): + def __init__(self, *args, **kwargs): """ - :param orig_exception: Exception to wrap. - :type orig_exception: optional + :param args: Exception arguments. If the first argument is + a socket exception, it is wrapped. + :type args: :obj:`tuple`, optional - :param args: Wrapped exception arguments. - :type args: :obj:`tuple` + :param kwargs: Exception to wrap. + :type args: :obj:`dict`, optional """ self.errno = 0 - if hasattr(orig_exception, 'errno'): - self.errno = orig_exception.errno - if orig_exception: - if is_ssl_supported and isinstance(orig_exception, ssl.SSLError): - super(SslError, self).__init__(orig_exception, *args) - else: - super(SslError, self).__init__(orig_exception, *args) + + if len(args) > 0: + orig_exception = args[0] + + if hasattr(orig_exception, 'errno'): + self.errno = orig_exception.errno + + super().__init__(*args, **kwargs) class ClusterDiscoveryWarning(UserWarning): """ Warning related to cluster discovery. """ - pass class ClusterConnectWarning(UserWarning): """ Warning related to cluster pool connection. """ - pass class PoolTolopogyWarning(UserWarning): @@ -317,7 +314,6 @@ class PoolTolopogyWarning(UserWarning): Warning related to unsatisfying `box.info.ro`_ state of pool instances. """ - pass class PoolTolopogyError(DatabaseError): @@ -327,7 +323,6 @@ class PoolTolopogyError(DatabaseError): .. _box.info.ro: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_info/ """ - pass class CrudModuleError(DatabaseError): @@ -341,12 +336,12 @@ class CrudModuleError(DatabaseError): def __init__(self, _, error): """ Sets fields with result and errors. - + :param args: The tuple from the crud module with result and errors. :type args: :obj:`tuple` """ - super(CrudModuleError, self).__init__(0, error.err) + super().__init__(0, error.err) # Sets tarantool.crud.CrudError object. self.extra_info_error = error @@ -362,13 +357,13 @@ class CrudModuleManyError(DatabaseError): def __init__(self, success, error): """ Sets fields with result and errors. - + :param args: The tuple from the crud module with result and errors. :type args: :obj:`tuple` """ exc_msg = "Got multiple errors, see errors_list and success_list" - super(CrudModuleManyError, self).__init__(0, exc_msg) + super().__init__(0, exc_msg) # Sets list of tarantool.crud.CrudResult objects. self.success_list = success # Sets list of tarantool.crud.CrudError objects. @@ -390,8 +385,9 @@ def warn(message, warning_class): :param warning_class: Warning class. :type warning_class: :class:`~tarantool.error.Warning` """ + # pylint: disable=protected-access - frame = sys._getframe(2) # pylint: disable=W0212 + frame = sys._getframe(2) module_name = frame.f_globals.get("__name__") line_no = frame.f_lineno warnings.warn_explicit(message, warning_class, module_name, line_no) diff --git a/tarantool/mesh_connection.py b/tarantool/mesh_connection.py index cc4def5a..02a20d7d 100644 --- a/tarantool/mesh_connection.py +++ b/tarantool/mesh_connection.py @@ -1,6 +1,7 @@ """ This module provides API for interaction with Tarantool servers cluster. """ +# pylint: disable=fixme,duplicate-code import time @@ -27,6 +28,9 @@ DEFAULT_SSL_PASSWORD, DEFAULT_SSL_PASSWORD_FILE, CLUSTER_DISCOVERY_DELAY, + DEFAULT_HOST, + DEFAULT_SOCKET_FD, + DEFAULT_PORT, ) from tarantool.request import ( @@ -34,6 +38,9 @@ ) default_addr_opts = { + 'host': DEFAULT_HOST, + 'port': DEFAULT_PORT, + 'socket_fd': DEFAULT_SOCKET_FD, 'transport': DEFAULT_TRANSPORT, 'ssl_key_file': DEFAULT_SSL_KEY_FILE, 'ssl_cert_file': DEFAULT_SSL_CERT_FILE, @@ -58,10 +65,11 @@ def parse_uri(uri): :rtype: first value: :obj:`dict` or ``None``, second value: ``None`` or :obj:`str` """ + # pylint: disable=too-many-branches # TODO: Support Unix sockets. def parse_error(uri, msg): - msg = 'URI "%s": %s' % (uri, msg) + msg = f'URI "{uri}": {msg}' return None, msg if not uri: @@ -88,17 +96,18 @@ def parse_error(uri, msg): except ValueError: return parse_error(uri, 'port should be a number') - for k, v in default_addr_opts.items(): - result[k] = v + for key, val in default_addr_opts.items(): + if key not in result: + result[key] = val if opts_str != "": for opt_str in opts_str.split('&'): opt = opt_str.split('=') if len(opt) != 2: continue - for k in default_addr_opts: - if k == opt[0]: - result[k] = opt[1] + for key in default_addr_opts: + if key == opt[0]: + result[key] = opt[1] return result, None @@ -117,23 +126,32 @@ def prepare_address(address): :rtype: first value: :obj:`dict` or ``None``, second value: ``None`` or :obj:`str` """ + # pylint: disable=too-many-return-statements,too-many-branches def format_error(address, err): - return None, 'Address %s: %s' % (str(address), err) + return None, f'Address {str(address)}: {err}' if not isinstance(address, dict): return format_error(address, 'address must be a dict') - if 'port' not in address or address['port'] is None: - return format_error(address, 'port is not set or None') - result = {} - for k, v in address.items(): - result[k] = v + for key, val in address.items(): + result[key] = val # Set default values. - for k, v in default_addr_opts.items(): - if k not in result: - result[k] = v + for key, val in default_addr_opts.items(): + if key not in result: + result[key] = val + + if result['socket_fd'] is not None: + # Looks like socket fd. + if result['host'] is not None or result['port'] is not None: + return format_error(result, + "specifying both socket_fd and host/port is not allowed") + + if not isinstance(result['socket_fd'], int): + return format_error(result, + 'socket_fd must be an int') + return result, None if isinstance(result['port'], int): # Looks like an inet address. @@ -152,11 +170,12 @@ def format_error(address, err): 'port must be an int for an inet result') if result['port'] < 1 or result['port'] > 65535: return format_error(result, 'port must be in range [1, 65535] ' - 'for an inet result') + 'for an inet result') # Looks okay. return result, None - elif isinstance(result['port'], str): + + if isinstance(result['port'], str): # Looks like a unix address. # Expect no host. @@ -188,6 +207,7 @@ def update_connection(conn, address): conn.host = address["host"] conn.port = address["port"] + conn.socket_fd = address["socket_fd"] conn.transport = address['transport'] conn.ssl_key_file = address['ssl_key_file'] conn.ssl_cert_file = address['ssl_cert_file'] @@ -198,7 +218,7 @@ def update_connection(conn, address): conn.auth_type = address['auth_type'] -class RoundRobinStrategy(object): +class RoundRobinStrategy(): """ Defines strategy to choose next pool server after fail. """ @@ -209,6 +229,8 @@ def __init__(self, addrs): :paramref:`~tarantool.ConnectionPool.params.addrs`. :type addrs: :obj:`list` of :obj:`dict` """ + self.pos = None + self.addrs = [] self.update(addrs) def update(self, new_addrs): @@ -230,7 +252,7 @@ def update(self, new_addrs): new_addrs = new_addrs_unique # Save a current address if any. - if 'pos' in self.__dict__ and 'addrs' in self.__dict__: + if self.pos is not None: current_addr = self.addrs[self.pos] else: current_addr = None @@ -262,7 +284,10 @@ class MeshConnection(Connection): Represents a connection to a cluster of Tarantool servers. """ - def __init__(self, host=None, port=None, + def __init__(self, + host=DEFAULT_HOST, + port=DEFAULT_PORT, + socket_fd=DEFAULT_SOCKET_FD, user=None, password=None, socket_timeout=SOCKET_TIMEOUT, @@ -283,7 +308,8 @@ def __init__(self, host=None, port=None, addrs=None, strategy_class=RoundRobinStrategy, cluster_discovery_function=None, - cluster_discovery_delay=CLUSTER_DISCOVERY_DELAY): + cluster_discovery_delay=CLUSTER_DISCOVERY_DELAY, + fetch_schema=True): """ :param host: Refer to :paramref:`~tarantool.Connection.params.host`. @@ -291,7 +317,12 @@ def __init__(self, host=None, port=None, :paramref:`~tarantool.MeshConnection.params.addrs` list. :param port: Refer to - :paramref:`~tarantool.Connection.params.host`. + :paramref:`~tarantool.Connection.params.port`. + Value would be used to add one more server in + :paramref:`~tarantool.MeshConnection.params.addrs` list. + + :param socket_fd: Refer to + :paramref:`~tarantol.Connection.params.socket_fd`. Value would be used to add one more server in :paramref:`~tarantool.MeshConnection.params.addrs` list. @@ -425,10 +456,14 @@ def __init__(self, host=None, port=None, list refresh. :type cluster_discovery_delay: :obj:`float`, optional + :param fetch_schema: Refer to + :paramref:`~tarantool.Connection.params.fetch_schema`. + :raises: :exc:`~tarantool.error.ConfigurationError`, :class:`~tarantool.Connection` exceptions, :class:`~tarantool.MeshConnection.connect` exceptions """ + # pylint: disable=too-many-arguments,too-many-locals,too-many-positional-arguments if addrs is None: addrs = [] @@ -436,9 +471,10 @@ def __init__(self, host=None, port=None, # Don't change user provided arguments. addrs = addrs[:] - if host and port: + if (host and port) or socket_fd: addrs.insert(0, {'host': host, 'port': port, + 'socket_fd': socket_fd, 'transport': transport, 'ssl_key_file': ssl_key_file, 'ssl_cert_file': ssl_cert_file, @@ -470,9 +506,10 @@ def __init__(self, host=None, port=None, self.cluster_discovery_delay = cluster_discovery_delay self.last_nodes_refresh = 0 - super(MeshConnection, self).__init__( + super().__init__( host=addr['host'], port=addr['port'], + socket_fd=addr['socket_fd'], user=user, password=password, socket_timeout=socket_timeout, @@ -489,7 +526,8 @@ def __init__(self, host=None, port=None, ssl_ciphers=addr['ssl_ciphers'], ssl_password=addr['ssl_password'], ssl_password_file=addr['ssl_password_file'], - auth_type=addr['auth_type']) + auth_type=addr['auth_type'], + fetch_schema=fetch_schema) def connect(self): """ @@ -503,7 +541,7 @@ def connect(self): :exc:`~tarantool.error.NetworkError`, :class:`~tarantool.Connection.connect` exceptions """ - super(MeshConnection, self).connect() + super().connect() if self.connected and self.cluster_discovery_function: self._opt_refresh_instances() @@ -519,11 +557,11 @@ def _opt_reconnect(self): last_error = None for _ in range(len(self.strategy.addrs)): try: - super(MeshConnection, self)._opt_reconnect() + super()._opt_reconnect() last_error = None break - except NetworkError as e: - last_error = e + except NetworkError as exc: + last_error = exc addr = self.strategy.getnext() update_connection(self, addr) @@ -553,8 +591,8 @@ def _opt_refresh_instances(self): self.call_16) try: resp = self._send_request_wo_reconnect(request) - except DatabaseError as e: - msg = 'got "%s" error, skipped address updates' % str(e) + except DatabaseError as exc: + msg = f'got "{str(exc)}" error, skipped address updates' warn(msg, ClusterDiscoveryWarning) return @@ -592,6 +630,7 @@ def _opt_refresh_instances(self): # an instance list and connect to one of new instances. current_addr = {'host': self.host, 'port': self.port, + 'socket_fd': self.socket_fd, 'transport': self.transport, 'ssl_key_file': self.ssl_key_file, 'ssl_cert_file': self.ssl_cert_file, @@ -621,4 +660,4 @@ def _send_request(self, request, on_push=None, on_push_ctx=None): """ self._opt_refresh_instances() - return super(MeshConnection, self)._send_request(request, on_push, on_push_ctx) + return super()._send_request(request, on_push, on_push_ctx) diff --git a/tarantool/msgpack_ext/datetime.py b/tarantool/msgpack_ext/datetime.py index 64422f5d..f7323c5f 100644 --- a/tarantool/msgpack_ext/datetime.py +++ b/tarantool/msgpack_ext/datetime.py @@ -40,7 +40,6 @@ from tarantool.msgpack_ext.types.datetime import ( NSEC_IN_SEC, - SEC_IN_MIN, Datetime, ) import tarantool.msgpack_ext.types.timezones as tt_timezones @@ -54,10 +53,10 @@ BYTEORDER = 'little' -SECONDS_SIZE_BYTES = 8 -NSEC_SIZE_BYTES = 4 +SECONDS_SIZE_BYTES = 8 +NSEC_SIZE_BYTES = 4 TZOFFSET_SIZE_BYTES = 2 -TZINDEX_SIZE_BYTES = 2 +TZINDEX_SIZE_BYTES = 2 def get_int_as_bytes(data, size): @@ -78,6 +77,7 @@ def get_int_as_bytes(data, size): return data.to_bytes(size, byteorder=BYTEORDER, signed=True) + def encode(obj, _): """ Encode a datetime object. @@ -95,9 +95,9 @@ def encode(obj, _): nsec = obj.nsec tzoffset = obj.tzoffset - tz = obj.tz - if tz != '': - tzindex = tt_timezones.timezoneToIndex[tz] + timezone = obj.tz + if timezone != '': + tzindex = tt_timezones.timezoneToIndex[timezone] else: tzindex = 0 @@ -134,6 +134,7 @@ def get_bytes_as_int(data, cursor, size): part = data[cursor:cursor + size] return int.from_bytes(part, BYTEORDER, signed=True), cursor + size + def decode(data, _): """ Decode a datetime object. @@ -152,11 +153,11 @@ def decode(data, _): seconds, cursor = get_bytes_as_int(data, cursor, SECONDS_SIZE_BYTES) data_len = len(data) - if data_len == (SECONDS_SIZE_BYTES + NSEC_SIZE_BYTES + \ - TZOFFSET_SIZE_BYTES + TZINDEX_SIZE_BYTES): - nsec, cursor = get_bytes_as_int(data, cursor, NSEC_SIZE_BYTES) + if data_len == (SECONDS_SIZE_BYTES + NSEC_SIZE_BYTES + + TZOFFSET_SIZE_BYTES + TZINDEX_SIZE_BYTES): + nsec, cursor = get_bytes_as_int(data, cursor, NSEC_SIZE_BYTES) tzoffset, cursor = get_bytes_as_int(data, cursor, TZOFFSET_SIZE_BYTES) - tzindex, cursor = get_bytes_as_int(data, cursor, TZINDEX_SIZE_BYTES) + tzindex, cursor = get_bytes_as_int(data, cursor, TZINDEX_SIZE_BYTES) elif data_len == SECONDS_SIZE_BYTES: nsec = 0 tzoffset = 0 @@ -170,9 +171,9 @@ def decode(data, _): tz = tt_timezones.indexToTimezone[tzindex] return Datetime(timestamp=seconds, nsec=nsec, tz=tz, timestamp_since_utc_epoch=True) - elif tzoffset != 0: + if tzoffset != 0: return Datetime(timestamp=seconds, nsec=nsec, tzoffset=tzoffset, timestamp_since_utc_epoch=True) - else: - return Datetime(timestamp=seconds, nsec=nsec, - timestamp_since_utc_epoch=True) + + return Datetime(timestamp=seconds, nsec=nsec, + timestamp_since_utc_epoch=True) diff --git a/tarantool/msgpack_ext/decimal.py b/tarantool/msgpack_ext/decimal.py index bad947fb..f1654784 100644 --- a/tarantool/msgpack_ext/decimal.py +++ b/tarantool/msgpack_ext/decimal.py @@ -56,6 +56,8 @@ from decimal import Decimal +import msgpack + from tarantool.error import MsgpackError, MsgpackWarning, warn EXT_ID = 1 @@ -65,6 +67,7 @@ TARANTOOL_DECIMAL_MAX_DIGITS = 38 + def get_mp_sign(sign): """ Parse decimal sign to a nibble. @@ -88,6 +91,7 @@ def get_mp_sign(sign): raise RuntimeError + def add_mp_digit(digit, bytes_reverted, digit_count): """ Append decimal digit to a binary data array. @@ -109,6 +113,7 @@ def add_mp_digit(digit, bytes_reverted, digit_count): else: bytes_reverted.append(digit) + def check_valid_tarantool_decimal(str_repr, scale, first_digit_ind): """ Decimal numbers have 38 digits of precision, that is, the total @@ -183,21 +188,21 @@ def check_valid_tarantool_decimal(str_repr, scale, first_digit_ind): return True if (digit_count - scale) > TARANTOOL_DECIMAL_MAX_DIGITS: - raise MsgpackError('Decimal cannot be encoded: Tarantool decimal ' + \ + raise MsgpackError('Decimal cannot be encoded: Tarantool decimal ' 'supports a maximum of 38 digits.') starts_with_zero = str_repr[first_digit_ind] == '0' - if ( (digit_count > TARANTOOL_DECIMAL_MAX_DIGITS + 1) or \ - (digit_count == TARANTOOL_DECIMAL_MAX_DIGITS + 1 \ - and not starts_with_zero)): - warn('Decimal encoded with loss of precision: ' + \ + if (digit_count > TARANTOOL_DECIMAL_MAX_DIGITS + 1) or \ + (digit_count == TARANTOOL_DECIMAL_MAX_DIGITS + 1 and not starts_with_zero): + warn('Decimal encoded with loss of precision: ' 'Tarantool decimal supports a maximum of 38 digits.', MsgpackWarning) return False return True + def strip_decimal_str(str_repr, scale, first_digit_ind): """ Strip decimal digits after the decimal point if decimal cannot be @@ -225,6 +230,7 @@ def strip_decimal_str(str_repr, scale, first_digit_ind): # Do not strips zeroes before the decimal point return str_repr + def encode(obj, _): """ Encode a decimal object. @@ -244,8 +250,7 @@ def encode(obj, _): bytes_reverted = bytearray() scale = 0 - for i in range(len(str_repr)): - str_digit = str_repr[i] + for i, str_digit in enumerate(str_repr): if str_digit == '.': scale = len(str_repr) - i - 1 break @@ -263,7 +268,7 @@ def encode(obj, _): bytes_reverted.append(get_mp_sign(sign)) digit_count = 0 - # We need to update the scale after possible strip_decimal_str() + # We need to update the scale after possible strip_decimal_str() scale = 0 for i in range(len(str_repr) - 1, first_digit_ind - 1, -1): @@ -301,14 +306,15 @@ def get_str_sign(nibble): :meta private: """ - if nibble == 0x0a or nibble == 0x0c or nibble == 0x0e or nibble == 0x0f: + if nibble in (0x0a, 0x0c, 0x0e, 0x0f): return '+' - if nibble == 0x0b or nibble == 0x0d: + if nibble in (0x0b, 0x0d): return '-' raise MsgpackError('Unexpected MP_DECIMAL sign nibble') + def add_str_digit(digit, digits_reverted, scale): """ Append decimal digit to a binary data array. @@ -327,7 +333,7 @@ def add_str_digit(digit, digits_reverted, scale): :meta private: """ - if not (0 <= digit <= 9): + if not 0 <= digit <= 9: raise MsgpackError('Unexpected MP_DECIMAL digit nibble') if len(digits_reverted) == scale: @@ -335,6 +341,7 @@ def add_str_digit(digit, digits_reverted, scale): digits_reverted.append(str(digit)) + def decode(data, _): """ Decode a decimal object. @@ -348,7 +355,12 @@ def decode(data, _): :raise: :exc:`~tarantool.error.MsgpackError` """ - scale = data[0] + # A decimal starts with mp_int or mp_uint followed by raw bytes. + unpacker = msgpack.Unpacker() + unpacker.feed(data) + + scale = unpacker.unpack() + scale_size = unpacker.tell() sign = get_str_sign(data[-1] & 0x0f) @@ -357,7 +369,7 @@ def decode(data, _): add_str_digit(data[-1] >> 4, digits_reverted, scale) - for i in range(len(data) - 2, 0, -1): + for i in range(len(data) - 2, scale_size - 1, -1): add_str_digit(data[i] & 0x0f, digits_reverted, scale) add_str_digit(data[i] >> 4, digits_reverted, scale) @@ -367,6 +379,12 @@ def decode(data, _): digits_reverted.append(sign) - str_repr = ''.join(digits_reverted[::-1]) + digits = digits_reverted[::-1] + + # Add trailing zeroes in case of a negative scale + for i in range(0, -1 * scale): + add_str_digit(0, digits, scale) + + str_repr = ''.join(digits) return Decimal(str_repr) diff --git a/tarantool/msgpack_ext/error.py b/tarantool/msgpack_ext/error.py index a3f13a04..ef322d58 100644 --- a/tarantool/msgpack_ext/error.py +++ b/tarantool/msgpack_ext/error.py @@ -16,6 +16,7 @@ `error`_ type id. """ + def encode(obj, packer): """ Encode an error object. @@ -33,6 +34,7 @@ def encode(obj, packer): err_map = encode_box_error(obj) return packer.pack(err_map) + def decode(data, unpacker): """ Decode an error object. diff --git a/tarantool/msgpack_ext/interval.py b/tarantool/msgpack_ext/interval.py index cd7ab16d..12136772 100644 --- a/tarantool/msgpack_ext/interval.py +++ b/tarantool/msgpack_ext/interval.py @@ -51,6 +51,7 @@ `datetime.interval`_ type id. """ + def encode(obj, _): """ Encode an interval object. @@ -65,8 +66,7 @@ def encode(obj, _): buf = bytes() count = 0 - for field_id in id_map.keys(): - field_name = id_map[field_id] + for field_id, field_name in id_map.items(): value = getattr(obj, field_name) if field_name == 'adjust': @@ -80,6 +80,7 @@ def encode(obj, _): return buf + def decode(data, unpacker): """ Decode an interval object. @@ -127,8 +128,8 @@ def decode(data, unpacker): if field_name == 'adjust': try: value = Adjust(value) - except ValueError as e: - raise MsgpackError(e) + except ValueError as exc: + raise MsgpackError(exc) from exc kwargs[id_map[field_id]] = value diff --git a/tarantool/msgpack_ext/packer.py b/tarantool/msgpack_ext/packer.py index 12faa29e..e2c03b8b 100644 --- a/tarantool/msgpack_ext/packer.py +++ b/tarantool/msgpack_ext/packer.py @@ -3,6 +3,7 @@ .. _extension: https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/ """ +# pylint: disable=duplicate-code from decimal import Decimal from uuid import UUID @@ -19,13 +20,14 @@ import tarantool.msgpack_ext.interval as ext_interval encoders = [ - {'type': Decimal, 'ext': ext_decimal }, - {'type': UUID, 'ext': ext_uuid }, - {'type': BoxError, 'ext': ext_error }, + {'type': Decimal, 'ext': ext_decimal}, + {'type': UUID, 'ext': ext_uuid}, + {'type': BoxError, 'ext': ext_error}, {'type': Datetime, 'ext': ext_datetime}, {'type': Interval, 'ext': ext_interval}, ] + def default(obj, packer=None): """ :class:`msgpack.Packer` encoder. @@ -48,4 +50,4 @@ def default(obj, packer=None): for encoder in encoders: if isinstance(obj, encoder['type']): return ExtType(encoder['ext'].EXT_ID, encoder['ext'].encode(obj, packer)) - raise TypeError("Unknown type: %r" % (obj,)) + raise TypeError(f"Unknown type: {repr(obj)}") diff --git a/tarantool/msgpack_ext/types/datetime.py b/tarantool/msgpack_ext/types/datetime.py index d2e4b903..d7068f57 100644 --- a/tarantool/msgpack_ext/types/datetime.py +++ b/tarantool/msgpack_ext/types/datetime.py @@ -1,10 +1,12 @@ """ Tarantool `datetime`_ extension type implementation module. """ +# pylint: disable=line-too-long +from calendar import monthrange from copy import deepcopy +from datetime import datetime, timedelta -import pandas import pytz import tarantool.msgpack_ext.types.timezones as tt_timezones @@ -15,15 +17,17 @@ NSEC_IN_MKSEC = 1000 SEC_IN_MIN = 60 MONTH_IN_YEAR = 12 +_EPOCH = datetime(1970, 1, 1, tzinfo=pytz.utc) -def compute_offset(timestamp): + +def compute_offset(_datetime): """ Compute timezone offset. Offset is computed each time and not stored since it could depend on current datetime value. It is expected that timestamp offset is not ``None``. - :param timestamp: Timestamp data. - :type timestamp: :class:`pandas.Timestamp` + :param _datetime: Datetime date. + :type _datetime: :class:`datetime.datetime` :return: Timezone offset, in minutes. :rtype: :obj:`int` @@ -31,7 +35,7 @@ def compute_offset(timestamp): :meta private: """ - utc_offset = timestamp.tzinfo.utcoffset(timestamp) + utc_offset = _datetime.tzinfo.utcoffset(_datetime) # `None` offset is a valid utcoffset implementation, # but it seems that pytz timezones never return `None`: @@ -41,6 +45,7 @@ def compute_offset(timestamp): # There is no precision loss since offset is in minutes return int(utc_offset.total_seconds()) // SEC_IN_MIN + def get_python_tzinfo(tz): """ All non-abbreviated Tarantool timezones are represented as pytz @@ -71,10 +76,29 @@ def get_python_tzinfo(tz): return pytz.FixedOffset(tt_tzinfo['offset']) + +def month_last_day(year, month): + """ + Get the number of the last day in month. + + :param year: Calendar year. + :type year: :obj:`int` + + :param month: Calendar month. + :type month: :obj:`int` + + :rtype: :obj:`int` + + :meta private: + """ + + return monthrange(year, month)[1] + + class Datetime(): """ Class representing Tarantool `datetime`_ info. Internals are based - on :class:`pandas.Timestamp`. + on :class:`datetime.datetime`. You can create :class:`~tarantool.Datetime` objects by using the same API as in Tarantool: @@ -160,43 +184,39 @@ def __init__(self, *, timestamp=None, year=None, month=None, :type timestamp: :obj:`float` or :obj:`int`, optional :param year: Datetime year value. Must be a valid - :class:`pandas.Timestamp` ``year`` parameter. + :class:`datetime.datetime` ``year`` parameter. Must be provided unless the object is built with :paramref:`~tarantool.Datetime.params.data` or :paramref:`~tarantool.Datetime.params.timestamp`. :type year: :obj:`int`, optional :param month: Datetime month value. Must be a valid - :class:`pandas.Timestamp` ``month`` parameter. + :class:`datetime.datetime` ``month`` parameter. Must be provided unless the object is built with :paramref:`~tarantool.Datetime.params.data` or :paramref:`~tarantool.Datetime.params.timestamp`. :type month: :obj:`int`, optional :param day: Datetime day value. Must be a valid - :class:`pandas.Timestamp` ``day`` parameter. + :class:`datetime.datetime` ``day`` parameter. Must be provided unless the object is built with :paramref:`~tarantool.Datetime.params.data` or :paramref:`~tarantool.Datetime.params.timestamp`. :type day: :obj:`int`, optional :param hour: Datetime hour value. Must be a valid - :class:`pandas.Timestamp` ``hour`` parameter. + :class:`datetime.datetime` ``hour`` parameter. :type hour: :obj:`int`, optional :param minute: Datetime minute value. Must be a valid - :class:`pandas.Timestamp` ``minute`` parameter. + :class:`datetime.datetime` ``minute`` parameter. :type minute: :obj:`int`, optional :param sec: Datetime seconds value. Must be a valid - :class:`pandas.Timestamp` ``second`` parameter. + :class:`datetime.datetime` ``second`` parameter. :type sec: :obj:`int`, optional - :param nsec: Datetime nanoseconds value. Quotient of a division - by 1000 (nanoseconds in microseconds) must be a valid - :class:`pandas.Timestamp` ``microsecond`` parameter, - remainder of a division by 1000 must be a valid - :class:`pandas.Timestamp` ``nanosecond`` parameter. + :param nsec: Datetime nanoseconds value. :type sec: :obj:`int`, optional :param tzoffset: Timezone offset. Ignored, if provided together @@ -207,7 +227,7 @@ def __init__(self, *, timestamp=None, year=None, month=None, :type tz: :obj:`str`, optional :param timestamp_since_utc_epoch: Parameter to set timestamp - convertion behavior for timezone-aware datetimes. + conversion behavior for timezone-aware datetimes. If ``False`` (default), behaves similar to Tarantool `datetime.new()`_: @@ -228,13 +248,13 @@ def __init__(self, *, timestamp=None, year=None, month=None, Thus, if ``False``, datetime is computed from timestamp since epoch and then timezone is applied without any - convertion. In that case, + conversion. In that case, :attr:`~tarantool.Datetime.timestamp` won't be equal to initialization :paramref:`~tarantool.Datetime.params.timestamp` for all timezones with non-zero offset. - If ``True``, behaves similar to :class:`pandas.Timestamp`: + If ``True``, behaves similar to :class:`datetime.datetime`: .. code-block:: python @@ -257,10 +277,11 @@ def __init__(self, *, timestamp=None, year=None, month=None, :type timestamp_since_utc_epoch: :obj:`bool`, optional :raise: :exc:`ValueError`, :exc:`~tarantool.error.MsgpackError`, - :class:`pandas.Timestamp` exceptions + :class:`datetime.datetime` exceptions .. _datetime.new(): https://www.tarantool.io/en/doc/latest/reference/reference_lua/datetime/new/ """ + # pylint: disable=too-many-branches,too-many-locals,too-many-statements,too-many-arguments tzinfo = None if tz != '': @@ -275,46 +296,95 @@ def __init__(self, *, timestamp=None, year=None, month=None, # The logic is same as in Tarantool, refer to datetime API. # https://www.tarantool.io/en/doc/latest/reference/reference_lua/datetime/new/ if timestamp is not None: - if ((year is not None) or (month is not None) or \ - (day is not None) or (hour is not None) or \ - (minute is not None) or (sec is not None)): - raise ValueError('Cannot provide both timestamp and year, month, ' + + # pylint: disable=too-many-boolean-expressions + if ((year is not None) or (month is not None) + or (day is not None) or (hour is not None) + or (minute is not None) or (sec is not None)): + raise ValueError('Cannot provide both timestamp and year, month, ' 'day, hour, minute, sec') if nsec is not None: if not isinstance(timestamp, int): raise ValueError('timestamp must be int if nsec provided') - total_nsec = timestamp * NSEC_IN_SEC + nsec - datetime = pandas.to_datetime(total_nsec, unit='ns') - else: - datetime = pandas.to_datetime(timestamp, unit='s') + # Tarantool may send negative nanoseconds or nanoseconds bigger + # than 999999999. datetime.datetime doesn't process overflows. + if (nsec >= NSEC_IN_SEC) or (nsec < 0): + timestamp += nsec // NSEC_IN_SEC + nsec = nsec % NSEC_IN_SEC + + # datetime.fromtimestamp may raise OverflowError, if the timestamp + # is out of the range of values supported by the platform C localtime() + # function, and OSError on localtime() failure. It’s common for this + # to be restricted to years from 1970 through 2038, yet we want + # to support a wider range. + _datetime = _EPOCH + timedelta(seconds=timestamp) - if not timestamp_since_utc_epoch: - self._datetime = datetime.tz_localize(tzinfo) - else: - self._datetime = datetime.tz_localize(pytz.UTC).tz_convert(tzinfo) - else: if nsec is not None: - microsecond = nsec // NSEC_IN_MKSEC - nanosecond = nsec % NSEC_IN_MKSEC + _datetime = _datetime.replace(microsecond=nsec // NSEC_IN_MKSEC) + _datetime_nsec = nsec % NSEC_IN_MKSEC else: - microsecond = 0 - nanosecond = 0 + _datetime_nsec = 0 + + if tzinfo is not None: + if not timestamp_since_utc_epoch: + # It seems that there is no way to get expected behavior without + # this hack. Localizing a timezone-naive datetime built + # from the timestamp fails since it uses local timezone to mess up + # the underlying timestamp. On the other hand, you cannot localize + # a timezone-aware datetime, even UTC one. Replaces don't work since + # they are broken for pytz + datetime, see + # https://pythonhosted.org/pytz/ + _datetime = datetime.combine(_datetime.date(), _datetime.time()) + _datetime = tzinfo.localize(_datetime) + else: + _datetime = _datetime.astimezone(tzinfo) + + self._datetime = _datetime + self._datetime_nsec = _datetime_nsec + else: + # datetime does not support None as defaults, + # we support them for backward compatibility. + if hour is None: + hour = 0 + + if minute is None: + minute = 0 - self._datetime = pandas.Timestamp( - year=year, month=month, day=day, - hour=hour, minute=minute, second=sec, - microsecond=microsecond, - nanosecond=nanosecond).tz_localize(tzinfo) + if sec is None: + sec = 0 - def _interval_operation(self, other, sign=1): + overflow = None + if nsec is None: + nsec = 0 + else: + # Tarantool may send negative nanoseconds or nanoseconds bigger + # than 999999999. datetime.datetime doesn't process overflows. + if (nsec >= NSEC_IN_SEC) or (nsec < 0): + overflow = timedelta(seconds=nsec // NSEC_IN_SEC) + nsec = nsec % NSEC_IN_SEC + + _datetime = datetime(year=year, month=month, day=day, + hour=hour, minute=minute, second=sec, + microsecond=nsec // NSEC_IN_MKSEC) + if overflow is not None: + _datetime = _datetime + overflow + # tzinfo as argument on the datetime not works as expected, see + # https://pythonhosted.org/pytz/ + # Timezone-naive datetime objects are treated by many datetime methods + # as local times, so we represent time in UTC explicitly if not provided. + if tzinfo is None: + tzinfo = pytz.UTC + self._datetime = tzinfo.localize(_datetime) + self._datetime_nsec = nsec % NSEC_IN_MKSEC + + def _interval_operation(self, interval, sign=1): """ Implementation of :class:`~tarantool.Interval` addition and subtraction. - :param other: Interval to add or subtract. - :type other: :class:`~tarantool.Interval` + :param interval: Interval to add or subtract. + :type interval: :class:`~tarantool.Interval` :param sign: Right operand multiplier: ``1`` for addition, ``-1`` for subtractiom. @@ -325,37 +395,45 @@ def _interval_operation(self, other, sign=1): :meta private: """ - self_dt = self._datetime + old_dt = self._datetime + new_dt = old_dt + + new_year = old_dt.year + sign * interval.year + new_month = old_dt.month + sign * interval.month + if (new_month < 1) or (new_month - 1 > MONTH_IN_YEAR): + new_year += (new_month - 1) // MONTH_IN_YEAR + new_month = (new_month - 1) % MONTH_IN_YEAR + 1 + + new_month_last_day = month_last_day(new_year, new_month) + old_month_last_day = month_last_day(old_dt.year, old_dt.month) # https://github.com/tarantool/tarantool/wiki/Datetime-Internals#date-adjustions-and-leap-years - months = other.year * MONTH_IN_YEAR + other.month - - res = self_dt + pandas.DateOffset(months = sign * months) - - # pandas.DateOffset works exactly like Adjust.NONE - if other.adjust == Adjust.EXCESS: - if self_dt.day > res.day: - res = res + pandas.DateOffset(days = self_dt.day - res.day) - elif other.adjust == Adjust.LAST: - if self_dt.is_month_end: - # day replaces days - res = res.replace(day = res.days_in_month) - - res = res + pandas.Timedelta(weeks = sign * other.week, - days = sign * other.day, - hours = sign * other.hour, - minutes = sign * other.minute, - seconds = sign * other.sec, - microseconds = sign * (other.nsec // NSEC_IN_MKSEC), - nanoseconds = sign * (other.nsec % NSEC_IN_MKSEC)) - - if res.tzinfo is not None: - tzoffset = compute_offset(res) + if (interval.adjust == Adjust.NONE) and (new_month_last_day < new_dt.day): + new_dt = new_dt.replace(year=new_year, month=new_month, day=new_month_last_day) + elif (interval.adjust == Adjust.EXCESS) and (new_month_last_day < new_dt.day): + new_dt = new_dt.replace(year=new_year, month=new_month, day=new_month_last_day) + \ + timedelta(days=new_dt.day - new_month_last_day) + elif (interval.adjust == Adjust.LAST) and (old_dt.day == old_month_last_day): + new_dt = new_dt.replace(year=new_year, month=new_month, day=new_month_last_day) + else: + new_dt = new_dt.replace(year=new_year, month=new_month) + + nsec = self._datetime_nsec + sign * interval.nsec + new_dt = new_dt + timedelta(weeks=sign * interval.week, + days=sign * interval.day, + hours=sign * interval.hour, + minutes=sign * interval.minute, + seconds=sign * interval.sec, + microseconds=nsec // NSEC_IN_MKSEC) + new_nsec = nsec % NSEC_IN_MKSEC + + if new_dt.tzinfo is not None: + tzoffset = compute_offset(new_dt) else: tzoffset = 0 - return Datetime(year=res.year, month=res.month, day=res.day, - hour=res.hour, minute=res.minute, sec=res.second, - nsec=res.nanosecond + res.microsecond * NSEC_IN_MKSEC, + return Datetime(year=new_dt.year, month=new_dt.month, day=new_dt.day, + hour=new_dt.hour, minute=new_dt.minute, sec=new_dt.second, + nsec=new_nsec + new_dt.microsecond * NSEC_IN_MKSEC, tzoffset=tzoffset, tz=self.tz) def __add__(self, other): @@ -421,7 +499,8 @@ def __add__(self, other): """ if not isinstance(other, Interval): - raise TypeError(f"unsupported operand type(s) for +: '{type(self)}' and '{type(other)}'") + raise TypeError("unsupported operand type(s) for +: " + f"'{type(self)}' and '{type(other)}'") return self._interval_operation(other, sign=1) @@ -451,61 +530,69 @@ def __sub__(self, other): self_dt = self._datetime other_dt = other._datetime - # Tarantool datetime subtraction ignores timezone info, but it is a bug: - # - # Tarantool 2.10.1-0-g482d91c66 - # - # tarantool> datetime.new{tz='MSK'} - datetime.new{tz='UTC'} - # --- - # - +0 seconds - # ... - # - # Refer to https://github.com/tarantool/tarantool/issues/7698 - # for possible updates. - if self_dt.tzinfo != other_dt.tzinfo: - other_dt = other_dt.tz_convert(self_dt.tzinfo) + other_dt = other_dt.astimezone(self_dt.tzinfo) - self_nsec = self_dt.microsecond * NSEC_IN_MKSEC + self_dt.nanosecond - other_nsec = other_dt.microsecond * NSEC_IN_MKSEC + other_dt.nanosecond + self_nsec = self_dt.microsecond * NSEC_IN_MKSEC + self._datetime_nsec + other_nsec = other_dt.microsecond * NSEC_IN_MKSEC + other._datetime_nsec return Interval( - year = self_dt.year - other_dt.year, - month = self_dt.month - other_dt.month, - day = self_dt.day - other_dt.day, - hour = self_dt.hour - other_dt.hour, - minute = self_dt.minute - other_dt.minute, - sec = self_dt.second - other_dt.second, - nsec = self_nsec - other_nsec, + year=self_dt.year - other_dt.year, + month=self_dt.month - other_dt.month, + day=self_dt.day - other_dt.day, + hour=self_dt.hour - other_dt.hour, + minute=self_dt.minute - other_dt.minute, + sec=self_dt.second - other_dt.second, + nsec=self_nsec - other_nsec, ) - elif isinstance(other, Interval): + if isinstance(other, Interval): return self._interval_operation(other, sign=-1) - else: - raise TypeError(f"unsupported operand type(s) for -: '{type(self)}' and '{type(other)}'") + + raise TypeError(f"unsupported operand type(s) for -: '{type(self)}' and '{type(other)}'") def __eq__(self, other): """ Datetimes are equal when underlying datetime infos are equal. :param other: Second operand. - :type other: :class:`~tarantool.Datetime` or - :class:`~pandas.Timestamp` + :type other: :class:`~tarantool.Datetime` :rtype: :obj:`bool` """ if isinstance(other, Datetime): - return self._datetime == other._datetime - elif isinstance(other, pandas.Timestamp): - return self._datetime == other - else: - return False + return self.value == other.value + return False def __str__(self): - return self._datetime.__str__() + # Based on pandas.Timestamp isofomat for backward compatibility. + # https://github.com/pandas-dev/pandas/blob/249d93e4abc59639983eb3e8fccac8382592d457/pandas/_libs/tslibs/timestamps.pyx#L1015-L1034 + base = self._datetime.isoformat(sep='T', timespec='auto') + + # Preserve explicit UTC and implicit UTC difference for backward compatibility. + implicit_utc = False + if (self._datetime.tzinfo == pytz.UTC) and (self._tz == ''): + implicit_utc = True + base = base[:-6] + + if self._datetime_nsec == 0: + return base + + if implicit_utc: + base1, base2 = base, "" + else: + base1, base2 = base[:-6], base[-6:] + + if self._datetime.microsecond: + base1 += f"{self._datetime_nsec:03d}" + else: + base1 += f".{self._datetime_nsec:09d}" + + return base1 + base2 def __repr__(self): - return f'datetime: {self._datetime.__repr__()}, tz: "{self.tz}"' + return f'datetime: {self._datetime.__repr__()}, nsec: {self._datetime_nsec}, ' + \ + f'tz: "{self.tz}"' def __copy__(self): cls = self.__class__ @@ -517,8 +604,8 @@ def __deepcopy__(self, memo): cls = self.__class__ result = cls.__new__(cls) memo[id(self)] = result - for k, v in self.__dict__.items(): - setattr(result, k, deepcopy(v, memo)) + for key, val in self.__dict__.items(): + setattr(result, key, deepcopy(val, memo)) return result @property @@ -589,7 +676,7 @@ def nsec(self): :rtype: :obj:`int` """ - return self._datetime.value % NSEC_IN_SEC + return self._datetime.microsecond * NSEC_IN_MKSEC + self._datetime_nsec @property def timestamp(self): @@ -599,7 +686,7 @@ def timestamp(self): :rtype: :obj:`float` """ - return self._datetime.timestamp() + return self._datetime.timestamp() + self._datetime_nsec / NSEC_IN_SEC @property def tzoffset(self): @@ -631,4 +718,7 @@ def value(self): :rtype: :obj:`int` """ - return self._datetime.value + # Python sources way to get ineteger time since epoch. + # https://github.com/python/cpython/blob/a6f95941a3d686707fb38e0f37758e666f25e180/Lib/datetime.py#L1879 + seconds = (self._datetime - _EPOCH) // timedelta(0, 1) + return seconds * NSEC_IN_SEC + self.nsec diff --git a/tarantool/msgpack_ext/types/interval.py b/tarantool/msgpack_ext/types/interval.py index e90ff0a0..734a9088 100644 --- a/tarantool/msgpack_ext/types/interval.py +++ b/tarantool/msgpack_ext/types/interval.py @@ -16,6 +16,61 @@ 8: 'adjust', } +# https://github.com/tarantool/tarantool/blob/ff57f990f359f6d7866c1947174d8ba0e97b1ea6/src/lua/datetime.lua#L112-L146 +SECS_PER_DAY = 86400 + +MIN_DATE_YEAR = -5879610 +MIN_DATE_MONTH = 6 +MIN_DATE_DAY = 22 +MAX_DATE_YEAR = 5879611 +MAX_DATE_MONTH = 7 +MAX_DATE_DAY = 11 + +AVERAGE_DAYS_YEAR = 365.25 +AVERAGE_WEEK_YEAR = AVERAGE_DAYS_YEAR / 7 +INT_MAX = 2147483647 +MAX_YEAR_RANGE = MAX_DATE_YEAR - MIN_DATE_YEAR +MAX_MONTH_RANGE = MAX_YEAR_RANGE * 12 +MAX_WEEK_RANGE = MAX_YEAR_RANGE * AVERAGE_WEEK_YEAR +MAX_DAY_RANGE = MAX_YEAR_RANGE * AVERAGE_DAYS_YEAR +MAX_HOUR_RANGE = MAX_DAY_RANGE * 24 +MAX_MIN_RANGE = MAX_HOUR_RANGE * 60 +MAX_SEC_RANGE = MAX_DAY_RANGE * SECS_PER_DAY +MAX_NSEC_RANGE = INT_MAX + +max_val = { + 'year': MAX_YEAR_RANGE, + 'month': MAX_MONTH_RANGE, + 'week': MAX_WEEK_RANGE, + 'day': MAX_DAY_RANGE, + 'hour': MAX_HOUR_RANGE, + 'minute': MAX_MIN_RANGE, + 'sec': MAX_SEC_RANGE, + 'nsec': MAX_NSEC_RANGE, +} + + +def verify_range(intv): + """ + Check allowed values. Approach is the same as in tarantool/tarantool. + + :param intv: Raw interval to verify. + :type intv: :class:`~tarantool.Interval` + + :raise: :exc:`ValueError` + + :meta private: + """ + + for field_name, range_max in max_val.items(): + val = getattr(intv, field_name) + # Tarantool implementation has a bug + # https://github.com/tarantool/tarantool/issues/8878 + if (val > range_max) or (val < -range_max): + raise ValueError(f"value {val} of {field_name} is out of " + f"allowed range [{-range_max}, {range_max}]") + + # https://github.com/tarantool/c-dt/blob/cec6acebb54d9e73ea0b99c63898732abd7683a6/dt_arithmetic.h#L34 class Adjust(Enum): """ @@ -38,6 +93,7 @@ class Adjust(Enum): Mode when day snaps to the end of month, if it happens. """ + class Interval(): """ Class representing Tarantool `datetime.interval`_ info. @@ -47,8 +103,8 @@ class Interval(): .. code-block:: python - di = tarantool.Interval(year=-1, month=2, day=3, - hour=4, minute=-5, sec=6, + di = tarantool.Interval(year=-1, month=2, week=-3, + day=4, hour=5, minute=-6, sec=7, nsec=308543321, adjust=tarantool.IntervalAdjust.NONE) @@ -57,6 +113,7 @@ class Interval(): .. _datetime.interval: https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/#the-interval-type """ + # pylint: disable=too-many-instance-attributes def __init__(self, *, year=0, month=0, week=0, day=0, hour=0, minute=0, sec=0, @@ -89,8 +146,11 @@ def __init__(self, *, year=0, month=0, week=0, :param adjust: Interval adjustment rule. Refer to :meth:`~tarantool.Datetime.__add__`. :type adjust: :class:`~tarantool.IntervalAdjust`, optional + + :raise: :exc:`ValueError` """ - + # pylint: disable=too-many-arguments + self.year = year self.month = month self.week = week @@ -101,6 +161,8 @@ def __init__(self, *, year=0, month=0, week=0, self.nsec = nsec self.adjust = adjust + verify_range(self) + def __add__(self, other): """ Valid operations: @@ -119,7 +181,8 @@ def __add__(self, other): """ if not isinstance(other, Interval): - raise TypeError(f"unsupported operand type(s) for +: '{type(self)}' and '{type(other)}'") + raise TypeError("unsupported operand type(s) for +: " + f"'{type(self)}' and '{type(other)}'") # Tarantool saves adjust of the first argument # @@ -139,14 +202,15 @@ def __add__(self, other): # ... return Interval( - year = self.year + other.year, - month = self.month + other.month, - day = self.day + other.day, - hour = self.hour + other.hour, - minute = self.minute + other.minute, - sec = self.sec + other.sec, - nsec = self.nsec + other.nsec, - adjust = self.adjust, + year=self.year + other.year, + month=self.month + other.month, + week=self.week + other.week, + day=self.day + other.day, + hour=self.hour + other.hour, + minute=self.minute + other.minute, + sec=self.sec + other.sec, + nsec=self.nsec + other.nsec, + adjust=self.adjust, ) def __sub__(self, other): @@ -167,7 +231,8 @@ def __sub__(self, other): """ if not isinstance(other, Interval): - raise TypeError(f"unsupported operand type(s) for -: '{type(self)}' and '{type(other)}'") + raise TypeError("unsupported operand type(s) for -: " + f"'{type(self)}' and '{type(other)}'") # Tarantool saves adjust of the first argument # @@ -187,14 +252,15 @@ def __sub__(self, other): # ... return Interval( - year = self.year - other.year, - month = self.month - other.month, - day = self.day - other.day, - hour = self.hour - other.hour, - minute = self.minute - other.minute, - sec = self.sec - other.sec, - nsec = self.nsec - other.nsec, - adjust = self.adjust, + year=self.year - other.year, + month=self.month - other.month, + week=self.week - other.week, + day=self.day - other.day, + hour=self.hour - other.hour, + minute=self.minute - other.minute, + sec=self.sec - other.sec, + nsec=self.nsec - other.nsec, + adjust=self.adjust, ) def __eq__(self, other): @@ -219,16 +285,15 @@ def __eq__(self, other): # - false # ... - for field_id in id_map.keys(): - field_name = id_map[field_id] + for _, field_name in id_map.items(): if getattr(self, field_name) != getattr(other, field_name): return False return True def __repr__(self): - return f'tarantool.Interval(year={self.year}, month={self.month}, day={self.day}, ' + \ - f'hour={self.hour}, minute={self.minute}, sec={self.sec}, ' + \ + return f'tarantool.Interval(year={self.year}, month={self.month}, week={self.week}, ' + \ + f'day={self.day}, hour={self.hour}, minute={self.minute}, sec={self.sec}, ' + \ f'nsec={self.nsec}, adjust={self.adjust})' __str__ = __repr__ diff --git a/tarantool/msgpack_ext/types/timezones/__init__.py b/tarantool/msgpack_ext/types/timezones/__init__.py index c0c4ce7e..47a9edf9 100644 --- a/tarantool/msgpack_ext/types/timezones/__init__.py +++ b/tarantool/msgpack_ext/types/timezones/__init__.py @@ -3,10 +3,10 @@ """ from tarantool.msgpack_ext.types.timezones.timezones import ( - TZ_AMBIGUOUS, - indexToTimezone, - timezoneToIndex, - timezoneAbbrevInfo, + TZ_AMBIGUOUS, + indexToTimezone, + timezoneToIndex, + timezoneAbbrevInfo, ) __all__ = ['TZ_AMBIGUOUS', 'indexToTimezone', 'timezoneToIndex', diff --git a/tarantool/msgpack_ext/types/timezones/gen-timezones.sh b/tarantool/msgpack_ext/types/timezones/gen-timezones.sh index 5de5a51a..bca568b0 100755 --- a/tarantool/msgpack_ext/types/timezones/gen-timezones.sh +++ b/tarantool/msgpack_ext/types/timezones/gen-timezones.sh @@ -26,23 +26,24 @@ cat < ${DST_FILE} Tarantool timezone info. Automatically generated by \`\`gen-timezones.sh\`\`. """ +# pylint: disable=too-many-lines -TZ_UTC = 0x01 -TZ_RFC = 0x02 -TZ_MILITARY = 0x04 +TZ_UTC = 0x01 +TZ_RFC = 0x02 +TZ_MILITARY = 0x04 TZ_AMBIGUOUS = 0x08 -TZ_NYI = 0x10 -TZ_OLSON = 0x20 -TZ_ALIAS = 0x40 -TZ_DST = 0x80 +TZ_NYI = 0x10 +TZ_OLSON = 0x20 +TZ_ALIAS = 0x40 +TZ_DST = 0x80 indexToTimezone = { EOF grep ZONE_ABBREV ${SRC_FILE} | sed "s/ZONE_ABBREV( *//g" | sed "s/[),]//g" \ - | awk '{printf("\t%s : %s,\n", $1, $3)}' >> ${DST_FILE} + | awk '{printf(" %s: %s,\n", $1, $3)}' >> ${DST_FILE} grep ZONE_UNIQUE ${SRC_FILE} | sed "s/ZONE_UNIQUE( *//g" | sed "s/[),]//g" \ - | awk '{printf("\t%s : %s,\n", $1, $2)}' >> ${DST_FILE} + | awk '{printf(" %s: %s,\n", $1, $2)}' >> ${DST_FILE} cat <> ${DST_FILE} } @@ -51,11 +52,11 @@ timezoneToIndex = { EOF grep ZONE_ABBREV ${SRC_FILE} | sed "s/ZONE_ABBREV( *//g" | sed "s/[),]//g" \ - | awk '{printf("\t%s : %s,\n", $3, $1)}' >> ${DST_FILE} + | awk '{printf(" %s: %s,\n", $3, $1)}' >> ${DST_FILE} grep ZONE_UNIQUE ${SRC_FILE} | sed "s/ZONE_UNIQUE( *//g" | sed "s/[),]//g" \ - | awk '{printf("\t%s : %s,\n", $2, $1)}' >> ${DST_FILE} + | awk '{printf(" %s: %s,\n", $2, $1)}' >> ${DST_FILE} grep ZONE_ALIAS ${SRC_FILE} | sed "s/ZONE_ALIAS( *//g" | sed "s/[),]//g" \ - | awk '{printf("\t%s : %s,\n", $2, $1)}' >> ${DST_FILE} + | awk '{printf(" %s: %s,\n", $2, $1)}' >> ${DST_FILE} cat <> ${DST_FILE} } @@ -63,10 +64,23 @@ cat <> ${DST_FILE} timezoneAbbrevInfo = { EOF -grep ZONE_ABBREV ${SRC_FILE} | sed "s/ZONE_ABBREV( *//g" | sed "s/[),]//g" \ - | awk '{printf("\t%s : {\"offset\" : %d, \"category\" : %s},\n", $3, $2, $4)}' >> ${DST_FILE} +grep ZONE_ABBREV ${SRC_FILE} | sed "s/ZONE_ABBREV( *//g" | sed "s/[),]//g" \ + | awk '{printf(" %s: {\"offset\": %d, \"category\": %s},\n", $3, $2, $4)}' | sed "s/|/ | /g" >> ${DST_FILE} echo "}" >> ${DST_FILE} rm timezones.h -python validate_timezones.py +python <= (1, 0, 0): unpacker_kwargs['strict_map_key'] = False - # We need configured unpacker to work with error extention + # We need configured unpacker to work with error extension # type payload, but module do not provide access to self # inside extension type unpackers. - unpacker_no_ext = msgpack.Unpacker(**unpacker_kwargs) - ext_hook = lambda code, data: unpacker_ext_hook(code, data, unpacker_no_ext) + def ext_hook(code, data): + unpacker_no_ext = msgpack.Unpacker(**unpacker_kwargs) + return unpacker_ext_hook(code, data, unpacker_no_ext) unpacker_kwargs['ext_hook'] = ext_hook return msgpack.Unpacker(**unpacker_kwargs) @@ -93,6 +94,7 @@ class Response(Sequence): received list of tuples) and parsing of binary packets received from the server. """ + # pylint: disable=too-many-instance-attributes def __init__(self, conn, response): """ @@ -129,8 +131,8 @@ def __init__(self, conn, response): self._return_code = 0 self._schema_version = header.get(IPROTO_SCHEMA_ID, None) self._data = self._body.get(IPROTO_DATA, None) - if (not isinstance(self._data, (list, tuple)) and - self._data is not None): + if (not isinstance(self._data, (list, tuple)) + and self._data is not None): self._data = [self._data] # # Backward-compatibility # if isinstance(self._data, (list, tuple)): @@ -192,7 +194,7 @@ def index(self, *args): raise InterfaceError("Trying to access data when there's no data") return self._data.index(*args) - def count(self, item): + def count(self, value): """ Refer to :class:`collections.abc.Sequence`. @@ -201,7 +203,7 @@ def count(self, item): if self._data is None: raise InterfaceError("Trying to access data when there's no data") - return self._data.count(item) + return self._data.count(value) @property def rowcount(self): @@ -302,7 +304,7 @@ def __str__(self): 'code': self.strerror[0], 'reason': self.return_message } - }, sort_keys = True, indent = 4, separators=(', ', ': ')) + }, sort_keys=True, indent=4, separators=(', ', ': ')) output = [] for tpl in self._data or (): output.extend(("- ", repr(tpl), "\n")) diff --git a/tarantool/schema.py b/tarantool/schema.py index d4f13f6b..556737b7 100644 --- a/tarantool/schema.py +++ b/tarantool/schema.py @@ -1,32 +1,35 @@ -# pylint: disable=R0903 """ Schema types definitions. For internal use only, there is no API to use pre-build schema objects. """ from tarantool.error import ( - Error, SchemaError, DatabaseError ) -import tarantool.const as const - - -class RecursionError(Error): - """ - Report the situation when max recursion depth is reached. +from tarantool.const import ( + INDEX_SPACE_NAME, + INDEX_INDEX_NAME, + INDEX_SPACE_PRIMARY, + INDEX_INDEX_PRIMARY, + SPACE_VSPACE, + SPACE_VINDEX, + SPACE_SPACE, + SPACE_INDEX +) - This is an internal error of - :func:`~tarantool.schema.to_unicode_recursive` caller and it should - be re-raised properly by the caller. - """ +MAX_RECURSION_DEPTH = 32 +""" +Max possible known schema depth is 4 if foreign keys are used (since +Tarantool 2.10), but there are no restrictions in protocol. +""" -def to_unicode(s): +def to_unicode(string): """ Decode :obj:`bytes` to unicode :obj:`str`. - :param s: Value to convert. + :param string: Value to convert. :return: Decoded unicode :obj:`str`, if value is :obj:`bytes`. Otherwise, it returns the original value. @@ -34,17 +37,17 @@ def to_unicode(s): :meta private: """ - if isinstance(s, bytes): - return s.decode(encoding='utf-8') - return s + if isinstance(string, bytes): + return string.decode(encoding='utf-8') + return string -def to_unicode_recursive(x, max_depth): +def to_unicode_recursive(value, max_depth): """ Recursively decode :obj:`bytes` to unicode :obj:`str` over :obj:`dict`, :obj:`list` and :obj:`tuple`. - :param x: Value to convert. + :param value: Value to convert. :param max_depth: Maximum depth recursion. :type max_depth: :obj:`int` @@ -52,7 +55,7 @@ def to_unicode_recursive(x, max_depth): :return: The same structure where all :obj:`bytes` are replaced with unicode :obj:`str`. - :raise: :exc:`~tarantool.schema.RecursionError` + :raise: :exc:`~RecursionError` :meta private: """ @@ -60,30 +63,31 @@ def to_unicode_recursive(x, max_depth): if max_depth <= 0: raise RecursionError('Max recursion depth is reached') - if isinstance(x, dict): - res = dict() - for key, val in x.items(): + if isinstance(value, dict): + res = {} + for key, val in value.items(): key = to_unicode_recursive(key, max_depth - 1) val = to_unicode_recursive(val, max_depth - 1) res[key] = val return res - if isinstance(x, list) or isinstance(x, tuple): + if isinstance(value, (list, tuple)): res = [] - for val in x: - val = to_unicode_recursive(val, max_depth - 1) - res.append(val) - if isinstance(x, tuple): + for item in value: + item = to_unicode_recursive(item, max_depth - 1) + res.append(item) + if isinstance(value, tuple): return tuple(res) return res - return to_unicode(x) + return to_unicode(value) -class SchemaIndex(object): +class SchemaIndex(): """ Contains schema for a space index. """ + # pylint: disable=too-few-public-methods def __init__(self, index_row, space): """ @@ -97,16 +101,15 @@ def __init__(self, index_row, space): """ self.iid = index_row[1] - self.name = index_row[2] self.name = to_unicode(index_row[2]) self.index = index_row[3] self.unique = index_row[4] self.parts = [] try: - parts_raw = to_unicode_recursive(index_row[5], 3) - except RecursionError as e: - errmsg = 'Unexpected index parts structure: ' + str(e) - raise SchemaError(errmsg) + parts_raw = to_unicode_recursive(index_row[5], MAX_RECURSION_DEPTH) + except RecursionError as exc: + errmsg = 'Unexpected index parts structure: ' + str(exc) + raise SchemaError(errmsg) from exc if isinstance(parts_raw, (list, tuple)): for val in parts_raw: if isinstance(val, dict): @@ -134,10 +137,11 @@ def flush(self): del self.space.indexes[self.name] -class SchemaSpace(object): +class SchemaSpace(): """ Contains schema for a space. """ + # pylint: disable=too-few-public-methods def __init__(self, space_row, schema): """ @@ -158,16 +162,16 @@ def __init__(self, space_row, schema): self.schema[self.sid] = self if self.name: self.schema[self.name] = self - self.format = dict() + self.format = {} try: - format_raw = to_unicode_recursive(space_row[6], 3) - except RecursionError as e: - errmsg = 'Unexpected space format structure: ' + str(e) - raise SchemaError(errmsg) + format_raw = to_unicode_recursive(space_row[6], MAX_RECURSION_DEPTH) + except RecursionError as exc: + errmsg = 'Unexpected space format structure: ' + str(exc) + raise SchemaError(errmsg) from exc for part_id, part in enumerate(format_raw): part['id'] = part_id self.format[part['name']] = part - self.format[part_id ] = part + self.format[part_id] = part def flush(self): """ @@ -179,10 +183,11 @@ def flush(self): del self.schema[self.name] -class Schema(object): +class Schema(): """ Contains Tarantool server spaces schema. """ + # pylint: disable=too-few-public-methods def __init__(self, con): """ @@ -236,10 +241,10 @@ def fetch_space(self, space): raise SchemaError( 'Some strange output from server: \n' + str(space_row) ) - elif len(space_row) == 0 or not len(space_row[0]): + if len(space_row) == 0 or not space_row[0]: # We can't find space with this name or id temp_name = 'name' if isinstance(space, str) else 'id' - errmsg = "There's no space with {1} '{0}'".format(space, temp_name) + errmsg = f"There's no space with {temp_name} '{space}'" raise SchemaError(errmsg) space_row = space_row[0] @@ -262,9 +267,9 @@ def fetch_space_from(self, space): _index = None if isinstance(space, str): - _index = const.INDEX_SPACE_NAME + _index = INDEX_SPACE_NAME else: - _index = const.INDEX_SPACE_PRIMARY + _index = INDEX_SPACE_PRIMARY if space is None: space = () @@ -272,16 +277,16 @@ def fetch_space_from(self, space): space_row = None try: # Try to fetch from '_vspace' - space_row = self.con.select(const.SPACE_VSPACE, space, + space_row = self.con.select(SPACE_VSPACE, space, index=_index) - except DatabaseError as e: + except DatabaseError as exc: # if space can't be found, then user is using old version of # tarantool, try again with '_space' - if e.args[0] != 36: + if exc.args[0] != 36: raise if space_row is None: # Try to fetch from '_space' - space_row = self.con.select(const.SPACE_SPACE, space, index=_index) + space_row = self.con.select(SPACE_SPACE, space, index=_index) return space_row @@ -348,12 +353,11 @@ def fetch_index(self, space_object, index): raise SchemaError( 'Some strange output from server: \n' + str(index_row) ) - elif len(index_row) == 0 or not len(index_row[0]): + if len(index_row) == 0 or not index_row[0]: # We can't find index with this name or id temp_name = 'name' if isinstance(index, str) else 'id' - errmsg = ("There's no index with {2} '{0}'" - " in space '{1}'").format(index, space_object.name, - temp_name) + errmsg = (f"There's no index with {temp_name} '{index}'" + f" in space '{space_object.name}'") raise SchemaError(errmsg) index_row = index_row[0] @@ -392,15 +396,15 @@ def fetch_index_from(self, space, index): _index = None if isinstance(index, str): - _index = const.INDEX_INDEX_NAME + _index = INDEX_INDEX_NAME else: - _index = const.INDEX_INDEX_PRIMARY + _index = INDEX_INDEX_PRIMARY _key_tuple = None if space is None and index is None: _key_tuple = () elif space is not None and index is None: - _key_tuple = (space) + _key_tuple = (space, ) elif space is not None and index is not None: _key_tuple = (space, index) else: @@ -409,16 +413,16 @@ def fetch_index_from(self, space, index): index_row = None try: # Try to fetch from '_vindex' - index_row = self.con.select(const.SPACE_VINDEX, _key_tuple, + index_row = self.con.select(SPACE_VINDEX, _key_tuple, index=_index) - except DatabaseError as e: + except DatabaseError as exc: # if space can't be found, then user is using old version of # tarantool, try again with '_index' - if e.args[0] != 36: + if exc.args[0] != 36: raise if index_row is None: # Try to fetch from '_index' - index_row = self.con.select(const.SPACE_INDEX, _key_tuple, + index_row = self.con.select(SPACE_INDEX, _key_tuple, index=_index) return index_row @@ -445,12 +449,10 @@ def get_field(self, space, field): _space = self.get_space(space) try: return _space.format[field] - except: - tp = 'name' if isinstance(field, str) else 'id' - errmsg = "There's no field with {2} '{0}' in space '{1}'".format( - field, _space.name, tp - ) - raise SchemaError(errmsg) + except KeyError as exc: + kind = 'name' if isinstance(field, str) else 'id' + errmsg = f"There's no field with {kind} '{field}' in space '{_space.name}'" + raise SchemaError(errmsg) from exc return field diff --git a/tarantool/space.py b/tarantool/space.py index 0fa61198..11a3e3ca 100644 --- a/tarantool/space.py +++ b/tarantool/space.py @@ -1,11 +1,10 @@ -# pylint: disable=C0301,W0105,W0401,W0614 """ Space type definition. It is an object-oriented wrapper for requests to a Tarantool server space. """ -class Space(object): +class Space(): """ Object-oriented wrapper for accessing a particular space. Encapsulates the identifier of the space and provides a more diff --git a/tarantool/types.py b/tarantool/types.py index 5ca86150..d849dcfa 100644 --- a/tarantool/types.py +++ b/tarantool/types.py @@ -5,6 +5,7 @@ import typing from dataclasses import dataclass + @dataclass class BoxError(): """ @@ -13,6 +14,7 @@ class BoxError(): .. _box.error: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_error/error/ """ + # pylint: disable=too-many-instance-attributes type: typing.Union[str, bytes] """ @@ -76,6 +78,7 @@ class BoxError(): MP_ERROR_ERRCODE = 0x05 MP_ERROR_FIELDS = 0x06 + def decode_box_error(err_map): """ Decode MessagePack map received from Tarantool to `box.error`_ @@ -100,13 +103,14 @@ def decode_box_error(err_map): message=item[MP_ERROR_MESSAGE], errno=item[MP_ERROR_ERRNO], errcode=item[MP_ERROR_ERRCODE], - fields=item.get(MP_ERROR_FIELDS), # omitted if empty + fields=item.get(MP_ERROR_FIELDS), # omitted if empty prev=prev, ) prev = err return prev + def encode_box_error(err): """ Encode Python `box.error`_ representation to MessagePack map. @@ -131,7 +135,7 @@ def encode_box_error(err): MP_ERROR_ERRCODE: err.errcode, } - if err.fields is not None: # omitted if empty + if err.fields is not None: # omitted if empty dict_item[MP_ERROR_FIELDS] = err.fields stack.append(dict_item) diff --git a/tarantool/utils.py b/tarantool/utils.py index e5238707..b7ab436d 100644 --- a/tarantool/utils.py +++ b/tarantool/utils.py @@ -1,9 +1,14 @@ -import sys +""" +This module provides utility functions for the package. +""" + +from base64 import decodebytes as base64_decode +from dataclasses import dataclass +import typing import uuid ENCODING_DEFAULT = "utf-8" -from base64 import decodebytes as base64_decode def strxor(rhs, lhs): """ @@ -20,6 +25,7 @@ def strxor(rhs, lhs): return bytes([x ^ y for x, y in zip(rhs, lhs)]) + def wrap_key(*args, first=True, select=False): """ Wrap request key in list, if needed. @@ -41,7 +47,7 @@ def wrap_key(*args, first=True, select=False): if len(args) == 1: if isinstance(args[0], (list, tuple)) and first: return wrap_key(*args[0], first=False, select=select) - elif args[0] is None and select: + if args[0] is None and select: return [] return list(args) @@ -65,6 +71,34 @@ def version_id(major, minor, patch): return (((major << 8) | minor) << 8) | patch + +@dataclass +class Greeting(): + """ + Connection greeting info. + """ + + version_id: typing.Optional = 0 + """ + :type: :obj:`tuple` or :obj:`list` + """ + + protocol: typing.Optional[str] = None + """ + :type: :obj:`str`, optional + """ + + uuid: typing.Optional[str] = None + """ + :type: :obj:`str`, optional + """ + + salt: typing.Optional[str] = None + """ + :type: :obj:`str`, optional + """ + + def greeting_decode(greeting_buf): """ Decode Tarantool server greeting. @@ -78,12 +112,6 @@ def greeting_decode(greeting_buf): :raise: :exc:`~Exception` """ - class Greeting: - version_id = 0 - protocol = None - uuid = None - salt = None - # Tarantool 1.6.6 # Tarantool 1.6.6-102-g4e9bde2 # Tarantool 1.6.8 (Binary) 3b151c25-4c4a-4b5d-8042-0f1b3a6f61c3 @@ -92,7 +120,7 @@ class Greeting: try: (product, _, tail) = greeting_buf[0:63].decode().partition(' ') if product.startswith("Tarantool "): - raise Exception() + raise ValueError() # Parse a version string - 1.6.6-83-gc6b2129 or 1.6.7 (version, _, tail) = tail.partition(' ') version = version.split('-')[0].split('.') @@ -112,9 +140,9 @@ class Greeting: # Tarantool < 1.6.7 doesn't add "(Binary)" to greeting result.protocol = "Binary" elif len(tail.strip()) != 0: - raise Exception("x") # Unsupported greeting + raise ValueError("x") # Unsupported greeting result.salt = base64_decode(greeting_buf[64:])[:20] return result - except Exception as e: - print('exx', e) - raise ValueError("Invalid greeting: " + str(greeting_buf)) + except ValueError as exc: + print('exx', exc) + raise ValueError("Invalid greeting: " + str(greeting_buf)) from exc diff --git a/test/setup_command.py b/test/setup_command.py index da0cb6c4..9df5f8cd 100755 --- a/test/setup_command.py +++ b/test/setup_command.py @@ -1,30 +1,47 @@ #!/usr/bin/env python +""" +This module describes class implementing `python setup.py test`. +""" -import os -import sys import unittest + import setuptools -from distutils.errors import DistutilsError -from glob import glob +try: + from setuptools.errors import BaseError +except (ModuleNotFoundError, ImportError): + # pylint: disable=deprecated-module + from distutils.errors import DistutilsError as BaseError + + +class Test(setuptools.Command): + """ + Class implementing `python setup.py test`. + """ + # pylint: disable=bad-option-value,no-self-use -class test(setuptools.Command): user_options = [] description = 'Run tests' def initialize_options(self): - pass + """ + Do nothing. setuptools requires to override this abstract + method. + """ def finalize_options(self): - pass + """ + Do nothing. setuptools requires to override this abstract + method. + """ def run(self): - ''' + """ Find all tests in test/tarantool/ and run them - ''' + """ tests = unittest.defaultTestLoader.discover('test', pattern='suites') test_runner = unittest.TextTestRunner(verbosity=2) result = test_runner.run(tests) if not result.wasSuccessful(): - raise DistutilsError('There are failed tests') + raise BaseError('There are failed tests') diff --git a/test/suites/__init__.py b/test/suites/__init__.py index c8fa35d7..7d092585 100644 --- a/test/suites/__init__.py +++ b/test/suites/__init__.py @@ -1,47 +1,58 @@ +""" +Module tests entrypoint. +""" + import os import unittest -__tmp = os.getcwd() -os.chdir(os.path.abspath(os.path.dirname(__file__))) +from .test_schema import TestSuiteSchemaUnicodeConnection +from .test_schema import TestSuiteSchemaBinaryConnection +from .test_dml import TestSuiteRequest +from .test_protocol import TestSuiteProtocol +from .test_reconnect import TestSuiteReconnect +from .test_mesh import TestSuiteMesh +from .test_pool import TestSuitePool +from .test_execute import TestSuiteExecute +from .test_dbapi import TestSuiteDBAPI +from .test_encoding import TestSuiteEncoding +from .test_socket_fd import TestSuiteSocketFD +from .test_ssl import TestSuiteSsl +from .test_decimal import TestSuiteDecimal +from .test_uuid import TestSuiteUUID +from .test_datetime import TestSuiteDatetime +from .test_interval import TestSuiteInterval +from .test_package import TestSuitePackage +from .test_error_ext import TestSuiteErrorExt +from .test_push import TestSuitePush +from .test_connection import TestSuiteConnection +from .test_crud import TestSuiteCrud + +test_cases = (TestSuiteSchemaUnicodeConnection, + TestSuiteSchemaBinaryConnection, + TestSuiteRequest, TestSuiteProtocol, TestSuiteReconnect, + TestSuiteMesh, TestSuiteExecute, TestSuiteDBAPI, + TestSuiteEncoding, TestSuitePool, TestSuiteSsl, + TestSuiteDecimal, TestSuiteUUID, TestSuiteDatetime, + TestSuiteInterval, TestSuitePackage, TestSuiteErrorExt, + TestSuitePush, TestSuiteConnection, TestSuiteCrud, TestSuiteSocketFD) -from .test_schema import TestSuite_Schema_UnicodeConnection -from .test_schema import TestSuite_Schema_BinaryConnection -from .test_dml import TestSuite_Request -from .test_protocol import TestSuite_Protocol -from .test_reconnect import TestSuite_Reconnect -from .test_mesh import TestSuite_Mesh -from .test_pool import TestSuite_Pool -from .test_execute import TestSuite_Execute -from .test_dbapi import TestSuite_DBAPI -from .test_encoding import TestSuite_Encoding -from .test_ssl import TestSuite_Ssl -from .test_decimal import TestSuite_Decimal -from .test_uuid import TestSuite_UUID -from .test_datetime import TestSuite_Datetime -from .test_interval import TestSuite_Interval -from .test_package import TestSuite_Package -from .test_error_ext import TestSuite_ErrorExt -from .test_push import TestSuite_Push -from .test_connection import TestSuite_Connection -from .test_crud import TestSuite_Crud - -test_cases = (TestSuite_Schema_UnicodeConnection, - TestSuite_Schema_BinaryConnection, - TestSuite_Request, TestSuite_Protocol, TestSuite_Reconnect, - TestSuite_Mesh, TestSuite_Execute, TestSuite_DBAPI, - TestSuite_Encoding, TestSuite_Pool, TestSuite_Ssl, - TestSuite_Decimal, TestSuite_UUID, TestSuite_Datetime, - TestSuite_Interval, TestSuite_ErrorExt, TestSuite_Push, - TestSuite_Connection, TestSuite_Crud,) def load_tests(loader, tests, pattern): + """ + Add suites to test run. + """ + # pylint: disable=unused-argument + suite = unittest.TestSuite() for testc in test_cases: suite.addTests(loader.loadTestsFromTestCase(testc)) return suite +__tmp = os.getcwd() +os.chdir(os.path.abspath(os.path.dirname(__file__))) + os.chdir(__tmp) # Workaround to disable unittest output truncating -__import__('sys').modules['unittest.util']._MAX_LENGTH = 99999 +__import__('sys').modules['unittest.util']._MAX_LENGTH = 99999 # pylint: disable=protected-access diff --git a/test/suites/box.lua b/test/suites/box.lua index f551bd1c..c8110d18 100644 --- a/test/suites/box.lua +++ b/test/suites/box.lua @@ -1,11 +1,17 @@ #!/usr/bin/env tarantool +local is_compat, compat = pcall(require, "compat") local os = require('os') local admin_listen = os.getenv("ADMIN") local primary_listen = os.getenv("LISTEN") local auth_type = os.getenv("AUTH_TYPE") +local SQL_SEQ_SCAN_DEFAULT = os.getenv("SQL_SEQ_SCAN_DEFAULT") +if is_compat and SQL_SEQ_SCAN_DEFAULT then + compat.sql_seq_scan_default = SQL_SEQ_SCAN_DEFAULT +end + require('console').listen(admin_listen) box.cfg{ listen = primary_listen, @@ -13,3 +19,5 @@ box.cfg{ pid_file = "box.pid", auth_type = (auth_type:len() > 0) and auth_type or nil, } + +rawset(_G, 'ready', true) diff --git a/test/suites/crud_mock_server.lua b/test/suites/crud_mock_server.lua new file mode 100644 index 00000000..252059df --- /dev/null +++ b/test/suites/crud_mock_server.lua @@ -0,0 +1,21 @@ +#!/usr/bin/env tarantool + +local admin_listen = os.getenv("ADMIN") +local primary_listen = os.getenv("LISTEN") + +require('console').listen(admin_listen) +box.cfg{ + listen = primary_listen, + memtx_memory = 0.1 * 1024^3, -- 0.1 GiB + pid_file = "box.pid", +} + +box.schema.user.grant('guest', 'execute', 'universe', nil, {if_not_exists = true}) + +local function mock_replace() + error('Unexpected connection error') +end + +rawset(_G, 'crud', {replace = mock_replace}) + +rawset(_G, 'ready', true) diff --git a/test/suites/crud_server.lua b/test/suites/crud_server.lua index ba6fe82a..f939818a 100644 --- a/test/suites/crud_server.lua +++ b/test/suites/crud_server.lua @@ -1,7 +1,66 @@ #!/usr/bin/env tarantool -local crud = require('crud') -local vshard = require('vshard') +local function replicaset_uuid() + if box.info().replicaset ~= nil and box.info().replicaset.uuid ~= nil then + return box.info().replicaset.uuid + end + + return box.info().cluster.uuid +end + +local function configure_crud_instance(primary_listen, crud, vshard) + box.schema.create_space( + 'tester', { + format = { + {name = 'id', type = 'unsigned'}, + {name = 'bucket_id', type = 'unsigned'}, + {name = 'name', type = 'string'}, + } + }) + box.space.tester:create_index('primary_index', { + parts = { + {field = 1, type = 'unsigned'}, + }, + }) + box.space.tester:create_index('bucket_id', { + parts = { + {field = 2, type = 'unsigned'}, + }, + unique = false, + }) + + -- Setup vshard. + _G.vshard = vshard + box.once('guest', function() + box.schema.user.grant('guest', 'super') + end) + local uri = 'guest@0.0.0.0:' .. primary_listen + local cfg = { + bucket_count = 300, + sharding = { + [replicaset_uuid()] = { + replicas = { + [box.info().uuid] = { + uri = uri, + name = 'storage', + master = true, + }, + }, + }, + }, + } + vshard.storage.cfg(cfg, box.info().uuid) + vshard.router.cfg(cfg) + vshard.router.bootstrap() + + -- Initialize crud. + crud.init_storage() + crud.init_router() + crud.cfg{stats = true} +end + +local crud_imported, crud = pcall(require, 'crud') +local vshard_imported, vshard = pcall(require, 'vshard') local admin_listen = os.getenv("ADMIN") local primary_listen = os.getenv("LISTEN") @@ -18,51 +77,19 @@ box.schema.user.grant( 'read,write,execute', 'universe' ) -box.schema.create_space( - 'tester', { - format = { - {name = 'id', type = 'unsigned'}, - {name = 'bucket_id', type = 'unsigned'}, - {name = 'name', type = 'string'}, - } -}) -box.space.tester:create_index('primary_index', { - parts = { - {field = 1, type = 'unsigned'}, - }, -}) -box.space.tester:create_index('bucket_id', { - parts = { - {field = 2, type = 'unsigned'}, - }, - unique = false, -}) --- Setup vshard. -_G.vshard = vshard -box.once('guest', function() - box.schema.user.grant('guest', 'super') -end) -local uri = 'guest@0.0.0.0:' .. primary_listen -local cfg = { - bucket_count = 300, - sharding = { - [box.info().cluster.uuid] = { - replicas = { - [box.info().uuid] = { - uri = uri, - name = 'storage', - master = true, - }, - }, - }, - }, -} -vshard.storage.cfg(cfg, box.info().uuid) -vshard.router.cfg(cfg) -vshard.router.bootstrap() +if crud_imported == false or vshard_imported == false then + -- Set flag for unittest. + _G['ROCKS_IMPORT_FAIL'] = true + local fail_msg = 'The crud/vshard modules are not detected, ' .. + 'installation via rocks install is required ' .. + 'for CRUD testing purposes. You can use ' .. + ' or ' .. + ' to install modules' + -- The print output will be captured in the logs. + print(fail_msg) +else + configure_crud_instance(primary_listen, crud, vshard) +end --- Initialize crud. -crud.init_storage() -crud.init_router() -crud.cfg{stats = true} +rawset(_G, 'ready', true) diff --git a/test/suites/lib/remote_tarantool_server.py b/test/suites/lib/remote_tarantool_server.py index 5d6afd77..7d4118ba 100644 --- a/test/suites/lib/remote_tarantool_server.py +++ b/test/suites/lib/remote_tarantool_server.py @@ -1,3 +1,8 @@ +""" +This module provides helpers to work with remote Tarantool server +(used on Windows). +""" + import sys import os import random @@ -15,10 +20,17 @@ def get_random_string(): + """ + :type: :obj:`str` + """ return ''.join(random.choice(string.ascii_lowercase) for _ in range(16)) -class RemoteTarantoolServer(object): +class RemoteTarantoolServer(): + """ + Class to work with remote Tarantool server. + """ + def __init__(self): self.host = os.environ['REMOTE_TARANTOOL_HOST'] @@ -26,7 +38,7 @@ def __init__(self): self.args['primary'] = BINARY_PORT self.args['admin'] = os.environ['REMOTE_TARANTOOL_CONSOLE_PORT'] - assert(self.args['primary'] != self.args['admin']) + assert self.args['primary'] != self.args['admin'] # a name to using for a lock self.whoami = get_random_string() @@ -39,54 +51,78 @@ def __init__(self): self.admin.execute('box.cfg{listen = box.NULL}') def acquire_lock(self): + """ + Acquire lock on the remote server so no concurrent tests would run. + """ + deadline = time.time() + AWAIT_TIME while True: - res = self.admin.execute('return acquire_lock("%s")' % self.whoami) + res = self.admin.execute(f'return acquire_lock("{ self.whoami}")') ok = res[0] err = res[1] if not ok else None if ok: break if time.time() > deadline: - raise RuntimeError('can not acquire "%s" lock: %s' % ( - self.whoami, str(err))) - print('waiting to acquire "%s" lock' % self.whoami, + raise RuntimeError(f'can not acquire "{self.whoami}" lock: {str(err)}') + print(f'waiting to acquire "{self.whoami}" lock', file=sys.stderr) time.sleep(1) self.lock_is_acquired = True def touch_lock(self): - assert(self.lock_is_acquired) - res = self.admin.execute('return touch_lock("%s")' % self.whoami) + """ + Refresh locked state on the remote server so no concurrent + tests would run. + """ + + assert self.lock_is_acquired + res = self.admin.execute(f'return touch_lock("{self.whoami}")') ok = res[0] err = res[1] if not ok else None if not ok: - raise RuntimeError('can not update "%s" lock: %s' % ( - self.whoami, str(err))) + raise RuntimeError(f'can not update "{self.whoami}" lock: {str(err)}') def release_lock(self): - res = self.admin.execute('return release_lock("%s")' % self.whoami) + """ + Release loack so another test suite can run on the remote + server. + """ + + res = self.admin.execute(f'return release_lock("{self.whoami}")') ok = res[0] err = res[1] if not ok else None if not ok: - raise RuntimeError('can not release "%s" lock: %s' % ( - self.whoami, str(err))) + raise RuntimeError(f'can not release "{self.whoami}" lock: {str(err)}') self.lock_is_acquired = False def start(self): + """ + Initialize the work with the remote server. + """ + if not self.lock_is_acquired: self.acquire_lock() - self.admin.execute('box.cfg{listen = "0.0.0.0:%s"}' % - self.args['primary']) + self.admin.execute(f'box.cfg{{listen = "0.0.0.0:{self.args["primary"]}"}}') def stop(self): + """ + Finish the work with the remote server. + """ + self.admin.execute('box.cfg{listen = box.NULL}') self.release_lock() def is_started(self): + """ + Check if we still work with the remote server. + """ + return self.lock_is_acquired def clean(self): - pass + """ + Do nothing. + """ def __del__(self): self.admin.disconnect() diff --git a/test/suites/lib/skip.py b/test/suites/lib/skip.py index 5cc40d65..625caf6a 100644 --- a/test/suites/lib/skip.py +++ b/test/suites/lib/skip.py @@ -1,10 +1,16 @@ +""" +This module provides helpers to skip specific tests. +""" + import functools -import pkg_resources -import re import sys +import pkg_resources + + def fetch_tarantool_version(self): - """Helper to fetch current Tarantool version. + """ + Helper to fetch current Tarantool version. """ if not hasattr(self, 'tnt_version') or self.tnt_version is None: srv = None @@ -19,11 +25,13 @@ def fetch_tarantool_version(self): try: self.tnt_version = srv.admin.tnt_version - except: + except AttributeError: self.__class__.tnt_version = srv.admin.tnt_version -def skip_or_run_test_tarantool_impl(self, REQUIRED_TNT_VERSION, msg): - """Helper to skip or run tests depending on the Tarantool + +def skip_or_run_test_tarantool_impl(self, required_tt_version, msg): + """ + Helper to skip or run tests depending on the Tarantool version. Also, it can be used with the 'setUp' method for skipping @@ -31,14 +39,15 @@ def skip_or_run_test_tarantool_impl(self, REQUIRED_TNT_VERSION, msg): """ fetch_tarantool_version(self) - support_version = pkg_resources.parse_version(REQUIRED_TNT_VERSION) + support_version = pkg_resources.parse_version(required_tt_version) if self.tnt_version < support_version: - self.skipTest('Tarantool %s %s' % (self.tnt_version, msg)) + self.skipTest(f'Tarantool {self.tnt_version} {msg}') -def skip_or_run_test_tarantool(func, REQUIRED_TNT_VERSION, msg): - """Decorator to skip or run tests depending on the tarantool +def skip_or_run_test_tarantool(func, required_tt_version, msg): + """ + Decorator to skip or run tests depending on the tarantool version. Also, it can be used with the 'setUp' method for skipping @@ -50,15 +59,17 @@ def wrapper(self, *args, **kwargs): if func.__name__ == 'setUp': func(self, *args, **kwargs) - skip_or_run_test_tarantool_impl(self, REQUIRED_TNT_VERSION, msg) + skip_or_run_test_tarantool_impl(self, required_tt_version, msg) if func.__name__ != 'setUp': func(self, *args, **kwargs) return wrapper -def skip_or_run_test_tarantool_call(self, REQUIRED_TNT_VERSION, msg): - """Function to skip or run tests depending on the tarantool + +def skip_or_run_test_tarantool_call(self, required_tt_version, msg): + """ + Function to skip or run tests depending on the tarantool version. Useful in cases when in is inconvenient to work with decorators. @@ -66,12 +77,13 @@ def skip_or_run_test_tarantool_call(self, REQUIRED_TNT_VERSION, msg): the whole test suite. """ - skip_or_run_test_tarantool_impl(self, REQUIRED_TNT_VERSION, msg) + skip_or_run_test_tarantool_impl(self, required_tt_version, msg) -def skip_or_run_test_pcall_require(func, REQUIRED_TNT_MODULE, msg): - """Decorator to skip or run tests depending on tarantool - module requre success or fail. +def skip_or_run_test_pcall_require(func, required_tt_module, msg): + """ + Decorator to skip or run tests depending on tarantool + module require success or fail. Also, it can be used with the 'setUp' method for skipping the whole test suite. @@ -84,17 +96,17 @@ def wrapper(self, *args, **kwargs): srv = None - if hasattr(self, 'servers'): + if hasattr(self, 'servers') and self.servers: srv = self.servers[0] - if hasattr(self, 'srv'): + if hasattr(self, 'srv') and self.srv: srv = self.srv assert srv is not None - resp = srv.admin("pcall(require, '%s')" % REQUIRED_TNT_MODULE) + resp = srv.admin(f"pcall(require, '{required_tt_module}')") if not resp[0]: - self.skipTest('Tarantool %s' % (msg, )) + self.skipTest(f'Tarantool {msg}') if func.__name__ != 'setUp': func(self, *args, **kwargs) @@ -102,8 +114,9 @@ def wrapper(self, *args, **kwargs): return wrapper -def skip_or_run_test_python(func, REQUIRED_PYTHON_VERSION, msg): - """Decorator to skip or run tests depending on the Python version. +def skip_or_run_test_python(func, required_python_version, msg): + """ + Decorator to skip or run tests depending on the Python version. Also, it can be used with the 'setUp' method for skipping the whole test suite. @@ -115,11 +128,11 @@ def wrapper(self, *args, **kwargs): func(self, *args, **kwargs) ver = sys.version_info - python_version_str = '%d.%d' % (ver.major, ver.minor) + python_version_str = f'{ver.major}.{ver.minor}' python_version = pkg_resources.parse_version(python_version_str) - support_version = pkg_resources.parse_version(REQUIRED_PYTHON_VERSION) + support_version = pkg_resources.parse_version(required_python_version) if python_version < support_version: - self.skipTest('Python %s connector %s' % (python_version, msg)) + self.skipTest(f'Python {python_version} connector {msg}') if func.__name__ != 'setUp': func(self, *args, **kwargs) @@ -128,7 +141,8 @@ def wrapper(self, *args, **kwargs): def skip_or_run_sql_test(func): - """Decorator to skip or run SQL-related tests depending on the + """ + Decorator to skip or run SQL-related tests depending on the tarantool version. Tarantool supports SQL-related stuff only since 2.0.0 version. @@ -140,7 +154,8 @@ def skip_or_run_sql_test(func): def skip_or_run_varbinary_test(func): - """Decorator to skip or run VARBINARY-related tests depending on + """ + Decorator to skip or run VARBINARY-related tests depending on the tarantool version. Tarantool supports VARBINARY type only since 2.2.1 version. @@ -152,7 +167,8 @@ def skip_or_run_varbinary_test(func): def skip_or_run_decimal_test(func): - """Decorator to skip or run decimal-related tests depending on + """ + Decorator to skip or run decimal-related tests depending on the tarantool version. Tarantool supports decimal type only since 2.2.1 version. @@ -160,10 +176,12 @@ def skip_or_run_decimal_test(func): """ return skip_or_run_test_pcall_require(func, 'decimal', - 'does not support decimal type') + 'does not support decimal type') + -def skip_or_run_UUID_test(func): - """Decorator to skip or run UUID-related tests depending on +def skip_or_run_uuid_test(func): + """ + Decorator to skip or run UUID-related tests depending on the tarantool version. Tarantool supports UUID type only since 2.4.1 version. @@ -173,8 +191,10 @@ def skip_or_run_UUID_test(func): return skip_or_run_test_tarantool(func, '2.4.1', 'does not support UUID type') + def skip_or_run_datetime_test(func): - """Decorator to skip or run datetime-related tests depending on + """ + Decorator to skip or run datetime-related tests depending on the tarantool version. Tarantool supports datetime type only since 2.10.0 version. @@ -182,10 +202,25 @@ def skip_or_run_datetime_test(func): """ return skip_or_run_test_pcall_require(func, 'datetime', - 'does not support datetime type') + 'does not support datetime type') + + +def skip_or_run_datetime_2_11_test(func): + """ + Decorator to skip or run tests related to datetime module with + fixes introduced in 2.11 release. + + See https://github.com/tarantool/tarantool/issues/7698 and + https://github.com/tarantool/tarantool/issues/7700 + """ + + return skip_or_run_test_tarantool(func, '2.11.0', + 'does not provide required datetime fixes') + def skip_or_run_error_extra_info_test(func): - """Decorator to skip or run tests related to extra error info + """ + Decorator to skip or run tests related to extra error info provided over iproto depending on the tarantool version. Tarantool provides extra error info only since 2.4.1 version. @@ -195,8 +230,10 @@ def skip_or_run_error_extra_info_test(func): return skip_or_run_test_tarantool(func, '2.4.1', 'does not provide extra error info') + def skip_or_run_error_ext_type_test(func): - """Decorator to skip or run tests related to error extension + """ + Decorator to skip or run tests related to error extension type depending on the tarantool version. Tarantool supports error extension type only since 2.4.1 version, @@ -208,8 +245,10 @@ def skip_or_run_error_ext_type_test(func): return skip_or_run_test_tarantool(func, '2.10.0', 'does not support error extension type') + def skip_or_run_ssl_password_test_call(self): - """Function to skip or run tests related to SSL password + """ + Function to skip or run tests related to SSL password and SSL password files support. Supported only in Tarantool EE. Do not check Enterprise prefix since TNT_SSL_TEST already assumes it. @@ -222,8 +261,10 @@ def skip_or_run_ssl_password_test_call(self): return skip_or_run_test_tarantool_call(self, '2.11.0', 'does not support SSL passwords') + def skip_or_run_auth_type_test_call(self): - """Function to skip or run tests related to configuring + """ + Function to skip or run tests related to configuring authentication method. Tarantool supports auth_type only in current master since @@ -236,3 +277,43 @@ def skip_or_run_auth_type_test_call(self): return skip_or_run_test_tarantool_call(self, '2.11.0', 'does not support auth type') + + +def skip_or_run_constraints_test(func): + """ + Decorator to skip or run tests related to spaces with + schema constraints. + + Tarantool supports schema constraints only since 2.10.0 version. + See https://github.com/tarantool/tarantool/issues/6436 + """ + + return skip_or_run_test_tarantool(func, '2.10.0', + 'does not support schema constraints') + + +def skip_or_run_iproto_basic_features_test(func): + """ + Decorator to skip or run tests related to iproto ID requests, + protocol version and features. + + Tarantool supports iproto ID requests only since 2.10.0 version. + Protocol version is 3 for Tarantool 2.10.0, + IPROTO_FEATURE_STREAMS, IPROTO_FEATURE_TRANSACTIONS + and IPROTO_FEATURE_ERROR_EXTENSION are supported in Tarantool 2.10.0. + See https://github.com/tarantool/tarantool/issues/6253 + """ + + return skip_or_run_test_tarantool(func, '2.10.0', + 'does not support iproto ID and iproto basic features') + + +def skip_or_run_box_session_new_tests(func): + """ + Decorator to skip or run tests that use box.session.new. + + Tarantool supports box.session.new only in current master since + commit 324872a. + See https://github.com/tarantool/tarantool/issues/8801. + """ + return skip_or_run_test_tarantool(func, '3.0.0', 'does not support box.session.new') diff --git a/test/suites/lib/tarantool_admin.py b/test/suites/lib/tarantool_admin.py index 33a6d61f..de75a0e2 100644 --- a/test/suites/lib/tarantool_admin.py +++ b/test/suites/lib/tarantool_admin.py @@ -1,10 +1,19 @@ +""" +This module provides helpers to setup running Tarantool server. +""" + import socket -import yaml import re + import pkg_resources +import yaml + +class TarantoolAdmin(): + """ + Class to setup running Tarantool server. + """ -class TarantoolAdmin(object): def __init__(self, host, port): self.host = host self.port = port @@ -13,18 +22,30 @@ def __init__(self, host, port): self._tnt_version = None def connect(self): + """ + Connect to running Tarantool server. + """ + self.socket = socket.create_connection((self.host, self.port)) self.socket.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) self.is_connected = True self.socket.recv(256) # skip greeting def disconnect(self): + """ + Disconnect from the Tarantool server. + """ + if self.is_connected: self.socket.close() self.socket = None self.is_connected = False def reconnect(self): + """ + Reconnect to the running Tarantool server. + """ + self.disconnect() self.connect() @@ -32,15 +53,19 @@ def __enter__(self): self.connect() return self - def __exit__(self, type, value, tb): + def __exit__(self, exception_type, exception_value, exception_traceback): self.disconnect() def __call__(self, command): return self.execute(command) def execute(self, command): + """ + Evaluate some Lua code on the Tarantool server. + """ + if not command: - return + return None if not self.is_connected: self.connect() @@ -68,6 +93,10 @@ def execute(self, command): @property def tnt_version(self): + """ + Connected Tarantool server version. + """ + if self._tnt_version is not None: return self._tnt_version diff --git a/test/suites/lib/tarantool_server.py b/test/suites/lib/tarantool_server.py index ceecb5a7..55f6ca63 100644 --- a/test/suites/lib/tarantool_server.py +++ b/test/suites/lib/tarantool_server.py @@ -1,3 +1,7 @@ +""" +This module provides helpers start up a Tarantool server. +""" + import os import os.path import errno @@ -10,14 +14,19 @@ import shutil import subprocess -from .tarantool_admin import TarantoolAdmin - from tarantool.const import ( SSL_TRANSPORT ) +from .tarantool_admin import TarantoolAdmin +from .remote_tarantool_server import RemoteTarantoolServer + def check_port(port, rais=True): + """ + Check if port is free. + """ + try: sock = socket.create_connection(("0.0.0.0", port)) except socket.error: @@ -25,11 +34,15 @@ def check_port(port, rais=True): sock.close() if rais: - raise RuntimeError("The server is already running on port {0}".format(port)) + raise RuntimeError(f"The server is already running on port {port}") return False def find_port(port=None): + """ + Pick some free socket. + """ + if port is None: port = random.randrange(3300, 9999) while port < 9999: @@ -39,44 +52,70 @@ def find_port(port=None): return find_port(3300) -class RunnerException(object): - pass - +class TarantoolServer(): + """ + Class to start up a new Tarantool server. + """ + # pylint: disable=too-many-instance-attributes,too-many-arguments,duplicate-code,too-many-positional-arguments -class TarantoolServer(object): default_tarantool = { - "bin": "tarantool", - "logfile": "tarantool.log", - "init": "init.lua"} + "bin": "tarantool", + "logfile": "tarantool.log", + "init": "init.lua"} default_cfg = { - "custom_proc_title": "\"tarantool-python testing\"", - "memtx_memory": 0.5 * 1024**3, # 0.5 GiB - "pid_file": "\"box.pid\""} + "custom_proc_title": "\"tarantool-python testing\"", + "memtx_memory": 0.5 * 1024**3, # 0.5 GiB + "pid_file": "\"box.pid\""} @property def logfile_path(self): + """ + Path to server logs. + """ + return os.path.join(self.vardir, self.default_tarantool['logfile']) @property def cfgfile_path(self): + """ + Path to server configuration. + """ + return os.path.join(self.vardir, self.default_tarantool['config']) @property def script_path(self): + """ + Path to server init.lua script. + """ + return os.path.join(self.vardir, self.default_tarantool['init']) @property def script_dst(self): + """ + Path to server init.lua folder. + """ + return os.path.join(self.vardir, os.path.basename(self.script)) @property def script(self): - if not hasattr(self, '_script'): self._script = None + """ + Get server init.lua script. + """ + + if not hasattr(self, '_script'): + self._script = None return self._script @script.setter def script(self, val): + """ + Set server init.lua script. + """ + if val is None: if hasattr(self, '_script'): delattr(self, '_script') @@ -85,39 +124,61 @@ def script(self, val): @property def binary(self): - if not hasattr(self, '_binary'): + """ + Get Tarantool binary used to start the server. + """ + + if self._binary is None: self._binary = self.find_exe() return self._binary @property def _admin(self): + """ + Get admin connection used to set up the server. + """ + if not hasattr(self, 'admin'): self.admin = None return self.admin @_admin.setter def _admin(self, port): + """ + Set admin connection used to set up the server. + """ + try: int(port) - except ValueError: - raise ValueError("Bad port number: '%s'" % port) + except ValueError as exc: + raise ValueError(f"Bad port number: '{port}'") from exc if hasattr(self, 'admin'): del self.admin self.admin = TarantoolAdmin('0.0.0.0', port) @property def log_des(self): - if not hasattr(self, '_log_des'): - self._log_des = open(self.logfile_path, 'a') + """ + Get server log file descriptor. + """ + # pylint: disable=consider-using-with + + if self._log_des is None: + self._log_des = open(self.logfile_path, 'a', encoding='utf-8') return self._log_des @log_des.deleter def log_des(self): - if not hasattr(self, '_log_des'): + """ + Set server log file descriptor. + """ + + if self._log_des is None: return if not self._log_des.closed: self._log_des.close() - delattr(self, '_log_des') + + self._log_des = None def __new__(cls, transport=None, @@ -128,9 +189,11 @@ def __new__(cls, ssl_password=None, ssl_password_file=None, create_unix_socket=False, - auth_type=None): + auth_type=None, + sql_seq_scan_default=None): + # pylint: disable=unused-argument + if os.name == 'nt': - from .remote_tarantool_server import RemoteTarantoolServer return RemoteTarantoolServer() return super(TarantoolServer, cls).__new__(cls) @@ -143,7 +206,10 @@ def __init__(self, ssl_password=None, ssl_password_file=None, create_unix_socket=False, - auth_type=None): + auth_type=None, + sql_seq_scan_default=None): + # pylint: disable=consider-using-with + os.popen('ulimit -c unlimited').close() if create_unix_socket: @@ -171,8 +237,15 @@ def __init__(self, self.ssl_password = ssl_password self.ssl_password_file = ssl_password_file self.auth_type = auth_type + self.sql_seq_scan_default = sql_seq_scan_default + self._binary = None + self._log_des = None def find_exe(self): + """ + Find Tarantool executable. + """ + if 'TARANTOOL_BOX_PATH' in os.environ: os.environ["PATH"] = os.environ["TARANTOOL_BOX_PATH"] + os.pathsep + os.environ["PATH"] @@ -183,26 +256,34 @@ def find_exe(self): raise RuntimeError("Can't find server executable in " + os.environ["PATH"]) def generate_listen(self, port, port_only): + """ + Generate Tarantool server box.cfg listen. + """ + if not port_only and self.transport == SSL_TRANSPORT: listen = self.host + ":" + str(port) + "?transport=ssl&" if self.ssl_key_file: - listen += "ssl_key_file={}&".format(self.ssl_key_file) + listen += f"ssl_key_file={self.ssl_key_file}&" if self.ssl_cert_file: - listen += "ssl_cert_file={}&".format(self.ssl_cert_file) + listen += f"ssl_cert_file={self.ssl_cert_file}&" if self.ssl_ca_file: - listen += "ssl_ca_file={}&".format(self.ssl_ca_file) + listen += f"ssl_ca_file={self.ssl_ca_file}&" if self.ssl_ciphers: - listen += "ssl_ciphers={}&".format(self.ssl_ciphers) + listen += f"ssl_ciphers={self.ssl_ciphers}&" if self.ssl_password: - listen += "ssl_password={}&".format(self.ssl_password) + listen += f"ssl_password={self.ssl_password}&" if self.ssl_password_file: - listen += "ssl_password_file={}&".format(self.ssl_password_file) + listen += f"ssl_password_file={self.ssl_password_file}&" listen = listen[:-1] else: listen = str(port) return listen def generate_configuration(self): + """ + Generate Tarantool box.cfg values. + """ + primary_listen = self.generate_listen(self.args['primary'], False) admin_listen = self.generate_listen(self.args['admin'], True) os.putenv("LISTEN", primary_listen) @@ -211,68 +292,86 @@ def generate_configuration(self): os.putenv("AUTH_TYPE", self.auth_type) else: os.putenv("AUTH_TYPE", "") + if self.sql_seq_scan_default is not None: + os.putenv("SQL_SEQ_SCAN_DEFAULT", self.sql_seq_scan_default) def prepare_args(self): + """ + Prepare Tarantool server init.lua script. + """ + return shlex.split(self.binary if not self.script else self.script_dst) - def wait_until_started(self): - """ Wait until server is started. + def wait_until_ready(self): + """ + Wait until server is configured and ready to work. Server consists of two parts: 1) wait until server is listening on sockets - 2) wait until server tells us his status + 2) wait until server finishes executing its script """ while True: try: temp = TarantoolAdmin('0.0.0.0', self.args['admin']) - while True: - ans = temp('box.info.status')[0] - if ans in ('running', 'hot_standby', 'orphan') or ans.startswith('replica'): - temp.disconnect() - return True - elif ans in ('loading',): - continue - else: - raise Exception("Strange output for `box.info.status`: %s" % (ans)) - except socket.error as e: - if e.errno == errno.ECONNREFUSED: + ans = temp('ready')[0] + temp.disconnect() + if isinstance(ans, bool) and ans: + return True + except socket.error as exc: + if exc.errno == errno.ECONNREFUSED: time.sleep(0.1) continue raise def start(self): - # Main steps for running Tarantool\Box - # * Find binary file --DONE(find_exe -> binary) - # * Create vardir --DONE(__init__) - # * Generate cfgfile --DONE(generate_configuration) - # * (MAYBE) Copy init.lua --INSIDE - # * Concatenate arguments and - # start Tarantool\Box --DONE(prepare_args) - # * Wait unitl Tarantool\Box - # started --DONE(wait_until_started) + """ + Main steps for running Tarantool\\Box + * Find binary file --DONE(find_exe -> binary) + * Create vardir --DONE(__init__) + * Generate cfgfile --DONE(generate_configuration) + * (MAYBE) Copy init.lua --INSIDE + * Concatenate arguments and + start Tarantool\\Box --DONE(prepare_args) + * Wait until Tarantool\\Box + started --DONE(wait_until_started) + """ + # pylint: disable=consider-using-with + self.generate_configuration() if self.script: shutil.copy(self.script, self.script_dst) os.chmod(self.script_dst, 0o777) args = self.prepare_args() self.process = subprocess.Popen(args, - cwd=self.vardir, - stdout=self.log_des, - stderr=self.log_des) - self.wait_until_started() + cwd=self.vardir, + stdout=self.log_des, + stderr=self.log_des) + self.wait_until_ready() def stop(self): + """ + Stop Tarantool server. + """ + self.admin.disconnect() if self.process.poll() is None: self.process.terminate() self.process.wait() def restart(self): + """ + Restart Tarantool server. + """ + self.stop() self.start() def clean(self): + """ + Clean Tarantool resources. + """ + if os.path.isdir(self.vardir): shutil.rmtree(self.vardir) @@ -289,9 +388,14 @@ def __del__(self): self.clean() def touch_lock(self): - # A stub method to be compatible with - # RemoteTarantoolServer. - pass + """ + A stub method to be compatible with + RemoteTarantoolServer. + """ def is_started(self): + """ + Is Tarantool server has need started. + """ + return self.process is not None diff --git a/test/suites/sidecar.py b/test/suites/sidecar.py new file mode 100644 index 00000000..6bee5af7 --- /dev/null +++ b/test/suites/sidecar.py @@ -0,0 +1,16 @@ +# pylint: disable=missing-module-docstring +import os + +import tarantool + +socket_fd = int(os.environ["SOCKET_FD"]) + +conn = tarantool.connect(None, None, socket_fd=socket_fd) + +# Check user. +assert conn.eval("return box.session.user()").data[0] == "test" + +# Check db operations. +conn.insert("test", [1]) +conn.insert("test", [2]) +assert conn.select("test").data == [[1], [2]] diff --git a/test/suites/test_connection.py b/test/suites/test_connection.py index 4132d35e..4402f0b0 100644 --- a/test/suites/test_connection.py +++ b/test/suites/test_connection.py @@ -1,7 +1,13 @@ +""" +This module tests basic connection behavior. +""" +# pylint: disable=missing-class-docstring,missing-function-docstring,duplicate-code + import sys import unittest - import decimal + +import pkg_resources import msgpack import tarantool @@ -9,46 +15,62 @@ from .lib.skip import skip_or_run_decimal_test, skip_or_run_varbinary_test from .lib.tarantool_server import TarantoolServer +from .utils import assert_admin_success -class TestSuite_Connection(unittest.TestCase): + +class TestSuiteConnection(unittest.TestCase): @classmethod - def setUpClass(self): + def setUpClass(cls): print(' CONNECTION '.center(70, '='), file=sys.stderr) print('-' * 70, file=sys.stderr) - self.srv = TarantoolServer() - self.srv.script = 'test/suites/box.lua' - self.srv.start() + cls.srv = TarantoolServer() + cls.srv.script = 'test/suites/box.lua' + cls.srv.start() - self.adm = self.srv.admin - self.adm(r""" + cls.adm = cls.srv.admin + resp = cls.adm(""" box.schema.user.create('test', {password = 'test', if_not_exists = true}) - box.schema.user.grant('test', 'read,write,execute', 'universe') - - box.schema.create_space('space_varbin') - - box.space['space_varbin']:format({ - { - 'id', - type = 'number', - is_nullable = false - }, - { - 'varbin', - type = 'varbinary', - is_nullable = false, - } - }) - - box.space['space_varbin']:create_index('id', { - type = 'tree', - parts = {1, 'number'}, - unique = true}) - - box.space['space_varbin']:create_index('varbin', { - type = 'tree', - parts = {2, 'varbinary'}, - unique = true}) + box.schema.user.grant('test', 'read,write,execute', 'universe', + nil, {if_not_exists = true}) + + return true """) + assert_admin_success(resp) + + if cls.srv.admin.tnt_version >= pkg_resources.parse_version('2.2.1'): + resp = cls.adm(""" + box.schema.create_space('space_varbin', {if_not_exists = true}) + + box.space['space_varbin']:format({ + { + 'id', + type = 'number', + is_nullable = false + }, + { + 'varbin', + type = 'varbinary', + is_nullable = false, + } + }) + + box.space['space_varbin']:create_index('id', { + type = 'tree', + parts = {1, 'number'}, + unique = true, + if_not_exists = true}) + + box.space['space_varbin']:create_index('varbin', { + type = 'tree', + parts = {2, 'varbinary'}, + unique = true, + if_not_exists = true}) + + return true + """) + assert_admin_success(resp) + + cls.con = None def setUp(self): # prevent a remote tarantool from clean our session @@ -61,7 +83,7 @@ def my_ext_type_encoder(obj): if isinstance(obj, decimal.Decimal): obj = obj + 1 return msgpack.ExtType(ext_decimal.EXT_ID, ext_decimal.encode(obj, None)) - raise TypeError("Unknown type: %r" % (obj,)) + raise TypeError(f"Unknown type: {repr(obj)}") def my_packer_factory(_): return msgpack.Packer(default=my_ext_type_encoder) @@ -92,14 +114,13 @@ def test_custom_unpacker(self): def my_ext_type_decoder(code, data): if code == ext_decimal.EXT_ID: return ext_decimal.decode(data, None) - 1 - raise NotImplementedError("Unknown msgpack extension type code %d" % (code,)) + raise NotImplementedError(f"Unknown msgpack extension type code {code}") def my_unpacker_factory(_): if msgpack.version >= (1, 0, 0): return msgpack.Unpacker(ext_hook=my_ext_type_decoder, strict_map_key=False) return msgpack.Unpacker(ext_hook=my_ext_type_decoder) - self.con = tarantool.Connection(self.srv.host, self.srv.args['primary'], user='test', password='test', unpacker_factory=my_unpacker_factory) @@ -127,13 +148,13 @@ def my_unpacker_factory(_): data = bytes(bytearray.fromhex(data_hex)) space = 'space_varbin' - self.con.execute(""" - INSERT INTO "%s" VALUES (%d, x'%s'); - """ % (space, data_id, data_hex)) + self.con.execute(f""" + INSERT INTO "{space}" VALUES ({data_id}, x'{data_hex}'); + """) - resp = self.con.execute(""" - SELECT * FROM "%s" WHERE "varbin" == x'%s'; - """ % (space, data_hex)) + resp = self.con.execute(f""" + SELECT * FROM "{space}" WHERE "varbin" == x'{data_hex}'; + """) self.assertSequenceEqual(resp, [[data_id, data]]) def test_custom_unpacker_supersedes_use_list(self): @@ -150,12 +171,11 @@ def my_unpacker_factory(_): resp = self.con.eval("return {1, 2, 3}") self.assertIsInstance(resp[0], tuple) - @classmethod def tearDown(self): - if hasattr(self, 'con'): + if self.con: self.con.close() @classmethod - def tearDownClass(self): - self.srv.stop() - self.srv.clean() + def tearDownClass(cls): + cls.srv.stop() + cls.srv.clean() diff --git a/test/suites/test_crud.py b/test/suites/test_crud.py index 633b7159..2bbc5513 100644 --- a/test/suites/test_crud.py +++ b/test/suites/test_crud.py @@ -1,11 +1,18 @@ +""" +This module tests integration with tarantool/crud. +""" +# pylint: disable=missing-class-docstring,missing-function-docstring,protected-access + import re import sys import time import unittest + import tarantool -from .lib.tarantool_server import TarantoolServer from tarantool.error import DatabaseError +from .lib.tarantool_server import TarantoolServer + def create_server(): srv = TarantoolServer() @@ -15,31 +22,51 @@ def create_server(): return srv +def create_mock_server(): + srv = TarantoolServer() + srv.script = 'test/suites/crud_mock_server.lua' + srv.start() + + return srv + + @unittest.skipIf(sys.platform.startswith("win"), - "Crud tests on windows platform are not supported: " + + "Crud tests on windows platform are not supported: " "complexity of the vshard replicaset configuration") -class TestSuite_Crud(unittest.TestCase): +class TestSuiteCrud(unittest.TestCase): @classmethod - def setUpClass(self): + def setUpClass(cls): print(' CRUD '.center(70, '='), file=sys.stderr) print('-' * 70, file=sys.stderr) # Create server and extract helpful fields for tests. - self.srv = create_server() - self.host = self.srv.host - self.port = self.srv.args['primary'] + cls.srv = create_server() + cls.mock_srv = create_mock_server() + cls.host = cls.srv.host + cls.port = cls.srv.args['primary'] + cls.mock_host = cls.mock_srv.host + cls.mock_port = cls.mock_srv.args['primary'] def setUp(self): time.sleep(1) # Open connections to instance. - self.conn = tarantool.Connection(host=self.host, port=self.port, - user='guest', password='') - self.conn_mesh = tarantool.MeshConnection(host=self.host, port=self.port, - user='guest', password='') - self.conn_pool = tarantool.ConnectionPool([{'host':self.host, 'port':self.port}], - user='guest', password='') + self.conn = tarantool.Connection(host=self.host, port=self.port, + user='guest', password='', fetch_schema=False) + self.mock_conn = tarantool.Connection(host=self.mock_host, port=self.mock_port, + user='guest', password='', fetch_schema=False) + self.conn_mesh = tarantool.MeshConnection(host=self.host, port=self.port, + user='guest', password='', fetch_schema=False) + self.conn_pool = tarantool.ConnectionPool([{'host': self.host, 'port': self.port}], + user='guest', password='', + fetch_schema=False) # Time for vshard group configuration. time.sleep(1) + if self.conn.eval('return ROCKS_IMPORT_FAIL').data[0] is True: + raise unittest.SkipTest('The crud/vshard modules are not detected, ' + 'installation via rocks install is required ' + 'for CRUD testing purposes. You can use ' + ' or ' + ' to install modules') crud_test_cases = { 'crud_insert': { @@ -56,9 +83,7 @@ def setUp(self): 'args': ['tester', [1, 100, 'Bob'], {'timeout': 10}], }, 'output': { - 'str': [ - r'Duplicate key exists', - ], + 'str': [r'Duplicate key exists'], }, }, }, @@ -73,12 +98,11 @@ def setUp(self): }, 'error': { 'input': { - 'args': ['tester', {'id': 2, 'bucket_id': 100, 'name': 'Logan'}, {'timeout': 10}], + 'args': ['tester', {'id': 2, 'bucket_id': 100, 'name': 'Logan'}, + {'timeout': 10}], }, 'output': { - 'str': [ - r'Duplicate key exists', - ], + 'str': [r'Duplicate key exists'], }, }, }, @@ -86,7 +110,7 @@ def setUp(self): 'success': { 'input': { 'args': [ - 'tester', + 'tester', [ [3, 100, 'Jacob'], [4, 100, 'Wyatt'], @@ -98,17 +122,17 @@ def setUp(self): }, 'output': { 'rows': [ - [3, 100, 'Jacob'], - [4, 100, 'Wyatt'], - [5, 100, 'David'], - [6, 100, 'Leo'], - ], + [3, 100, 'Jacob'], + [4, 100, 'Wyatt'], + [5, 100, 'David'], + [6, 100, 'Leo'], + ], }, }, 'error': { 'input': { 'args': [ - 'tester', + 'tester', [ [3, 100, 'Julian'], [4, 100, 'Hudson'], @@ -119,9 +143,7 @@ def setUp(self): ], }, 'output': { - 'str': [ - r'Duplicate key exists', - ], + 'str': [r'Duplicate key exists'], 'res_rows': [[7, 100, 'Grayson'], [8, 100, 'Ezra']] }, }, @@ -130,7 +152,7 @@ def setUp(self): 'success': { 'input': { 'args': [ - 'tester', + 'tester', [ {'id': 9, 'bucket_id': 100, 'name': 'Sharar'}, {'id': 10, 'bucket_id': 100, 'name': 'Thaddeus'}, @@ -142,17 +164,17 @@ def setUp(self): }, 'output': { 'rows': [ - [9, 100, 'Sharar'], - [10, 100, 'Thaddeus'], - [11, 100, 'Tobit'], - [12, 100, 'Zeb'], - ], + [9, 100, 'Sharar'], + [10, 100, 'Thaddeus'], + [11, 100, 'Tobit'], + [12, 100, 'Zeb'], + ], }, }, 'error': { 'input': { 'args': [ - 'tester', + 'tester', [ {'id': 9, 'bucket_id': 100, 'name': 'Silvanus'}, {'id': 10, 'bucket_id': 100, 'name': 'Timeus'}, @@ -163,9 +185,7 @@ def setUp(self): ], }, 'output': { - 'str': [ - r'Duplicate key exists', - ], + 'str': [r'Duplicate key exists'], 'res_rows': [[13, 100, 'Uzzi'], [14, 100, 'Zimiri']] }, }, @@ -184,9 +204,7 @@ def setUp(self): 'args': ['no-such-space-name', [1, 100, 'Bob'], {'timeout': 10}], }, 'output': { - 'str': [ - r'GetError: Space "no-such-space-name" doesn\'t exist', - ], + 'str': [r'GetError: Space "no-such-space-name" doesn\'t exist'], }, }, }, @@ -204,9 +222,7 @@ def setUp(self): 'args': ['tester', 1, [['+', 'age', 1]], {'timeout': 10}], }, 'output': { - 'str': [ - r"UpdateError", - ], + 'str': [r"UpdateError"], }, }, }, @@ -224,9 +240,7 @@ def setUp(self): 'args': ['no-such-space-name', 1, {'timeout': 10}], }, 'output': { - 'str': [ - r'DeleteError: Space "no-such-space-name" doesn\'t exist', - ], + 'str': [r'DeleteError: Space "no-such-space-name" doesn\'t exist'], }, }, }, @@ -244,16 +258,17 @@ def setUp(self): 'args': ['tester', [1, 100, 0], {'timeout': 10}], }, 'output': { - 'str': [ - r'expected string', - ], + 'str': [r'expected string'], }, }, }, 'crud_replace_object': { 'success': { 'input': { - 'args': ['tester', {'id': 2, 'bucket_id': 100, 'name': 'Eliza'}, {'timeout': 10}], + 'args': [ + 'tester', {'id': 2, 'bucket_id': 100, 'name': 'Eliza'}, + {'timeout': 10} + ], }, 'output': { 'rows': [[2, 100, 'Eliza']], @@ -264,9 +279,7 @@ def setUp(self): 'args': ['tester', {'id': 2, 'bucket_id': 100, 'name': 0}, {'timeout': 10}], }, 'output': { - 'str': [ - r'expected string', - ], + 'str': [r'expected string'], }, }, }, @@ -274,7 +287,7 @@ def setUp(self): 'success': { 'input': { 'args': [ - 'tester', + 'tester', [ [2, 100, 'Cephus'], [3, 100, 'Esau'], @@ -286,17 +299,17 @@ def setUp(self): }, 'output': { 'rows': [ - [2, 100, 'Cephus'], - [3, 100, 'Esau'], - [4, 100, 'Haman'], - [5, 100, 'Gershon'], - ], + [2, 100, 'Cephus'], + [3, 100, 'Esau'], + [4, 100, 'Haman'], + [5, 100, 'Gershon'], + ], }, }, 'error': { 'input': { 'args': [ - 'tester', + 'tester', [ [3, 100, 'Ephron'], [4, 100, 'Ethan'], @@ -307,9 +320,7 @@ def setUp(self): ], }, 'output': { - 'str': [ - r'expected string', - ], + 'str': [r'expected string'], 'res_rows': [[3, 100, 'Ephron'], [4, 100, 'Ethan']] }, }, @@ -318,7 +329,7 @@ def setUp(self): 'success': { 'input': { 'args': [ - 'tester', + 'tester', [ {'id': 2, 'bucket_id': 100, 'name': 'Cephus'}, {'id': 3, 'bucket_id': 100, 'name': 'Esau'}, @@ -330,17 +341,17 @@ def setUp(self): }, 'output': { 'rows': [ - [2, 100, 'Cephus'], - [3, 100, 'Esau'], - [4, 100, 'Haman'], - [5, 100, 'Gershon'], - ], + [2, 100, 'Cephus'], + [3, 100, 'Esau'], + [4, 100, 'Haman'], + [5, 100, 'Gershon'], + ], }, }, 'error': { 'input': { 'args': [ - 'tester', + 'tester', [ {'id': 3, 'bucket_id': 100, 'name': 'Ephron'}, {'id': 4, 'bucket_id': 100, 'name': 'Ethan'}, @@ -351,9 +362,7 @@ def setUp(self): ], }, 'output': { - 'str': [ - r'expected string', - ], + 'str': [r'expected string'], 'res_rows': [[3, 100, 'Ephron'], [4, 100, 'Ethan']] }, }, @@ -361,7 +370,8 @@ def setUp(self): 'crud_upsert': { 'success': { 'input': { - 'args': ['tester', [2, 100, 'Cephus'], [['+', 'bucket_id', 1]], {'timeout': 10}], + 'args': ['tester', [2, 100, 'Cephus'], [['+', 'bucket_id', 1]], + {'timeout': 10}], }, 'output': { 'rows': [], @@ -372,17 +382,15 @@ def setUp(self): 'args': ['tester', [2, 100, 'Cephus'], [['+', 'age', 1]], {'timeout': 10}], }, 'output': { - 'str': [ - r"UpsertError", - ], + 'str': [r"UpsertError"], }, }, }, 'crud_upsert_object': { 'success': { 'input': { - 'args': ['tester', {'id': 2, 'bucket_id': 100, 'name': 'Cephus'}, - [['+', 'bucket_id', 1]], {'timeout': 10}], + 'args': ['tester', {'id': 2, 'bucket_id': 100, 'name': 'Cephus'}, + [['+', 'bucket_id', 1]], {'timeout': 10}], }, 'output': { 'rows': [], @@ -390,13 +398,11 @@ def setUp(self): }, 'error': { 'input': { - 'args': ['tester', {'id': 2, 'bucket_id': 100, 'name': 'Cephus'}, - [['+', 'age', 1]], {'timeout': 10}], + 'args': ['tester', {'id': 2, 'bucket_id': 100, 'name': 'Cephus'}, + [['+', 'age', 1]], {'timeout': 10}], }, 'output': { - 'str': [ - r"UpsertError", - ], + 'str': [r"UpsertError"], }, }, }, @@ -419,7 +425,7 @@ def setUp(self): 'error': { 'input': { 'args': [ - 'tester', + 'tester', [ [[3, 100, 'Ephron'], [['+', 'bucket_id', 1]]], [[4, 100, 'Ethan'], [['+', 'bucket_id', 1]]], @@ -430,9 +436,7 @@ def setUp(self): ], }, 'output': { - 'str': [ - r'expected string', - ], + 'str': [r'expected string'], }, }, }, @@ -442,10 +446,22 @@ def setUp(self): 'args': [ 'tester', [ - [{'id': 2, 'bucket_id': 100, 'name': 'Cephus'}, [['+', 'bucket_id', 1]]], - [{'id': 3, 'bucket_id': 100, 'name': 'Esau'}, [['+', 'bucket_id', 1]]], - [{'id': 4, 'bucket_id': 100, 'name': 'Haman'}, [['+', 'bucket_id', 1]]], - [{'id': 5, 'bucket_id': 100, 'name': 'Gershon'}, [['+', 'bucket_id', 1]]], + [ + {'id': 2, 'bucket_id': 100, 'name': 'Cephus'}, + [['+', 'bucket_id', 1]] + ], + [ + {'id': 3, 'bucket_id': 100, 'name': 'Esau'}, + [['+', 'bucket_id', 1]] + ], + [ + {'id': 4, 'bucket_id': 100, 'name': 'Haman'}, + [['+', 'bucket_id', 1]] + ], + [ + {'id': 5, 'bucket_id': 100, 'name': 'Gershon'}, + [['+', 'bucket_id', 1]] + ], ], {'timeout': 10}, ], @@ -455,20 +471,30 @@ def setUp(self): 'error': { 'input': { 'args': [ - 'tester', + 'tester', [ - [{'id': 3, 'bucket_id': 100, 'name': 'Ephron'}, [['+', 'bucket_id', 1]]], - [{'id': 4, 'bucket_id': 100, 'name': 'Ethan'}, [['+', 'bucket_id', 1]]], - [{'id': 7, 'bucket_id': 100, 'name': 0}, [['+', 'bucket_id', 1]]], - [{'id': 8, 'bucket_id': 100, 'name': 0}, [['+', 'bucket_id', 1]]], + [ + {'id': 3, 'bucket_id': 100, 'name': 'Ephron'}, + [['+', 'bucket_id', 1]] + ], + [ + {'id': 4, 'bucket_id': 100, 'name': 'Ethan'}, + [['+', 'bucket_id', 1]] + ], + [ + {'id': 7, 'bucket_id': 100, 'name': 0}, + [['+', 'bucket_id', 1]] + ], + [ + {'id': 8, 'bucket_id': 100, 'name': 0}, + [['+', 'bucket_id', 1]] + ], ], {'timeout': 10}, ], }, 'output': { - 'str': [ - r'expected string', - ], + 'str': [r'expected string'], }, }, }, @@ -486,9 +512,7 @@ def setUp(self): 'args': ['no-such-space-name'], }, 'output': { - 'str': [ - r'SelectError: Space "no-such-space-name" doesn\'t exist', - ], + 'str': [r'SelectError: Space "no-such-space-name" doesn\'t exist'], }, }, }, @@ -506,9 +530,7 @@ def setUp(self): 'args': ['tester', 'no-idx'], }, 'output': { - 'str': [ - r'BorderError: Index "no-idx" of space "tester" doesn\'t exist', - ], + 'str': [r'BorderError: Index "no-idx" of space "tester" doesn\'t exist'], }, }, }, @@ -526,9 +548,7 @@ def setUp(self): 'args': ['tester', 'no-idx', {'timeout': 10}], }, 'output': { - 'str': [ - r'BorderError: Index "no-idx" of space "tester" doesn\'t exist', - ], + 'str': [r'BorderError: Index "no-idx" of space "tester" doesn\'t exist'], }, }, }, @@ -546,9 +566,7 @@ def setUp(self): 'args': ['no-such-space-name', {'timeout': 10}], }, 'output': { - 'str': [ - r'LenError: Space "no-such-space-name" doesn\'t exist', - ], + 'str': [r'LenError: Space "no-such-space-name" doesn\'t exist'], }, }, }, @@ -566,9 +584,7 @@ def setUp(self): 'args': ['no-such-space-name', [['==', 'bucket_id', 100]], {'timeout': 10}], }, 'output': { - 'str': [ - r'CountError: Space "no-such-space-name" doesn\'t exist', - ], + 'str': [r'CountError: Space "no-such-space-name" doesn\'t exist'], }, }, }, @@ -577,34 +593,34 @@ def setUp(self): 'input': { 'args': [ [ - [1, 100, 'Mike'], - [2, 100, 'Mike'], - [3, 100, 'Mike'], - [4, 100, 'Mike'], - [5, 200, 'Bill'], + [1, 100, 'Mike'], + [2, 100, 'Mike'], + [3, 100, 'Mike'], + [4, 100, 'Mike'], + [5, 200, 'Bill'], [6, 300, 'Rob'], ], [ - {'name': 'id', 'type': 'unsigned'}, - {'name': 'bucket_id', 'type': 'unsigned'}, + {'name': 'id', 'type': 'unsigned'}, + {'name': 'bucket_id', 'type': 'unsigned'}, {'name': 'name', 'type': 'string'} ], ], }, 'output': { 'scalar': [ - {'bucket_id': 100, 'name': 'Mike', 'id': 1}, - {'bucket_id': 100, 'name': 'Mike', 'id': 2}, - {'bucket_id': 100, 'name': 'Mike', 'id': 3}, - {'bucket_id': 100, 'name': 'Mike', 'id': 4}, - {'bucket_id': 200, 'name': 'Bill', 'id': 5}, + {'bucket_id': 100, 'name': 'Mike', 'id': 1}, + {'bucket_id': 100, 'name': 'Mike', 'id': 2}, + {'bucket_id': 100, 'name': 'Mike', 'id': 3}, + {'bucket_id': 100, 'name': 'Mike', 'id': 4}, + {'bucket_id': 200, 'name': 'Bill', 'id': 5}, {'bucket_id': 300, 'name': 'Rob', 'id': 6}, ], }, }, 'error': { 'input': { - 'args': [[],[]], + 'args': [[], []], }, 'output': { 'str': [], @@ -625,9 +641,7 @@ def setUp(self): 'args': ['no-such-space-name', {'timeout': 10}], }, 'output': { - 'str': [ - r'"no-such-space-name" doesn\'t exist', - ], + 'str': [r'"no-such-space-name" doesn\'t exist'], }, }, }, @@ -638,13 +652,13 @@ def setUp(self): }, 'output': { 'operations': [ - 'insert', 'replace', - 'upsert', 'len', - 'delete', 'get', - 'select', 'borders', - 'update', 'count', - 'truncate', - ], + 'insert', 'replace', + 'upsert', 'len', + 'delete', 'get', + 'select', 'borders', + 'update', 'count', + 'truncate', + ], }, }, 'error': { @@ -669,17 +683,17 @@ def _correct_operation_with_crud(self, testing_function, case, mode=None): *case['success']['input']['args'], ) if 'rows' in case['success']['output']: - # Case for crud responce as tarantool.crud.CrudResult obj. + # Case for crud response as tarantool.crud.CrudResult obj. self.assertEqual(resp.rows, case['success']['output']['rows']) if 'scalar' in case['success']['output']: - # Case for scalar value as crud responce, not tarantool.crud.CrudResult obj. + # Case for scalar value as crud response, not tarantool.crud.CrudResult obj. self.assertEqual(resp, case['success']['output']['scalar']) if 'operations' in case['success']['output']: # Case for statistics testing. for operation in case['success']['output']['operations']: - self.assertEqual(operation in resp.__dict__, True, - 'Problem with finding a field with a statistic about operation ' - + operation) + self.assertEqual( + operation in resp.__dict__, True, + 'Problem with finding a field with a statistic about operation ' + operation) def _exception_operation_with_crud(self, testing_function, case, mode=None): try: @@ -692,60 +706,65 @@ def _exception_operation_with_crud(self, testing_function, case, mode=None): _ = testing_function( *case['error']['input']['args'], ) - except DatabaseError as e: + except DatabaseError as exc: for regexp_case in case['error']['output']['str']: - if hasattr(e, 'extra_info_error'): + if hasattr(exc, 'extra_info_error'): # Case for non-batch operations. - self.assertNotEqual(re.search(regexp_case, e.extra_info_error.str), None) - if hasattr(e, 'errors_list'): + self.assertNotEqual(re.search(regexp_case, exc.extra_info_error.str), None) + if hasattr(exc, 'errors_list'): # Case for *_many() operations. err_sum = str() - for err in e.errors_list: + for err in exc.errors_list: err_sum = err_sum + err.str self.assertNotEqual(re.search(regexp_case, err_sum), None) - if hasattr(e, 'success_list'): + if hasattr(exc, 'success_list'): # Case for *_many() operations. if 'res_rows' in case['error']['output']: - self.assertEqual(e.success_list.rows, case['error']['output']['res_rows']) + self.assertEqual(exc.success_list.rows, case['error']['output']['res_rows']) def test_crud_module_via_connection(self): - for case_name in self.crud_test_cases.keys(): - with self.subTest(name=case_name): - case = self.crud_test_cases[case_name] - testing_function = getattr(self.conn, case_name) + for name, case in self.crud_test_cases.items(): + with self.subTest(name=name): + testing_function = getattr(self.conn, name) # Correct try testing. self._correct_operation_with_crud(testing_function, case) # Exception try testing. self._exception_operation_with_crud(testing_function, case) def test_crud_module_via_mesh_connection(self): - for case_name in self.crud_test_cases.keys(): - with self.subTest(name=case_name): - case = self.crud_test_cases[case_name] - testing_function = getattr(self.conn_mesh, case_name) + for name, case in self.crud_test_cases.items(): + with self.subTest(name=name): + testing_function = getattr(self.conn_mesh, name) # Correct try testing. self._correct_operation_with_crud(testing_function, case) # Exception try testing. self._exception_operation_with_crud(testing_function, case) def test_crud_module_via_pool_connection(self): - for case_name in self.crud_test_cases.keys(): - with self.subTest(name=case_name): - case = self.crud_test_cases[case_name] - testing_function = getattr(self.conn_pool, case_name) + for name, case in self.crud_test_cases.items(): + with self.subTest(name=name): + testing_function = getattr(self.conn_pool, name) # Correct try testing. self._correct_operation_with_crud(testing_function, case, mode=tarantool.Mode.RW) # Exception try testing. self._exception_operation_with_crud(testing_function, case, mode=tarantool.Mode.RW) + def test_error_rethrow(self): + self.assertRaisesRegex( + DatabaseError, "Unexpected connection error", + lambda: self.mock_conn.crud_replace('tester', [2, 100, 'Alice'], {'timeout': 10})) + def tearDown(self): # Close connections to instance. self.conn.close() + self.mock_conn.close() self.conn_mesh.close() self.conn_pool.close() @classmethod - def tearDownClass(self): + def tearDownClass(cls): # Stop instance. - self.srv.stop() - self.srv.clean() + cls.srv.stop() + cls.srv.clean() + cls.mock_srv.stop() + cls.mock_srv.clean() diff --git a/test/suites/test_datetime.py b/test/suites/test_datetime.py index fb2d2855..80958fdc 100644 --- a/test/suites/test_datetime.py +++ b/test/suites/test_datetime.py @@ -1,47 +1,56 @@ +""" +This module tests work with datetime type. +""" +# pylint: disable=missing-class-docstring,missing-function-docstring,too-many-public-methods,too-many-function-args,duplicate-code + import sys import re import unittest + import msgpack -import warnings -import tarantool -import pandas +import tarantool +from tarantool.error import MsgpackError from tarantool.msgpack_ext.packer import default as packer_default from tarantool.msgpack_ext.unpacker import ext_hook as unpacker_ext_hook from .lib.tarantool_server import TarantoolServer -from .lib.skip import skip_or_run_datetime_test -from tarantool.error import MsgpackError, MsgpackWarning +from .lib.skip import skip_or_run_datetime_test, skip_or_run_datetime_2_11_test +from .utils import assert_admin_success + -class TestSuite_Datetime(unittest.TestCase): +class TestSuiteDatetime(unittest.TestCase): @classmethod - def setUpClass(self): + def setUpClass(cls): print(' DATETIME EXT TYPE '.center(70, '='), file=sys.stderr) print('-' * 70, file=sys.stderr) - self.srv = TarantoolServer() - self.srv.script = 'test/suites/box.lua' - self.srv.start() + cls.srv = TarantoolServer() + cls.srv.script = 'test/suites/box.lua' + cls.srv.start() - self.adm = self.srv.admin - self.adm(r""" + cls.adm = cls.srv.admin + resp = cls.adm(""" _, datetime = pcall(require, 'datetime') - box.schema.space.create('test') + box.schema.space.create('test', {if_not_exists = true}) box.space['test']:create_index('primary', { type = 'tree', parts = {1, 'string'}, - unique = true}) + unique = true, + if_not_exists = true}) pcall(function() box.schema.space.create('test_pk') box.space['test_pk']:create_index('primary', { type = 'tree', parts = {1, 'datetime'}, - unique = true}) + unique = true, + if_not_exists = true}) end) box.schema.user.create('test', {password = 'test', if_not_exists = true}) - box.schema.user.grant('test', 'read,write,execute', 'universe') + box.schema.user.grant('test', 'read,write,execute', 'universe', + nil, {if_not_exists = true}) local function add(arg1, arg2) return arg1 + arg2 @@ -52,53 +61,60 @@ def setUpClass(self): return arg1 - arg2 end rawset(_G, 'sub', sub) + + return true """) + assert_admin_success(resp) - self.con = tarantool.Connection(self.srv.host, self.srv.args['primary'], - user='test', password='test') + cls.con = tarantool.Connection(cls.srv.host, cls.srv.args['primary'], + user='test', password='test') def setUp(self): # prevent a remote tarantool from clean our session if self.srv.is_started(): self.srv.touch_lock() - self.adm("box.space['test']:truncate()") - - - def test_Datetime_class_API(self): - dt = tarantool.Datetime(year=2022, month=8, day=31, hour=18, minute=7, sec=54, - nsec=308543321, tzoffset=180) - - self.assertEqual(dt.year, 2022) - self.assertEqual(dt.month, 8) - self.assertEqual(dt.day, 31) - self.assertEqual(dt.hour, 18) - self.assertEqual(dt.minute, 7) - self.assertEqual(dt.sec, 54) - self.assertEqual(dt.nsec, 308543321) - # Both Tarantool and pandas prone to precision loss for timestamp() floats - self.assertEqual(dt.timestamp, 1661958474.308543) - self.assertEqual(dt.tzoffset, 180) - self.assertEqual(dt.tz, '') - self.assertEqual(dt.value, 1661958474308543321) - - def test_Datetime_class_API_wth_tz(self): - dt = tarantool.Datetime(year=2022, month=8, day=31, hour=18, minute=7, sec=54, - nsec=308543321, tzoffset=123, tz='Europe/Moscow') - - self.assertEqual(dt.year, 2022) - self.assertEqual(dt.month, 8) - self.assertEqual(dt.day, 31) - self.assertEqual(dt.hour, 18) - self.assertEqual(dt.minute, 7) - self.assertEqual(dt.sec, 54) - self.assertEqual(dt.nsec, 308543321) - # Both Tarantool and pandas prone to precision loss for timestamp() floats - self.assertEqual(dt.timestamp, 1661958474.308543) - self.assertEqual(dt.tzoffset, 180) - self.assertEqual(dt.tz, 'Europe/Moscow') - self.assertEqual(dt.value, 1661958474308543321) - + resp = self.adm(""" + box.space['test']:truncate() + return true + """) + assert_admin_success(resp) + + def test_datetime_class_api(self): + datetime = tarantool.Datetime(year=2022, month=8, day=31, hour=18, minute=7, sec=54, + nsec=308543321, tzoffset=180) + + self.assertEqual(datetime.year, 2022) + self.assertEqual(datetime.month, 8) + self.assertEqual(datetime.day, 31) + self.assertEqual(datetime.hour, 18) + self.assertEqual(datetime.minute, 7) + self.assertEqual(datetime.sec, 54) + self.assertEqual(datetime.nsec, 308543321) + # Both Tarantool and python prone to precision loss for timestamp() floats + self.assertEqual(datetime.timestamp, 1661958474.3085432) + self.assertEqual(datetime.tzoffset, 180) + self.assertEqual(datetime.tz, '') + self.assertEqual(datetime.value, 1661958474308543321) + self.assertEqual(str(datetime), '2022-08-31T18:07:54.308543321+03:00') + + def test_datetime_class_api_wth_tz(self): + datetime = tarantool.Datetime(year=2022, month=8, day=31, hour=18, minute=7, sec=54, + nsec=308543321, tzoffset=123, tz='Europe/Moscow') + + self.assertEqual(datetime.year, 2022) + self.assertEqual(datetime.month, 8) + self.assertEqual(datetime.day, 31) + self.assertEqual(datetime.hour, 18) + self.assertEqual(datetime.minute, 7) + self.assertEqual(datetime.sec, 54) + self.assertEqual(datetime.nsec, 308543321) + # Both Tarantool and python prone to precision loss for timestamp() floats + self.assertEqual(datetime.timestamp, 1661958474.3085432) + self.assertEqual(datetime.tzoffset, 180) + self.assertEqual(datetime.tz, 'Europe/Moscow') + self.assertEqual(datetime.value, 1661958474308543321) + self.assertEqual(str(datetime), '2022-08-31T18:07:54.308543321+03:00') datetime_class_invalid_init_cases = { 'positional_year': { @@ -137,17 +153,35 @@ def test_Datetime_class_API_wth_tz(self): 'type': ValueError, 'msg': 'Failed to create datetime with ambiguous timezone "AET"' }, + 'under_min_timestamp_1': { + 'args': [], + 'kwargs': {'timestamp': -62135596801}, + 'type': OverflowError, + 'msg': 'date value out of range' + }, + 'under_min_timestamp_2': { + 'args': [], + 'kwargs': {'timestamp': -62135596800, 'nsec': -1}, + 'type': OverflowError, + 'msg': 'date value out of range' + }, + 'over_max_timestamp': { + 'args': [], + 'kwargs': {'timestamp': 253402300800}, + 'type': OverflowError, + 'msg': 'date value out of range' + }, } - def test_Datetime_class_invalid_init(self): - for name in self.datetime_class_invalid_init_cases.keys(): + def test_datetime_class_invalid_init(self): + # pylint: disable=cell-var-from-loop + + for name, case in self.datetime_class_invalid_init_cases.items(): with self.subTest(msg=name): - case = self.datetime_class_invalid_init_cases[name] self.assertRaisesRegex( case['type'], re.escape(case['msg']), lambda: tarantool.Datetime(*case['args'], **case['kwargs'])) - integration_cases = { 'date': { 'python': tarantool.Datetime(year=2022, month=8, day=31), @@ -178,14 +212,14 @@ def test_Datetime_class_invalid_init(self): 'python': tarantool.Datetime(year=2022, month=8, day=31, hour=18, minute=7, sec=54, nsec=308543000), 'msgpack': (b'\x7a\xa3\x0f\x63\x00\x00\x00\x00\x18\xfe\x63\x12\x00\x00\x00\x00'), - 'tarantool': r"datetime.new({year=2022, month=8, day=31, hour=18, min=7, sec=54, " + + 'tarantool': r"datetime.new({year=2022, month=8, day=31, hour=18, min=7, sec=54, " r"nsec=308543000})", }, 'datetime_with_nanoseconds': { 'python': tarantool.Datetime(year=2022, month=8, day=31, hour=18, minute=7, sec=54, nsec=308543321), 'msgpack': (b'\x7a\xa3\x0f\x63\x00\x00\x00\x00\x59\xff\x63\x12\x00\x00\x00\x00'), - 'tarantool': r"datetime.new({year=2022, month=8, day=31, hour=18, min=7, sec=54, " + + 'tarantool': r"datetime.new({year=2022, month=8, day=31, hour=18, min=7, sec=54, " r"nsec=308543321})", }, 'date_before_1970_with_nanoseconds': { @@ -207,14 +241,14 @@ def test_Datetime_class_invalid_init(self): 'python': tarantool.Datetime(year=2022, month=8, day=31, hour=18, minute=7, sec=54, nsec=308543321, tzoffset=180), 'msgpack': (b'\x4a\x79\x0f\x63\x00\x00\x00\x00\x59\xff\x63\x12\xb4\x00\x00\x00'), - 'tarantool': r"datetime.new({year=2022, month=8, day=31, hour=18, min=7, sec=54, " + + 'tarantool': r"datetime.new({year=2022, month=8, day=31, hour=18, min=7, sec=54, " r"nsec=308543321, tzoffset=180})", }, 'datetime_with_negative_offset': { 'python': tarantool.Datetime(year=2022, month=8, day=31, hour=18, minute=7, sec=54, nsec=308543321, tzoffset=-60), 'msgpack': (b'\x8a\xb1\x0f\x63\x00\x00\x00\x00\x59\xff\x63\x12\xc4\xff\x00\x00'), - 'tarantool': r"datetime.new({year=2022, month=8, day=31, hour=18, min=7, sec=54, " + + 'tarantool': r"datetime.new({year=2022, month=8, day=31, hour=18, min=7, sec=54, " r"nsec=308543321, tzoffset=-60})", }, 'timestamp_with_positive_offset': { @@ -241,7 +275,7 @@ def test_Datetime_class_invalid_init(self): 'python': tarantool.Datetime(year=2022, month=8, day=31, hour=18, minute=7, sec=54, nsec=308543321, tz='Europe/Moscow'), 'msgpack': (b'\x4a\x79\x0f\x63\x00\x00\x00\x00\x59\xff\x63\x12\xb4\x00\xb3\x03'), - 'tarantool': r"datetime.new({year=2022, month=8, day=31, hour=18, min=7, sec=54, " + + 'tarantool': r"datetime.new({year=2022, month=8, day=31, hour=18, min=7, sec=54, " r"nsec=308543321, tz='Europe/Moscow'})", }, 'datetime_with_tz_winter_time': { @@ -253,69 +287,84 @@ def test_Datetime_class_invalid_init(self): 'python': tarantool.Datetime(year=2022, month=8, day=31, hour=18, minute=7, sec=54, nsec=308543321, tz='Europe/Moscow', tzoffset=123), 'msgpack': (b'\x4a\x79\x0f\x63\x00\x00\x00\x00\x59\xff\x63\x12\xb4\x00\xb3\x03'), - 'tarantool': r"datetime.new({year=2022, month=8, day=31, hour=18, min=7, sec=54, " + + 'tarantool': r"datetime.new({year=2022, month=8, day=31, hour=18, min=7, sec=54, " r"nsec=308543321, tz='Europe/Moscow', tzoffset=123})", }, 'datetime_with_abbrev_tz': { 'python': tarantool.Datetime(year=2022, month=8, day=31, hour=18, minute=7, sec=54, nsec=308543321, tz='MSK'), 'msgpack': (b'\x4a\x79\x0f\x63\x00\x00\x00\x00\x59\xff\x63\x12\xb4\x00\xee\x00'), - 'tarantool': r"datetime.new({year=2022, month=8, day=31, hour=18, min=7, sec=54, " + + 'tarantool': r"datetime.new({year=2022, month=8, day=31, hour=18, min=7, sec=54, " r"nsec=308543321, tz='MSK'})", }, 'datetime_with_abbrev_tz_and_zero_offset': { 'python': tarantool.Datetime(year=2022, month=8, day=31, hour=18, minute=7, sec=54, nsec=308543321, tz='AZODT'), 'msgpack': (b'\x7a\xa3\x0f\x63\x00\x00\x00\x00\x59\xff\x63\x12\x00\x00\x12\x02'), - 'tarantool': r"datetime.new({year=2022, month=8, day=31, hour=18, min=7, sec=54, " + + 'tarantool': r"datetime.new({year=2022, month=8, day=31, hour=18, min=7, sec=54, " r"nsec=308543321, tz='AZODT'})", }, 'timestamp_since_utc_epoch': { 'python': tarantool.Datetime(timestamp=1661958474, nsec=308543321, tz='Europe/Moscow', timestamp_since_utc_epoch=True), 'msgpack': (b'\x4a\x79\x0f\x63\x00\x00\x00\x00\x59\xff\x63\x12\xb4\x00\xb3\x03'), - 'tarantool': r"datetime.new({timestamp=1661969274, nsec=308543321, tz='Europe/Moscow'})", + 'tarantool': r"datetime.new({timestamp=1661969274, nsec=308543321, " + r"tz='Europe/Moscow'})", + }, + 'min_datetime': { # Python datetime.MINYEAR is 1. + 'python': tarantool.Datetime(year=1, month=1, day=1, hour=0, minute=0, sec=0), + 'msgpack': (b'\x00\x09\x6e\x88\xf1\xff\xff\xff'), + 'tarantool': r"datetime.new({year=1, month=1, day=1, hour=0, min=0, sec=0})", + }, + 'max_datetime': { # Python datetime.MAXYEAR is 9999. + 'python': tarantool.Datetime(year=9999, month=12, day=31, hour=23, minute=59, sec=59, + nsec=999999999), + 'msgpack': (b'\x7f\x41\xf4\xff\x3a\x00\x00\x00\xff\xc9\x9a\x3b\x00\x00\x00\x00'), + 'tarantool': r"datetime.new({year=9999, month=12, day=31, hour=23, min=59, sec=59," + r"nsec=999999999})", + }, + 'min_datetime_timestamp': { # Python datetime.MINYEAR is 1. + 'python': tarantool.Datetime(timestamp=-62135596800), + 'msgpack': (b'\x00\x09\x6e\x88\xf1\xff\xff\xff'), + 'tarantool': r"datetime.new({timestamp=-62135596800})", + }, + 'max_datetime_timestamp': { # Python datetime.MAXYEAR is 9999. + 'python': tarantool.Datetime(timestamp=253402300799, nsec=999999999), + 'msgpack': (b'\x7f\x41\xf4\xff\x3a\x00\x00\x00\xff\xc9\x9a\x3b\x00\x00\x00\x00'), + 'tarantool': r"datetime.new({timestamp=253402300799, nsec=999999999})", }, } def test_msgpack_decode(self): - for name in self.integration_cases.keys(): + for name, case in self.integration_cases.items(): with self.subTest(msg=name): - case = self.integration_cases[name] - self.assertEqual(unpacker_ext_hook(4, case['msgpack']), case['python']) @skip_or_run_datetime_test def test_tarantool_decode(self): - for name in self.integration_cases.keys(): + for name, case in self.integration_cases.items(): with self.subTest(msg=name): - case = self.integration_cases[name] - self.adm(f"box.space['test']:replace{{'{name}', {case['tarantool']}, 'field'}}") self.assertSequenceEqual(self.con.select('test', name), [[name, case['python'], 'field']]) def test_msgpack_encode(self): - for name in self.integration_cases.keys(): + for name, case in self.integration_cases.items(): with self.subTest(msg=name): - case = self.integration_cases[name] - self.assertEqual(packer_default(case['python']), msgpack.ExtType(code=4, data=case['msgpack'])) @skip_or_run_datetime_test def test_tarantool_encode(self): - for name in self.integration_cases.keys(): + for name, case in self.integration_cases.items(): with self.subTest(msg=name): - case = self.integration_cases[name] - self.con.insert('test', [name, case['python'], 'field']) lua_eval = f""" local dt = {case['tarantool']} - + local tuple = box.space['test']:get('{name}') assert(tuple ~= nil) @@ -341,7 +390,6 @@ def test_msgpack_decode_ambiguous_tzindex(self): ValueError, 'Failed to create datetime with ambiguous timezone "AET"', lambda: unpacker_ext_hook(4, case)) - datetime_subtraction_cases = { 'date': { 'arg_1': tarantool.Datetime(year=2008, month=2, day=3), @@ -370,41 +418,37 @@ def test_msgpack_decode_ambiguous_tzindex(self): } def test_python_datetime_subtraction(self): - for name in self.datetime_subtraction_cases.keys(): + for name, case in self.datetime_subtraction_cases.items(): with self.subTest(msg=name): - case = self.datetime_subtraction_cases[name] - self.assertEqual(case['arg_1'] - case['arg_2'], case['res']) @skip_or_run_datetime_test def test_tarantool_datetime_subtraction(self): - for name in self.datetime_subtraction_cases.keys(): + for name, case in self.datetime_subtraction_cases.items(): with self.subTest(msg=name): - case = self.datetime_subtraction_cases[name] - self.assertSequenceEqual(self.con.call('sub', case['arg_1'], case['arg_2']), [case['res']]) - datetime_subtraction_different_timezones_case = { 'arg_1': tarantool.Datetime(year=2001, month=2, day=3, tz='UTC'), 'arg_2': tarantool.Datetime(year=2001, month=2, day=3, tz='MSK'), - 'res': tarantool.Interval(day=1, hour=-21), + # Tarantool datetime comparison is naive, our tarantool.Interval comparison is naive too. + # So even though day=1, hour=-21 is the same as minute=180, test assertion fails. + 'res_python': tarantool.Interval(day=1, hour=-21), + 'res_tarantool': tarantool.Interval(minute=180), } def test_python_datetime_subtraction_different_timezones(self): case = self.datetime_subtraction_different_timezones_case - self.assertEqual(case['arg_1'] - case['arg_2'], case['res']) + self.assertEqual(case['arg_1'] - case['arg_2'], case['res_python']) - @skip_or_run_datetime_test - @unittest.skip('See https://github.com/tarantool/tarantool/issues/7698') + @skip_or_run_datetime_2_11_test def test_tarantool_datetime_subtraction_different_timezones(self): case = self.datetime_subtraction_different_timezones_case self.assertSequenceEqual(self.con.call('sub', case['arg_1'], case['arg_2']), - [case['res']]) - + [case['res_tarantool']]) interval_arithmetic_cases = { 'year': { @@ -441,9 +485,9 @@ def test_tarantool_datetime_subtraction_different_timezones(self): 'arg_1': tarantool.Datetime(year=2008, month=2, day=3, hour=3, minute=36, sec=43), 'arg_2': tarantool.Interval(nsec=10000023), 'res_add': tarantool.Datetime(year=2008, month=2, day=3, hour=3, minute=36, sec=43, - nsec=10000023), + nsec=10000023), 'res_sub': tarantool.Datetime(year=2008, month=2, day=3, hour=3, minute=36, sec=42, - nsec=989999977), + nsec=989999977), }, 'zero': { 'arg_1': tarantool.Datetime(year=2008, month=2, day=3, hour=3, minute=36, sec=43), @@ -487,41 +531,38 @@ def test_tarantool_datetime_subtraction_different_timezones(self): 'res_add': tarantool.Datetime(year=2009, month=3, day=31), 'res_sub': tarantool.Datetime(year=2009, month=1, day=31), }, + 'week': { + 'arg_1': tarantool.Datetime(year=2008, month=2, day=3), + 'arg_2': tarantool.Interval(week=1), + 'res_add': tarantool.Datetime(year=2008, month=2, day=10), + 'res_sub': tarantool.Datetime(year=2008, month=1, day=27), + }, } - def test_python_datetime_addition(self): - for name in self.interval_arithmetic_cases.keys(): + def test_python_interval_addition(self): + for name, case in self.interval_arithmetic_cases.items(): with self.subTest(msg=name): - case = self.interval_arithmetic_cases[name] - self.assertEqual(case['arg_1'] + case['arg_2'], case['res_add']) - def test_python_datetime_subtraction(self): - for name in self.interval_arithmetic_cases.keys(): + def test_python_interval_subtraction(self): + for name, case in self.interval_arithmetic_cases.items(): with self.subTest(msg=name): - case = self.interval_arithmetic_cases[name] - self.assertEqual(case['arg_1'] - case['arg_2'], case['res_sub']) @skip_or_run_datetime_test - def test_tarantool_datetime_addition(self): - for name in self.interval_arithmetic_cases.keys(): + def test_tarantool_interval_addition(self): + for name, case in self.interval_arithmetic_cases.items(): with self.subTest(msg=name): - case = self.interval_arithmetic_cases[name] - self.assertSequenceEqual(self.con.call('add', case['arg_1'], case['arg_2']), [case['res_add']]) @skip_or_run_datetime_test - def test_tarantool_datetime_subtraction(self): - for name in self.interval_arithmetic_cases.keys(): + def test_tarantool_interval_subtraction(self): + for name, case in self.interval_arithmetic_cases.items(): with self.subTest(msg=name): - case = self.interval_arithmetic_cases[name] - self.assertSequenceEqual(self.con.call('sub', case['arg_1'], case['arg_2']), [case['res_sub']]) - datetime_addition_winter_time_switch_case = { 'arg_1': tarantool.Datetime(year=2008, month=1, day=1, hour=12, tz='Europe/Moscow'), 'arg_2': tarantool.Interval(month=6), @@ -533,15 +574,13 @@ def test_python_datetime_addition_winter_time_switch(self): self.assertEqual(case['arg_1'] + case['arg_2'], case['res']) - @skip_or_run_datetime_test - @unittest.skip('See https://github.com/tarantool/tarantool/issues/7700') + @skip_or_run_datetime_2_11_test def test_tarantool_datetime_addition_winter_time_switch(self): case = self.datetime_addition_winter_time_switch_case self.assertSequenceEqual(self.con.call('add', case['arg_1'], case['arg_2']), [case['res']]) - @skip_or_run_datetime_test def test_primary_key(self): data = [tarantool.Datetime(year=1970, month=1, day=1), 'content'] @@ -549,8 +588,100 @@ def test_primary_key(self): self.assertSequenceEqual(self.con.insert('test_pk', data), [data]) self.assertSequenceEqual(self.con.select('test_pk', data[0]), [data]) + datetime_nsec_overflow_cases = { + 'overflow_datetime': { + 'arg_1': tarantool.Datetime(year=2008, month=1, day=1, nsec=1230000000), + 'arg_2': tarantool.Datetime(year=2008, month=1, day=1, sec=1, nsec=230000000), + }, + 'underflow_datetime': { + 'arg_1': tarantool.Datetime(year=2008, month=1, day=1, nsec=-123456789), + 'arg_2': tarantool.Datetime(year=2007, month=12, day=31, hour=23, minute=59, + sec=59, nsec=876543211), + }, + 'overflow_timestamp': { + 'arg_1': tarantool.Datetime(timestamp=1199145600, nsec=1230000000), + 'arg_2': tarantool.Datetime(timestamp=1199145600 + 1, nsec=230000000), + }, + 'underflow_timestamp': { + 'arg_1': tarantool.Datetime(timestamp=1199145600, nsec=-123456789), + 'arg_2': tarantool.Datetime(timestamp=1199145600 - 1, nsec=876543211), + } + } + + def test_python_datetime_nsec_overflow(self): + for name, case in self.datetime_nsec_overflow_cases.items(): + with self.subTest(msg=name): + self.assertEqual(case['arg_1'], case['arg_2']) + + datetime_str_format = { + 'date': { + 'python': tarantool.Datetime(year=2022, month=8, day=31), + 'str': '2022-08-31T00:00:00', + }, + 'date_before_1970': { + 'python': tarantool.Datetime(year=1900, month=1, day=1), + 'str': '1900-01-01T00:00:00', + }, + 'datetime_with_minutes': { + 'python': tarantool.Datetime(year=2022, month=8, day=31, hour=18, minute=7), + 'str': '2022-08-31T18:07:00', + }, + 'datetime_with_seconds': { + 'python': tarantool.Datetime(year=2022, month=8, day=31, hour=18, minute=7, sec=54), + 'str': '2022-08-31T18:07:54', + }, + 'datetime_with_microseconds': { + 'python': tarantool.Datetime(year=2022, month=8, day=31, hour=18, minute=7, sec=54, + nsec=308543000), + 'str': '2022-08-31T18:07:54.308543', + }, + 'datetime_with_nanoseconds': { + 'python': tarantool.Datetime(year=2022, month=8, day=31, hour=18, minute=7, sec=54, + nsec=308543321), + 'str': '2022-08-31T18:07:54.308543321', + }, + 'datetime_with_positive_offset': { + 'python': tarantool.Datetime(year=2022, month=8, day=31, hour=18, minute=7, sec=54, + nsec=308543321, tzoffset=180), + 'str': '2022-08-31T18:07:54.308543321+03:00', + }, + 'datetime_with_negative_offset': { + 'python': tarantool.Datetime(year=2022, month=8, day=31, hour=18, minute=7, sec=54, + nsec=308543321, tzoffset=-60), + 'str': '2022-08-31T18:07:54.308543321-01:00', + }, + 'date_with_utc_tz': { + 'python': tarantool.Datetime(year=1970, month=1, day=1, tz='UTC'), + 'str': '1970-01-01T00:00:00+00:00', + }, + 'date_with_tz': { + 'python': tarantool.Datetime(year=2022, month=8, day=31, tz='Europe/Moscow'), + 'str': '2022-08-31T00:00:00+03:00', + }, + 'datetime_with_tz': { + 'python': tarantool.Datetime(year=2022, month=8, day=31, hour=18, minute=7, sec=54, + nsec=308543321, tz='Europe/Moscow'), + 'str': '2022-08-31T18:07:54.308543321+03:00', + }, + 'datetime_with_tz_and_offset': { + 'python': tarantool.Datetime(year=2022, month=8, day=31, hour=18, minute=7, sec=54, + nsec=308543321, tz='Europe/Moscow', tzoffset=123), + 'str': '2022-08-31T18:07:54.308543321+03:00', + }, + 'datetime_with_abbrev_tz': { + 'python': tarantool.Datetime(year=2022, month=8, day=31, hour=18, minute=7, sec=54, + nsec=308543321, tz='MSK'), + 'str': '2022-08-31T18:07:54.308543321+03:00', + }, + } + + def test_python_datetime_string(self): + for name, case in self.datetime_str_format.items(): + with self.subTest(msg=name): + self.assertEqual(str(case['python']), case['str']) + @classmethod - def tearDownClass(self): - self.con.close() - self.srv.stop() - self.srv.clean() + def tearDownClass(cls): + cls.con.close() + cls.srv.stop() + cls.srv.clean() diff --git a/test/suites/test_dbapi.py b/test/suites/test_dbapi.py index 0df06337..bd8c7487 100644 --- a/test/suites/test_dbapi.py +++ b/test/suites/test_dbapi.py @@ -1,3 +1,8 @@ +""" +This module tests compatibility with DBAPI standards. +""" +# pylint: disable=missing-class-docstring,missing-function-docstring,protected-access,fixme + import sys import unittest @@ -7,29 +12,41 @@ from tarantool import dbapi from .lib.tarantool_server import TarantoolServer from .lib.skip import skip_or_run_sql_test +from .utils import assert_admin_success -class TestSuite_DBAPI(dbapi20.DatabaseAPI20Test): +class TestSuiteDBAPI(dbapi20.DatabaseAPI20Test): table_prefix = 'dbapi20test_' # If you need to specify a prefix for tables - ddl0 = 'create table %s (id INTEGER PRIMARY KEY AUTOINCREMENT, ' \ + ddl0 = f'create table {table_prefix} (id INTEGER PRIMARY KEY AUTOINCREMENT, ' \ 'name varchar(20))' - ddl1 = 'create table %sbooze (name varchar(20) primary key)' % table_prefix - ddl2 = 'create table %sbarflys (name varchar(20) primary key, ' \ - 'drink varchar(30))' % table_prefix + ddl1 = f'create table {table_prefix}booze (name varchar(20) primary key)' + ddl2 = f'create table {table_prefix}barflys (name varchar(20) primary key, ' \ + 'drink varchar(30))' @classmethod - def setUpClass(self): + def setUpClass(cls): print(' DBAPI '.center(70, '='), file=sys.stderr) print('-' * 70, file=sys.stderr) - self.srv = TarantoolServer() - self.srv.script = 'test/suites/box.lua' - self.srv.start() - self.con = tarantool.Connection(self.srv.host, self.srv.args['primary']) - self.driver = dbapi - self.connect_kw_args = dict( - host=self.srv.host, - port=self.srv.args['primary']) + # Select scans are not allowed with compat.sql_seq_scan_default = "new", + # but tests create cursors with fullscan. + cls.srv = TarantoolServer(sql_seq_scan_default="old") + cls.srv.script = 'test/suites/box.lua' + cls.srv.start() + cls.con = tarantool.Connection(cls.srv.host, cls.srv.args['primary']) + cls.driver = dbapi + cls.connect_kw_args = { + "host": cls.srv.host, + "port": cls.srv.args['primary'] + } + + # grant full access to guest + resp = cls.srv.admin(""" + box.schema.user.grant('guest', 'create,read,write,execute', 'universe', + nil, {if_not_exists = true}) + return true + """) + assert_admin_success(resp) @skip_or_run_sql_test def setUp(self): @@ -38,42 +55,48 @@ def setUp(self): self.srv.touch_lock() self.con.flush_schema() - # grant full access to guest - self.srv.admin("box.schema.user.grant('guest', 'create,read,write," - "execute', 'universe')") - @classmethod - def tearDownClass(self): - self.con.close() - self.srv.stop() - self.srv.clean() + def tearDownClass(cls): + cls.con.close() + cls.srv.stop() + cls.srv.clean() + + def help_nextset_setUp(self, cur): + # pylint: disable=unused-argument + pass + + def help_nextset_tearDown(self, cur): + # pylint: disable=unused-argument + pass def test_rowcount(self): con = self._connect() try: cur = con.cursor() self.executeDDL1(cur) - dbapi20._failUnless(self,cur.rowcount in (-1, 1), + dbapi20._failUnless( + self, cur.rowcount in (-1, 1), 'cursor.rowcount should be -1 or 1 after executing no-result ' 'statements' + str(cur.rowcount) - ) - cur.execute("%s into %sbooze values ('Victoria Bitter')" % ( - self.insert, self.table_prefix - )) - dbapi20._failUnless(self,cur.rowcount == 1, + ) + cur.execute(f"{self.insert} into {self.table_prefix}booze values ('Victoria Bitter')") + dbapi20._failUnless( + self, cur.rowcount == 1, 'cursor.rowcount should == number or rows inserted, or ' 'set to -1 after executing an insert statement' - ) - cur.execute("select name from %sbooze" % self.table_prefix) - dbapi20._failUnless(self,cur.rowcount == -1, + ) + cur.execute(f"select name from {self.table_prefix}booze") + dbapi20._failUnless( + self, cur.rowcount == -1, 'cursor.rowcount should == number of rows returned, or ' 'set to -1 after executing a select statement' - ) + ) self.executeDDL2(cur) - dbapi20._failUnless(self,cur.rowcount in (-1, 1), + dbapi20._failUnless( + self, cur.rowcount in (-1, 1), 'cursor.rowcount should be -1 or 1 after executing no-result ' 'statements' - ) + ) finally: con.close() @@ -127,43 +150,3 @@ def test_setoutputsize(self): # Do nothing @unittest.skip('Not implemented') def test_description(self): pass - - def test_ExceptionsAsConnectionAttributes(self): - # Workaround for https://github.com/baztian/dbapi-compliance/issues/5 - - # OPTIONAL EXTENSION - # Test for the optional DB API 2.0 extension, where the exceptions - # are exposed as attributes on the Connection object - # I figure this optional extension will be implemented by any - # driver author who is using this test suite, so it is enabled - # by default. - drv = self.driver - con = self._connect() - try: - dbapi20._failUnless(self,con.Warning is drv.Warning) - dbapi20._failUnless(self,con.Error is drv.Error) - dbapi20._failUnless(self,con.InterfaceError is drv.InterfaceError) - dbapi20._failUnless(self,con.DatabaseError is drv.DatabaseError) - dbapi20._failUnless(self,con.OperationalError is drv.OperationalError) - dbapi20._failUnless(self,con.IntegrityError is drv.IntegrityError) - dbapi20._failUnless(self,con.InternalError is drv.InternalError) - dbapi20._failUnless(self,con.ProgrammingError is drv.ProgrammingError) - dbapi20. _failUnless(self,con.NotSupportedError is drv.NotSupportedError) - finally: - con.close() - - - def test_rollback(self): - # Workaround for https://github.com/baztian/dbapi-compliance/issues/5 - - con = self._connect() - try: - # If rollback is defined, it should either work or throw - # the documented exception - if hasattr(con,'rollback'): - try: - con.rollback() - except self.driver.NotSupportedError: - pass - finally: - con.close() diff --git a/test/suites/test_decimal.py b/test/suites/test_decimal.py index 8029d7b9..2875a7da 100644 --- a/test/suites/test_decimal.py +++ b/test/suites/test_decimal.py @@ -1,58 +1,74 @@ +""" +This module tests work with decimal type. +""" +# pylint: disable=missing-class-docstring,missing-function-docstring,duplicate-code + import sys import unittest import decimal + import msgpack -import warnings -import tarantool +import tarantool +from tarantool.error import MsgpackError, MsgpackWarning from tarantool.msgpack_ext.packer import default as packer_default from tarantool.msgpack_ext.unpacker import ext_hook as unpacker_ext_hook from .lib.tarantool_server import TarantoolServer from .lib.skip import skip_or_run_decimal_test -from tarantool.error import MsgpackError, MsgpackWarning +from .utils import assert_admin_success + -class TestSuite_Decimal(unittest.TestCase): +class TestSuiteDecimal(unittest.TestCase): @classmethod - def setUpClass(self): + def setUpClass(cls): print(' DECIMAL EXT TYPE '.center(70, '='), file=sys.stderr) print('-' * 70, file=sys.stderr) - self.srv = TarantoolServer() - self.srv.script = 'test/suites/box.lua' - self.srv.start() + cls.srv = TarantoolServer() + cls.srv.script = 'test/suites/box.lua' + cls.srv.start() - self.adm = self.srv.admin - self.adm(r""" - _, decimal = pcall(require, 'decimal') + cls.adm = cls.srv.admin + resp = cls.adm(""" + decimal_supported, decimal = pcall(require, 'decimal') - box.schema.space.create('test') + box.schema.space.create('test', {if_not_exists = true}) box.space['test']:create_index('primary', { type = 'tree', parts = {1, 'string'}, - unique = true}) + unique = true, + if_not_exists = true}) - pcall(function() + if decimal_supported then box.schema.space.create('test_pk') box.space['test_pk']:create_index('primary', { type = 'tree', parts = {1, 'decimal'}, - unique = true}) - end) + unique = true, + if_not_exists = true}) + end box.schema.user.create('test', {password = 'test', if_not_exists = true}) - box.schema.user.grant('test', 'read,write,execute', 'universe') + box.schema.user.grant('test', 'read,write,execute', 'universe', + nil, {if_not_exists = true}) + + return true """) + assert_admin_success(resp) - self.con = tarantool.Connection(self.srv.host, self.srv.args['primary'], - user='test', password='test') + cls.con = tarantool.Connection(cls.srv.host, cls.srv.args['primary'], + user='test', password='test') def setUp(self): # prevent a remote tarantool from clean our session if self.srv.is_started(): self.srv.touch_lock() - self.adm("box.space['test']:truncate()") - + resp = self.adm(""" + box.space['test']:truncate() + return true + """) + assert_admin_success(resp) valid_cases = { 'simple_decimal_1': { @@ -157,13 +173,13 @@ def setUp(self): }, 'decimal_limits_1': { 'python': decimal.Decimal('11111111111111111111111111111111111111'), - 'msgpack': (b'\x00\x01\x11\x11\x11\x11\x11\x11\x11\x11\x11' + + 'msgpack': (b'\x00\x01\x11\x11\x11\x11\x11\x11\x11\x11\x11' b'\x11\x11\x11\x11\x11\x11\x11\x11\x11\x1c'), 'tarantool': "decimal.new('11111111111111111111111111111111111111')", }, 'decimal_limits_2': { 'python': decimal.Decimal('-11111111111111111111111111111111111111'), - 'msgpack': (b'\x00\x01\x11\x11\x11\x11\x11\x11\x11\x11\x11' + + 'msgpack': (b'\x00\x01\x11\x11\x11\x11\x11\x11\x11\x11\x11' b'\x11\x11\x11\x11\x11\x11\x11\x11\x11\x1d'), 'tarantool': "decimal.new('-11111111111111111111111111111111111111')", }, @@ -199,44 +215,73 @@ def setUp(self): }, 'decimal_limits_9': { 'python': decimal.Decimal('99999999999999999999999999999999999999'), - 'msgpack': (b'\x00\x09\x99\x99\x99\x99\x99\x99\x99\x99\x99' + + 'msgpack': (b'\x00\x09\x99\x99\x99\x99\x99\x99\x99\x99\x99' b'\x99\x99\x99\x99\x99\x99\x99\x99\x99\x9c'), 'tarantool': "decimal.new('99999999999999999999999999999999999999')", }, 'decimal_limits_10': { 'python': decimal.Decimal('-99999999999999999999999999999999999999'), - 'msgpack': (b'\x00\x09\x99\x99\x99\x99\x99\x99\x99\x99\x99' + + 'msgpack': (b'\x00\x09\x99\x99\x99\x99\x99\x99\x99\x99\x99' b'\x99\x99\x99\x99\x99\x99\x99\x99\x99\x9d'), 'tarantool': "decimal.new('-99999999999999999999999999999999999999')", }, 'decimal_limits_11': { 'python': decimal.Decimal('1234567891234567890.0987654321987654321'), - 'msgpack': (b'\x13\x01\x23\x45\x67\x89\x12\x34\x56\x78\x90' + + 'msgpack': (b'\x13\x01\x23\x45\x67\x89\x12\x34\x56\x78\x90' b'\x09\x87\x65\x43\x21\x98\x76\x54\x32\x1c'), 'tarantool': "decimal.new('1234567891234567890.0987654321987654321')", }, 'decimal_limits_12': { 'python': decimal.Decimal('-1234567891234567890.0987654321987654321'), - 'msgpack': (b'\x13\x01\x23\x45\x67\x89\x12\x34\x56\x78\x90' + + 'msgpack': (b'\x13\x01\x23\x45\x67\x89\x12\x34\x56\x78\x90' b'\x09\x87\x65\x43\x21\x98\x76\x54\x32\x1d'), 'tarantool': "decimal.new('-1234567891234567890.0987654321987654321')", }, + 'decimal_exponent_1': { + 'python': decimal.Decimal('1e33'), + 'msgpack': (b'\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x0c'), + 'tarantool': "decimal.new('1e33')", + }, + 'decimal_exponent_2': { + 'python': decimal.Decimal('1.2345e33'), + 'msgpack': (b'\x00\x01\x23\x45\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x0c'), + 'tarantool': "decimal.new('1.2345e33')", + }, + 'decimal_exponent_3': { + 'python': decimal.Decimal('1.2345e2'), + 'msgpack': (b'\x02\x12\x34\x5c'), + 'tarantool': "decimal.new('1.2345e2')", + }, + 'decimal_exponent_4': { + 'python': decimal.Decimal('1.2345e4'), + 'msgpack': (b'\x00\x12\x34\x5c'), + 'tarantool': "decimal.new('1.2345e4')", + }, + 'decimal_exponent_5': { + 'python': decimal.Decimal('-1e33'), + 'msgpack': (b'\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x0d'), + 'tarantool': "decimal.new('-1e33')", + }, + 'decimal_exponent_6': { + 'python': decimal.Decimal('1e-33'), + 'msgpack': (b'\x21\x1c'), + 'tarantool': "decimal.new('1e-33')", + }, } def test_msgpack_decode(self): - for name in self.valid_cases.keys(): + for name, case in self.valid_cases.items(): with self.subTest(msg=name): - case = self.valid_cases[name] - self.assertEqual(unpacker_ext_hook(1, case['msgpack']), case['python']) @skip_or_run_decimal_test def test_tarantool_decode(self): - for name in self.valid_cases.keys(): + for name, case in self.valid_cases.items(): with self.subTest(msg=name): - case = self.valid_cases[name] - self.adm(f"box.space['test']:replace{{'{name}', {case['tarantool']}}}") self.assertSequenceEqual( @@ -244,19 +289,15 @@ def test_tarantool_decode(self): [[name, case['python']]]) def test_msgpack_encode(self): - for name in self.valid_cases.keys(): + for name, case in self.valid_cases.items(): with self.subTest(msg=name): - case = self.valid_cases[name] - self.assertEqual(packer_default(case['python']), msgpack.ExtType(code=1, data=case['msgpack'])) @skip_or_run_decimal_test def test_tarantool_encode(self): - for name in self.valid_cases.keys(): + for name, case in self.valid_cases.items(): with self.subTest(msg=name): - case = self.valid_cases[name] - self.con.insert('test', [name, case['python']]) lua_eval = f""" @@ -274,7 +315,6 @@ def test_tarantool_encode(self): self.assertSequenceEqual(self.con.eval(lua_eval), [True]) - error_cases = { 'decimal_limit_break_head_1': { 'python': decimal.Decimal('999999999999999999999999999999999999999'), @@ -303,10 +343,10 @@ def test_tarantool_encode(self): } def test_msgpack_encode_error(self): - for name in self.error_cases.keys(): - with self.subTest(msg=name): - case = self.error_cases[name] + # pylint: disable=cell-var-from-loop + for name, case in self.error_cases.items(): + with self.subTest(msg=name): msg = 'Decimal cannot be encoded: Tarantool decimal ' + \ 'supports a maximum of 38 digits.' self.assertRaisesRegex( @@ -315,17 +355,16 @@ def test_msgpack_encode_error(self): @skip_or_run_decimal_test def test_tarantool_encode_error(self): - for name in self.error_cases.keys(): - with self.subTest(msg=name): - case = self.error_cases[name] + # pylint: disable=cell-var-from-loop + for name, case in self.error_cases.items(): + with self.subTest(msg=name): msg = 'Decimal cannot be encoded: Tarantool decimal ' + \ 'supports a maximum of 38 digits.' self.assertRaisesRegex( MsgpackError, msg, lambda: self.con.insert('test', [name, case['python']])) - precision_loss_cases = { 'decimal_limit_break_tail_1': { 'python': decimal.Decimal('1.00000000000000000000000000000000000001'), @@ -359,53 +398,54 @@ def test_tarantool_encode_error(self): }, 'decimal_limit_break_tail_7': { 'python': decimal.Decimal('99999999999999999999999999999999999999.1'), - 'msgpack': (b'\x00\x09\x99\x99\x99\x99\x99\x99\x99\x99\x99' + + 'msgpack': (b'\x00\x09\x99\x99\x99\x99\x99\x99\x99\x99\x99' b'\x99\x99\x99\x99\x99\x99\x99\x99\x99\x9c'), 'tarantool': "decimal.new('99999999999999999999999999999999999999')", }, 'decimal_limit_break_tail_8': { 'python': decimal.Decimal('-99999999999999999999999999999999999999.1'), - 'msgpack': (b'\x00\x09\x99\x99\x99\x99\x99\x99\x99\x99\x99' + + 'msgpack': (b'\x00\x09\x99\x99\x99\x99\x99\x99\x99\x99\x99' b'\x99\x99\x99\x99\x99\x99\x99\x99\x99\x9d'), 'tarantool': "decimal.new('-99999999999999999999999999999999999999')", }, 'decimal_limit_break_tail_9': { - 'python': decimal.Decimal('99999999999999999999999999999999999999.1111111111111111111111111'), - 'msgpack': (b'\x00\x09\x99\x99\x99\x99\x99\x99\x99\x99\x99' + + 'python': decimal.Decimal('99999999999999999999999999999999999999.11111111111111' + '11111111111'), + 'msgpack': (b'\x00\x09\x99\x99\x99\x99\x99\x99\x99\x99\x99' b'\x99\x99\x99\x99\x99\x99\x99\x99\x99\x9c'), 'tarantool': "decimal.new('99999999999999999999999999999999999999')", }, 'decimal_limit_break_tail_10': { - 'python': decimal.Decimal('-99999999999999999999999999999999999999.1111111111111111111111111'), - 'msgpack': (b'\x00\x09\x99\x99\x99\x99\x99\x99\x99\x99\x99' + + 'python': decimal.Decimal('-99999999999999999999999999999999999999.11111111111111' + '11111111111'), + 'msgpack': (b'\x00\x09\x99\x99\x99\x99\x99\x99\x99\x99\x99' b'\x99\x99\x99\x99\x99\x99\x99\x99\x99\x9d'), 'tarantool': "decimal.new('-99999999999999999999999999999999999999')", }, } def test_msgpack_encode_with_precision_loss(self): - for name in self.precision_loss_cases.keys(): - with self.subTest(msg=name): - case = self.precision_loss_cases[name] + # pylint: disable=cell-var-from-loop + for name, case in self.precision_loss_cases.items(): + with self.subTest(msg=name): msg = 'Decimal encoded with loss of precision: ' + \ 'Tarantool decimal supports a maximum of 38 digits.' self.assertWarnsRegex( MsgpackWarning, msg, lambda: self.assertEqual( - packer_default(case['python']), - msgpack.ExtType(code=1, data=case['msgpack']) - ) + packer_default(case['python']), + msgpack.ExtType(code=1, data=case['msgpack']) ) - + ) @skip_or_run_decimal_test def test_tarantool_encode_with_precision_loss(self): - for name in self.precision_loss_cases.keys(): - with self.subTest(msg=name): - case = self.precision_loss_cases[name] + # pylint: disable=cell-var-from-loop + for name, case in self.precision_loss_cases.items(): + with self.subTest(msg=name): msg = 'Decimal encoded with loss of precision: ' + \ 'Tarantool decimal supports a maximum of 38 digits.' @@ -428,7 +468,6 @@ def test_tarantool_encode_with_precision_loss(self): self.assertSequenceEqual(self.con.eval(lua_eval), [True]) - @skip_or_run_decimal_test def test_primary_key(self): data = [decimal.Decimal('0'), 'content'] @@ -436,9 +475,8 @@ def test_primary_key(self): self.assertSequenceEqual(self.con.insert('test_pk', data), [data]) self.assertSequenceEqual(self.con.select('test_pk', data[0]), [data]) - @classmethod - def tearDownClass(self): - self.con.close() - self.srv.stop() - self.srv.clean() + def tearDownClass(cls): + cls.con.close() + cls.srv.stop() + cls.srv.clean() diff --git a/test/suites/test_dml.py b/test/suites/test_dml.py index 2dbff1b5..26539eec 100644 --- a/test/suites/test_dml.py +++ b/test/suites/test_dml.py @@ -1,3 +1,8 @@ +""" +This module tests basic data operations. +""" +# pylint: disable=missing-class-docstring,missing-function-docstring,protected-access,fixme,too-many-public-methods,duplicate-code + import sys import unittest import tarantool @@ -5,47 +10,55 @@ from .lib.tarantool_server import TarantoolServer from .lib.skip import skip_or_run_error_extra_info_test +from .utils import assert_admin_success + -class TestSuite_Request(unittest.TestCase): +class TestSuiteRequest(unittest.TestCase): @classmethod - def setUpClass(self): + def setUpClass(cls): print(' DML '.center(70, '='), file=sys.stderr) print('-' * 70, file=sys.stderr) - self.srv = TarantoolServer() - self.srv.script = 'test/suites/box.lua' - self.srv.start() - self.con = tarantool.Connection(self.srv.host, self.srv.args['primary']) - self.adm = self.srv.admin - self.space_created = self.adm("box.schema.create_space('space_1')") - self.adm(""" - box.space['space_1']:create_index('primary', { - type = 'tree', - parts = {1, 'num'}, - unique = true}) - """.replace('\n', ' ')) - self.adm(""" - box.space['space_1']:create_index('secondary', { - type = 'tree', - parts = {2, 'num', 3, 'str'}, - unique = false}) - """.replace('\n', ' ')) - self.space_created = self.adm("box.schema.create_space('space_2')") - self.adm(""" - box.space['space_2']:create_index('primary', { - type = 'hash', - parts = {1, 'num'}, - unique = true}) - """.replace('\n', ' ')) - self.adm("json = require('json')") - self.adm("fiber = require('fiber')") - self.adm("uuid = require('uuid')") + cls.srv = TarantoolServer() + cls.srv.script = 'test/suites/box.lua' + cls.srv.start() + cls.con = tarantool.Connection(cls.srv.host, cls.srv.args['primary']) + cls.adm = cls.srv.admin + cls.space_created = cls.adm("box.schema.create_space('space_1', {if_not_exists = true})") + resp = cls.adm(""" + box.space['space_1']:create_index('primary', { + type = 'tree', + parts = {1, 'num'}, + unique = true, + if_not_exists = true}) + + box.space['space_1']:create_index('secondary', { + type = 'tree', + parts = {2, 'num', 3, 'str'}, + unique = false, + if_not_exists = true}) + + box.schema.create_space('space_2', {if_not_exists = true}) + + box.space['space_2']:create_index('primary', { + type = 'hash', + parts = {1, 'num'}, + unique = true, + if_not_exists = true}) + + json = require('json') + fiber = require('fiber') + uuid = require('uuid') + + return true + """) + assert_admin_success(resp) if not sys.platform.startswith("win"): - self.sock_srv = TarantoolServer(create_unix_socket=True) - self.sock_srv.script = 'test/suites/box.lua' - self.sock_srv.start() + cls.sock_srv = TarantoolServer(create_unix_socket=True) + cls.sock_srv.script = 'test/suites/box.lua' + cls.sock_srv.start() else: - self.sock_srv = None + cls.sock_srv = None def setUp(self): # prevent a remote tarantool from clean our session @@ -54,10 +67,11 @@ def setUp(self): def test_00_00_authenticate(self): self.assertIsNone(self.srv.admin(""" - box.schema.user.create('test', { password = 'test' }) + box.schema.user.create('test', { password = 'test', if_not_exists = true }) """)) self.assertIsNone(self.srv.admin(""" - box.schema.user.grant('test', 'execute,read,write', 'universe') + box.schema.user.grant('test', 'execute,read,write', 'universe', + nil, {if_not_exists = true}) """)) self.assertEqual(self.con.authenticate('test', 'test')._data, None) @@ -72,9 +86,10 @@ def test_00_02_fill_space(self): # prevent a remote tarantool from clean our session self.srv.touch_lock() self.assertEqual( - self.con.insert('space_1', [i, i%5, 'tuple_'+str(i)])[0], - [i, i%5, 'tuple_'+str(i)] + self.con.insert('space_1', [i, i % 5, 'tuple_' + str(i)])[0], + [i, i % 5, 'tuple_' + str(i)] ) + def test_00_03_answer_repr(self): repr_str = """- [1, 1, 'tuple_1']""" self.assertEqual(repr(self.con.select('space_1', 1)), repr_str) @@ -83,47 +98,52 @@ def test_02_select(self): # Check that select with different keys are Ok. (With and without index names) self.assertSequenceEqual(self.con.select('space_1', 20), [[20, 0, 'tuple_20']]) self.assertSequenceEqual(self.con.select('space_1', [21]), [[21, 1, 'tuple_21']]) - self.assertSequenceEqual(self.con.select('space_1', [22], index='primary'), [[22, 2, 'tuple_22']]) - self.assertSequenceEqual(self.con.select('space_1', [23], index='primary'), [[23, 3, 'tuple_23']]) + self.assertSequenceEqual(self.con.select('space_1', [22], index='primary'), + [[22, 2, 'tuple_22']]) + self.assertSequenceEqual(self.con.select('space_1', [23], index='primary'), + [[23, 3, 'tuple_23']]) # Check that Offset and Limit args are working fine. - self.assertSequenceEqual(self.con.select('space_1', [20], index='primary', limit=1), [[20, 0, 'tuple_20']]) + self.assertSequenceEqual(self.con.select('space_1', [20], index='primary', limit=1), + [[20, 0, 'tuple_20']]) # With other indexes too self.assertSequenceEqual( - sorted( - self.con.select('space_1', [0], index='secondary', offset=3, limit=0), - key = lambda x: x[0]), - [] - ) + sorted( + self.con.select('space_1', [0], index='secondary', offset=3, limit=0), + key=lambda x: x[0]), + [] + ) self.assertSequenceEqual( - sorted( - self.con.select('space_1', [0], index='secondary', offset=3, limit=1), - key = lambda x: x[0]), - [[110, 0, 'tuple_110']] - ) + sorted( + self.con.select('space_1', [0], index='secondary', offset=3, limit=1), + key=lambda x: x[0]), + [[110, 0, 'tuple_110']] + ) self.assertSequenceEqual( - sorted( - self.con.select('space_1', [0], index='secondary', offset=3, limit=2), - key = lambda x: x[0]), - [[110, 0, 'tuple_110'],\ - [115, 0, 'tuple_115']] - ) + sorted( + self.con.select('space_1', [0], index='secondary', offset=3, limit=2), + key=lambda x: x[0]), + [[110, 0, 'tuple_110'], [115, 0, 'tuple_115']] + ) select_req = self.con.select('space_1', [0], index='secondary') self.assertEqual(len(select_req), 99) for i in select_req: - self.assertTrue(not (i[0] % 5)) + self.assertTrue(not i[0] % 5) self.assertTrue(not i[1]) self.assertTrue(i[2] == 'tuple_' + str(i[0])) # Check limit again. - self.assertEqual(len(self.con.select('space_1', [0, 'tuple_20'], index='secondary', limit=0)), 0) + self.assertEqual( + len(self.con.select('space_1', [0, 'tuple_20'], index='secondary', limit=0)), + 0) self.assertEqual(len(self.con.select('space_1', [0], index='secondary', limit=0)), 0) self.assertEqual(len(self.con.select('space_1', [0], index='secondary', limit=100)), 99) self.assertEqual(len(self.con.select('space_1', [0], index='secondary', limit=50)), 50) # TODO: Check iterator_types self.assertSequenceEqual( - self.con.select('space_1', [0, 'tuple_20'], index='secondary', limit=2, iterator=tarantool.const.ITERATOR_GT), + self.con.select('space_1', [0, 'tuple_20'], index='secondary', limit=2, + iterator=tarantool.const.ITERATOR_GT), [[200, 0, 'tuple_200'], [205, 0, 'tuple_205']] ) @@ -133,18 +153,18 @@ def test_03_delete(self): self.assertSequenceEqual(self.con.delete('space_1', [20]), []) self.assertSequenceEqual(self.con.select('space_1', [20], index='primary'), []) # Check that field has no meaning, yet. - with self.assertRaisesRegex(tarantool.DatabaseError, - '(19, .*)'): - self.con.delete('space_1', [1, 'tuple_21']) - self.assertSequenceEqual(self.con.select('space_1', [21], index='primary'), [[21, 1, 'tuple_21']]) + with self.assertRaisesRegex(tarantool.DatabaseError, '(19, .*)'): + self.con.delete('space_1', [1, 'tuple_21']) + self.assertSequenceEqual(self.con.select('space_1', [21], index='primary'), + [[21, 1, 'tuple_21']]) def test_04_replace(self): # Check replace that is Ok. - self.assertSequenceEqual(self.con.replace('space_1', [2, 2, 'tuple_3']), [[2, 2, 'tuple_3']]) + self.assertSequenceEqual(self.con.replace('space_1', [2, 2, 'tuple_3']), + [[2, 2, 'tuple_3']]) self.assertSequenceEqual(self.con.select('space_1', 2), [[2, 2, 'tuple_3']]) # Check replace that isn't Ok. - with self.assertRaisesRegex(tarantool.DatabaseError, - '(39, .*)'): + with self.assertRaisesRegex(tarantool.DatabaseError, '(39, .*)'): self.assertSequenceEqual(self.con.replace('space_1', [2, 2]), [[2, 2, 'tuple_2']]) def test_05_ping(self): @@ -159,25 +179,28 @@ def test_05_ping(self): def test_06_update(self): self.assertSequenceEqual(self.con.update('space_1', (2,), [('+', 1, 3)]), - [[2, 5, 'tuple_3']]) + [[2, 5, 'tuple_3']]) self.assertSequenceEqual(self.con.update('space_1', (2,), [('-', 1, 3)]), - [[2, 2, 'tuple_3']]) + [[2, 2, 'tuple_3']]) self.assertSequenceEqual(self.con.update('space_1', (2,), [(':', 2, 3, 2, 'lalal')]), - [[2, 2, 'tuplalal_3']]) + [[2, 2, 'tuplalal_3']]) self.assertSequenceEqual(self.con.update('space_1', (2,), [('!', 2, '1')]), - [[2, 2, '1', 'tuplalal_3']]) + [[2, 2, '1', 'tuplalal_3']]) self.assertSequenceEqual(self.con.update('space_1', (2,), [('!', 2, 'oingo, boingo')]), - [[2, 2, 'oingo, boingo', '1', 'tuplalal_3']]) + [[2, 2, 'oingo, boingo', '1', 'tuplalal_3']]) self.assertSequenceEqual(self.con.update('space_1', (2,), [('#', 2, 2)]), - [[2, 2, 'tuplalal_3']]) + [[2, 2, 'tuplalal_3']]) def test_07_call_16(self): - con = tarantool.Connection(self.srv.host, self.srv.args['primary'], call_16 = True) + con = tarantool.Connection(self.srv.host, self.srv.args['primary'], call_16=True) try: con.authenticate('test', 'test') - self.assertSequenceEqual(con.call('json.decode', '[123, 234, 345]'), [[123, 234, 345]]) - self.assertSequenceEqual(con.call('json.decode', ['[123, 234, 345]']), [[123, 234, 345]]) - self.assertSequenceEqual(con.call('json.decode', ('[123, 234, 345]',)), [[123, 234, 345]]) + self.assertSequenceEqual(con.call('json.decode', '[123, 234, 345]'), + [[123, 234, 345]]) + self.assertSequenceEqual(con.call('json.decode', ['[123, 234, 345]']), + [[123, 234, 345]]) + self.assertSequenceEqual(con.call('json.decode', ('[123, 234, 345]',)), + [[123, 234, 345]]) with self.assertRaisesRegex(tarantool.DatabaseError, '(32, .*)'): con.call('json.decode') with self.assertRaisesRegex(tarantool.DatabaseError, '(32, .*)'): @@ -195,7 +218,8 @@ def test_07_call_16(self): self.assertEqual(len(ans[0]), 1) self.assertIsInstance(ans[0][0], str) - self.assertSequenceEqual(con.call('box.tuple.new', [1, 2, 3, 'fld_1']), [[1, 2, 3, 'fld_1']]) + self.assertSequenceEqual(con.call('box.tuple.new', [[1, 2, 3, 'fld_1']]), + [[1, 2, 3, 'fld_1']]) self.assertSequenceEqual(con.call('box.tuple.new', 'fld_1'), [['fld_1']]) finally: con.close() @@ -220,36 +244,42 @@ def test_07_call_17(self): self.assertEqual(len(ans), 1) self.assertIsInstance(ans[0], str) - self.assertSequenceEqual(con.call('box.tuple.new', [1, 2, 3, 'fld_1']), [[1, 2, 3, 'fld_1']]) + self.assertSequenceEqual(con.call('box.tuple.new', [[1, 2, 3, 'fld_1']]), + [[1, 2, 3, 'fld_1']]) self.assertSequenceEqual(con.call('box.tuple.new', 'fld_1'), [['fld_1']]) con.close() def test_08_eval(self): self.assertSequenceEqual(self.con.eval('return json.decode(...)', - '[123, 234, 345]'), [[123, 234, 345]]) + '[123, 234, 345]'), [[123, 234, 345]]) self.assertSequenceEqual(self.con.eval('return json.decode(...)', - ['[123, 234, 345]']), [[123, 234, 345]]) + ['[123, 234, 345]']), [[123, 234, 345]]) self.assertSequenceEqual(self.con.eval('return json.decode(...)', - ('[123, 234, 345]',)), [[123, 234, 345]]) + ('[123, 234, 345]',)), [[123, 234, 345]]) self.assertSequenceEqual(self.con.eval('return json.decode("[123, 234, 345]")'), - [[123, 234, 345]]) - self.assertSequenceEqual(self.con.eval('return json.decode("[123, 234, 345]"), '+ - 'json.decode("[123, 234, 345]")'), - [[123, 234, 345], [123, 234, 345]]) + [[123, 234, 345]]) + self.assertSequenceEqual( + self.con.eval('return json.decode("[123, 234, 345]"), json.decode("[123, 234, 345]")'), + [[123, 234, 345], [123, 234, 345]]) self.assertSequenceEqual(self.con.eval('json.decode("[123, 234, 345]")'), []) def test_09_upsert(self): - self.assertSequenceEqual(self.con.select('space_1', [22], index='primary'), [[22, 2, 'tuple_22']]) - self.assertSequenceEqual(self.con.select('space_1', [23], index='primary'), [[23, 3, 'tuple_23']]) - self.assertSequenceEqual(self.con.select('space_1', [499], index='primary'), [[499, 4, 'tuple_499']]) + self.assertSequenceEqual(self.con.select('space_1', [22], index='primary'), + [[22, 2, 'tuple_22']]) + self.assertSequenceEqual(self.con.select('space_1', [23], index='primary'), + [[23, 3, 'tuple_23']]) + self.assertSequenceEqual(self.con.select('space_1', [499], index='primary'), + [[499, 4, 'tuple_499']]) self.assertSequenceEqual(self.con.select('space_1', [500], index='primary'), []) + self.assertSequenceEqual( + self.con.upsert('space_1', [500, 123, 'hello, world'], [(':', 2, 2, 3, "---")]), []) + self.assertSequenceEqual(self.con.select('space_1', [500], index='primary'), + [[500, 123, 'hello, world']]) self.assertSequenceEqual(self.con.upsert('space_1', [500, 123, 'hello, world'], - [(':', 2, 2, 3, "---")]), []) - self.assertSequenceEqual(self.con.select('space_1', [500], index='primary'), [[500, 123, 'hello, world']]) - self.assertSequenceEqual(self.con.upsert('space_1', [500, 123, 'hello, world'], - [(':', 2, 2, 3, "---")]), []) - self.assertSequenceEqual(self.con.select('space_1', [500], index='primary'), [[500, 123, 'he---, world']]) + [(':', 2, 2, 3, "---")]), []) + self.assertSequenceEqual(self.con.select('space_1', [500], index='primary'), + [[500, 123, 'he---, world']]) def test_10_space(self): space = self.con.space('space_1') @@ -266,15 +296,15 @@ def test_10_space(self): [22, 10, 'lol'] ]) self.assertSequenceEqual(space.select([501], index='primary'), []) - self.assertSequenceEqual(space.upsert([501, 123, 'hello, world'], - [(':', 2, 2, 3, "---")]), []) + self.assertSequenceEqual( + space.upsert([501, 123, 'hello, world'], [(':', 2, 2, 3, "---")]), []) self.assertSequenceEqual(space.select([501], index='primary'), [[501, 123, 'hello, world']]) - self.assertSequenceEqual(space.upsert([501, 123, 'hello, world'], - [(':', 2, 2, 3, "---")]), []) + self.assertSequenceEqual( + space.upsert([501, 123, 'hello, world'], [(':', 2, 2, 3, "---")]), []) self.assertSequenceEqual(space.update([400], [('!', 2, 'oingo, boingo')]), - [[400, 0, 'oingo, boingo', 'tuple_400']]) + [[400, 0, 'oingo, boingo', 'tuple_400']]) self.assertSequenceEqual(space.update([400], [('#', 2, 1)]), - [[400, 0, 'tuple_400']]) + [[400, 0, 'tuple_400']]) self.assertSequenceEqual(space.delete([900]), [[900, 10, 'foo']]) def test_11_select_all_hash(self): @@ -289,21 +319,23 @@ def test_11_select_all_hash(self): space.select((), iterator=tarantool.const.ITERATOR_EQ) def test_12_update_fields(self): - self.srv.admin( - """ - do - local sp = box.schema.create_space('sp', { - format = { - { name = 'fir', type = 'unsigned' }, - { name = 'sec', type = 'string' }, - { name = 'thi', type = 'unsigned' }, - } - }) - sp:create_index('pr', { - parts = {1, 'unsigned'} - }) - end - """) + resp = self.srv.admin( + """ + do + local sp = box.schema.create_space('sp', { + format = { + { name = 'fir', type = 'unsigned' }, + { name = 'sec', type = 'string' }, + { name = 'thi', type = 'unsigned' }, + } + }) + sp:create_index('pr', { + parts = {1, 'unsigned'} + }) + end + return true + """) + assert_admin_success(resp) self.con.insert('sp', [2, 'help', 4]) self.assertSequenceEqual( self.con.update('sp', (2,), [('+', 'thi', 3)]), @@ -359,7 +391,12 @@ def test_15_extra_error_info_stacked(self): self.assertEqual(exc.extra_info.message, "Timeout exceeded") self.assertEqual(exc.extra_info.errno, 0) self.assertEqual(exc.extra_info.errcode, 78) - self.assertEqual(exc.extra_info.fields, None) + actual_fields = exc.extra_info.fields + if actual_fields is None: + actual_fields = {} + expected_fields = {} + self.assertGreaterEqual(actual_fields.items(), + expected_fields.items()) self.assertNotEqual(exc.extra_info.prev, None) prev = exc.extra_info.prev self.assertEqual(prev.type, 'ClientError') @@ -368,7 +405,12 @@ def test_15_extra_error_info_stacked(self): self.assertEqual(prev.message, "Unknown error") self.assertEqual(prev.errno, 0) self.assertEqual(prev.errcode, 0) - self.assertEqual(prev.fields, None) + actual_fields = prev.fields + if actual_fields is None: + actual_fields = {} + expected_fields = {} + self.assertGreaterEqual(actual_fields.items(), + expected_fields.items()) else: self.fail('Expected error') @@ -383,27 +425,27 @@ def test_16_extra_error_info_fields(self): self.assertTrue(isinstance(exc.extra_info.file, str)) self.assertTrue(exc.extra_info.line > 0) self.assertEqual( - exc.extra_info.message, - "Create access to function 'forbidden_function' is denied for user 'test'") + exc.extra_info.message, + "Create access to function 'forbidden_function' is denied for user 'test'") self.assertEqual(exc.extra_info.errno, 0) self.assertEqual(exc.extra_info.errcode, 42) - self.assertEqual( - exc.extra_info.fields, + self.assertGreaterEqual( + exc.extra_info.fields.items(), { 'object_type': 'function', 'object_name': 'forbidden_function', 'access_type': 'Create' - }) + }.items()) self.assertEqual(exc.extra_info.prev, None) else: self.fail('Expected error') @classmethod - def tearDownClass(self): - self.con.close() - self.srv.stop() - self.srv.clean() - - if self.sock_srv is not None: - self.sock_srv.stop() - self.sock_srv.clean() + def tearDownClass(cls): + cls.con.close() + cls.srv.stop() + cls.srv.clean() + + if cls.sock_srv is not None: + cls.sock_srv.stop() + cls.sock_srv.clean() diff --git a/test/suites/test_encoding.py b/test/suites/test_encoding.py index 45bc6053..dcda5983 100644 --- a/test/suites/test_encoding.py +++ b/test/suites/test_encoding.py @@ -1,73 +1,93 @@ +""" +This module tests various type encoding cases. +""" +# pylint: disable=missing-class-docstring,missing-function-docstring,duplicate-code + import sys import unittest +import pkg_resources import tarantool from tarantool.error import DatabaseError from .lib.skip import skip_or_run_varbinary_test, skip_or_run_error_extra_info_test from .lib.tarantool_server import TarantoolServer +from .utils import assert_admin_success + + +class TestSuiteEncoding(unittest.TestCase): + # pylint: disable=invalid-name -class TestSuite_Encoding(unittest.TestCase): @classmethod - def setUpClass(self): + def setUpClass(cls): print(' ENCODING '.center(70, '='), file=sys.stderr) print('-' * 70, file=sys.stderr) - self.srv = TarantoolServer() - self.srv.script = 'test/suites/box.lua' - self.srv.start() + cls.srv = TarantoolServer() + cls.srv.script = 'test/suites/box.lua' + cls.srv.start() - self.srv.admin(""" - box.schema.user.create('test', { password = 'test' }) - box.schema.user.grant('test', 'execute,read,write', 'universe') + resp = cls.srv.admin(""" + box.schema.user.create('test', { password = 'test', if_not_exists = true }) + box.schema.user.grant('test', 'execute,read,write', 'universe', + nil, {if_not_exists = true}) + + return true """) + assert_admin_success(resp) - args = [self.srv.host, self.srv.args['primary']] - kwargs = { 'user': 'test', 'password': 'test' } - self.con_encoding_utf8 = tarantool.Connection(*args, encoding='utf-8', **kwargs) - self.con_encoding_none = tarantool.Connection(*args, encoding=None, **kwargs) - self.conns = [self.con_encoding_utf8, self.con_encoding_none] + args = [cls.srv.host, cls.srv.args['primary']] + kwargs = {'user': 'test', 'password': 'test'} + cls.con_encoding_utf8 = tarantool.Connection(*args, encoding='utf-8', **kwargs) + cls.con_encoding_none = tarantool.Connection(*args, encoding=None, **kwargs) + cls.conns = [cls.con_encoding_utf8, cls.con_encoding_none] - self.srv.admin("box.schema.create_space('space_str')") - self.srv.admin(""" + resp = cls.srv.admin(""" + box.schema.create_space('space_str', {if_not_exists = true}) box.space['space_str']:create_index('primary', { type = 'tree', parts = {1, 'str'}, - unique = true}) - """.replace('\n', ' ')) - - self.srv.admin("box.schema.create_space('space_varbin')") - self.srv.admin(r""" - box.space['space_varbin']:format({ - { - 'id', - type = 'number', - is_nullable = false - }, - { - 'varbin', - type = 'varbinary', - is_nullable = false, - } - }) - """.replace('\n', ' ')) - self.srv.admin(""" - box.space['space_varbin']:create_index('id', { - type = 'tree', - parts = {1, 'number'}, - unique = true}) - """.replace('\n', ' ')) - self.srv.admin(""" - box.space['space_varbin']:create_index('varbin', { - type = 'tree', - parts = {2, 'varbinary'}, - unique = true}) - """.replace('\n', ' ')) + unique = true, + if_not_exists = true}) + + return true + """) + assert_admin_success(resp) + + if cls.srv.admin.tnt_version >= pkg_resources.parse_version('2.2.1'): + resp = cls.srv.admin(""" + box.schema.create_space('space_varbin', {if_not_exists = true}) + box.space['space_varbin']:format({ + { + 'id', + type = 'number', + is_nullable = false + }, + { + 'varbin', + type = 'varbinary', + is_nullable = false, + } + }) + box.space['space_varbin']:create_index('id', { + type = 'tree', + parts = {1, 'number'}, + unique = true, + if_not_exists = true}) + box.space['space_varbin']:create_index('varbin', { + type = 'tree', + parts = {2, 'varbinary'}, + unique = true, + if_not_exists = true}) + + return true + """) + assert_admin_success(resp) def assertNotRaises(self, func, *args, **kwargs): try: func(*args, **kwargs) - except Exception as e: - self.fail('Function raised Exception: %s' % repr(e)) + except Exception as exc: # pylint: disable=bad-option-value,broad-exception-caught,broad-except + self.fail(f'Function raised Exception: {repr(exc)}') def setUp(self): # prevent a remote tarantool from clean our session @@ -92,9 +112,9 @@ def test_01_02_string_decode_for_encoding_utf8_behavior(self): data = 'test_01_02' space = 'space_str' - self.srv.admin("box.space['%s']:insert{'%s'}" % (space, data)) + self.srv.admin(f"box.space['{space}']:insert{{'{data}'}}") - resp = self.con_encoding_utf8.eval("return box.space['%s']:get('%s')" % (space, data)) + resp = self.con_encoding_utf8.eval(f"return box.space['{space}']:get('{data}')") self.assertSequenceEqual(resp, [[data]]) @skip_or_run_varbinary_test @@ -105,7 +125,7 @@ def test_01_03_bytes_encode_for_encoding_utf8_behavior(self): self.assertNotRaises(self.con_encoding_utf8.insert, space, [data_id, data]) - resp = self.con_encoding_utf8.select(space, [ data ], index='varbin') + resp = self.con_encoding_utf8.select(space, [data], index='varbin') self.assertSequenceEqual(resp, [[data_id, data]]) @skip_or_run_varbinary_test @@ -115,13 +135,13 @@ def test_01_04_varbinary_decode_for_encoding_utf8_behavior(self): data = bytes(bytearray.fromhex(data_hex)) space = 'space_varbin' - self.con_encoding_utf8.execute(""" - INSERT INTO "%s" VALUES (%d, x'%s'); - """ % (space, data_id, data_hex)) + self.con_encoding_utf8.execute(f""" + INSERT INTO "{space}" VALUES ({data_id}, x'{data_hex}'); + """) - resp = self.con_encoding_utf8.execute(""" - SELECT * FROM "%s" WHERE "varbin" == x'%s'; - """ % (space, data_hex)) + resp = self.con_encoding_utf8.execute(f""" + SELECT * FROM "{space}" WHERE "varbin" == x'{data_hex}'; + """) self.assertSequenceEqual(resp, [[data_id, data]]) # encoding = None @@ -144,9 +164,9 @@ def test_02_02_string_decode_for_encoding_none_behavior(self): data_decoded = b'test_02_02' space = 'space_str' - self.srv.admin("box.space['%s']:insert{'%s'}" % (space, data)) + self.srv.admin(f"box.space['{space}']:insert{{'{data}'}}") - resp = self.con_encoding_none.eval("return box.space['%s']:get('%s')" % (space, data)) + resp = self.con_encoding_none.eval(f"return box.space['{space}']:get('{data}')") self.assertSequenceEqual(resp, [[data_decoded]]) def test_02_03_bytes_encode_for_encoding_none_behavior(self): @@ -165,13 +185,13 @@ def test_02_04_varbinary_decode_for_encoding_none_behavior(self): data = bytes(bytearray.fromhex(data_hex)) space = 'space_varbin' - self.con_encoding_none.execute(""" - INSERT INTO "%s" VALUES (%d, x'%s'); - """ % (space, data_id, data_hex)) + self.con_encoding_none.execute(f""" + INSERT INTO "{space}" VALUES ({data_id}, x'{data_hex}'); + """) - resp = self.con_encoding_none.execute(""" - SELECT * FROM "%s" WHERE "varbin" == x'%s'; - """ % (space, data_hex)) + resp = self.con_encoding_none.execute(f""" + SELECT * FROM "{space}" WHERE "varbin" == x'{data_hex}'; + """) self.assertSequenceEqual(resp, [[data_id, data]]) @skip_or_run_error_extra_info_test @@ -195,8 +215,8 @@ def test_02_05_error_extra_info_decode_for_encoding_none_behavior(self): self.fail('Expected error') @classmethod - def tearDownClass(self): - for con in self.conns: + def tearDownClass(cls): + for con in cls.conns: con.close() - self.srv.stop() - self.srv.clean() + cls.srv.stop() + cls.srv.clean() diff --git a/test/suites/test_error_ext.py b/test/suites/test_error_ext.py index 89f5c35f..d5e96e75 100644 --- a/test/suites/test_error_ext.py +++ b/test/suites/test_error_ext.py @@ -1,57 +1,68 @@ +""" +This module tests work with extended error type. +""" +# pylint: disable=missing-class-docstring,missing-function-docstring,protected-access,line-too-long,duplicate-code + import sys import unittest -import uuid -import msgpack -import warnings -import tarantool import pkg_resources +import msgpack + +import tarantool from tarantool.msgpack_ext.packer import default as packer_default from tarantool.msgpack_ext.unpacker import ext_hook as unpacker_ext_hook from .lib.tarantool_server import TarantoolServer from .lib.skip import skip_or_run_error_ext_type_test +from .utils import assert_admin_success + -class TestSuite_ErrorExt(unittest.TestCase): +class TestSuiteErrorExt(unittest.TestCase): @classmethod - def setUpClass(self): + def setUpClass(cls): print(' ERROR EXT TYPE '.center(70, '='), file=sys.stderr) print('-' * 70, file=sys.stderr) - self.srv = TarantoolServer() - self.srv.script = 'test/suites/box.lua' - self.srv.start() + cls.srv = TarantoolServer() + cls.srv.script = 'test/suites/box.lua' + cls.srv.start() - self.adm = self.srv.admin - self.adm(r""" - box.schema.space.create('test') + cls.adm = cls.srv.admin + resp = cls.adm(""" + box.schema.space.create('test', {if_not_exists = true}) box.space['test']:create_index('primary', { type = 'tree', parts = {1, 'string'}, - unique = true}) + unique = true, + if_not_exists = true}) box.schema.user.create('test', {password = 'test', if_not_exists = true}) - box.schema.user.grant('test', 'read,write,execute,create', 'universe') + box.schema.user.grant('test', 'read,write,execute,create', 'universe', + nil, {if_not_exists = true}) box.schema.user.create('no_grants', {if_not_exists = true}) + + return true """) + assert_admin_success(resp) - self.conn_encoding_utf8 = tarantool.Connection( - self.srv.host, self.srv.args['primary'], + cls.conn_encoding_utf8 = tarantool.Connection( + cls.srv.host, cls.srv.args['primary'], user='test', password='test', encoding='utf-8') - self.conn_encoding_none = tarantool.Connection( - self.srv.host, self.srv.args['primary'], + cls.conn_encoding_none = tarantool.Connection( + cls.srv.host, cls.srv.args['primary'], user='test', password='test', encoding=None) - if self.adm.tnt_version >= pkg_resources.parse_version('2.10.0'): - self.conn_encoding_utf8.eval(r""" + if cls.adm.tnt_version >= pkg_resources.parse_version('2.10.0'): + cls.conn_encoding_utf8.eval(r""" local err = box.error.new(box.error.UNKNOWN) rawset(_G, 'simple_error', err) """) # https://github.com/tarantool/tarantool/blob/125c13c81abb302708771ba04d59382d44a4a512/test/box-tap/extended_error.test.lua - self.conn_encoding_utf8.eval(r""" + cls.conn_encoding_utf8.eval(r""" local user = box.session.user() box.schema.func.create('forbidden_function', {body = 'function() end'}) box.session.su('no_grants') @@ -61,7 +72,7 @@ def setUpClass(self): """) # https://github.com/tarantool/tarantool/blob/125c13c81abb302708771ba04d59382d44a4a512/test/box-tap/extended_error.test.lua - self.conn_encoding_utf8.eval(r""" + cls.conn_encoding_utf8.eval(r""" local e1 = box.error.new(box.error.UNKNOWN) local e2 = box.error.new(box.error.UNKNOWN) e2:set_prev(e1) @@ -73,12 +84,15 @@ def setUp(self): if self.srv.is_started(): self.srv.touch_lock() - self.adm("box.space['test']:truncate()") - + resp = self.adm(""" + box.space['test']:truncate() + return true + """) + assert_admin_success(resp) # msgpack data for different encodings are actually the same, # but sometimes python msgpack module use different string - # types (str8 and str16) for the same strings depending on use_bin_type: + # types (str8 and str16) for the same strings depending on use_bin_type: # # >>> msgpack.Packer(use_bin_type=True).pack('[string " local err = box.error.ne..."]') # b'\xd9;[string " local err = box.error.ne..."]' @@ -97,9 +111,9 @@ def setUp(self): errno=0, errcode=0, ), - 'msgpack': (b'\x81\x00\x91\x86\x00\xab\x43\x6c\x69\x65\x6e\x74' + - b'\x45\x72\x72\x6f\x72\x01\xa4\x65\x76\x61\x6c\x02' + - b'\x01\x03\xad\x55\x6e\x6b\x6e\x6f\x77\x6e\x20\x65' + + 'msgpack': (b'\x81\x00\x91\x86\x00\xab\x43\x6c\x69\x65\x6e\x74' + b'\x45\x72\x72\x6f\x72\x01\xa4\x65\x76\x61\x6c\x02' + b'\x01\x03\xad\x55\x6e\x6b\x6e\x6f\x77\x6e\x20\x65' b'\x72\x72\x6f\x72\x04\x00\x05\x00'), 'tarantool': "simple_error", }, @@ -114,9 +128,9 @@ def setUp(self): errno=0, errcode=0, ), - 'msgpack': (b'\x81\x00\x91\x86\x00\xab\x43\x6c\x69\x65\x6e\x74' + - b'\x45\x72\x72\x6f\x72\x01\xa4\x65\x76\x61\x6c\x02' + - b'\x01\x03\xad\x55\x6e\x6b\x6e\x6f\x77\x6e\x20\x65' + + 'msgpack': (b'\x81\x00\x91\x86\x00\xab\x43\x6c\x69\x65\x6e\x74' + b'\x45\x72\x72\x6f\x72\x01\xa4\x65\x76\x61\x6c\x02' + b'\x01\x03\xad\x55\x6e\x6b\x6e\x6f\x77\x6e\x20\x65' b'\x72\x72\x6f\x72\x04\x00\x05\x00'), 'tarantool': "simple_error", }, @@ -127,7 +141,8 @@ def setUp(self): type='AccessDeniedError', file='/__w/sdk/sdk/tarantool-2.10/tarantool/src/box/func.c', line=535, - message="Execute access to function 'forbidden_function' is denied for user 'no_grants'", + message=("Execute access to function 'forbidden_function' is denied " + "for user 'no_grants'"), errno=0, errcode=42, fields={ @@ -136,26 +151,26 @@ def setUp(self): 'access_type': 'Execute', }, ), - 'msgpack': (b'\x81\x00\x91\x87\x00\xb1\x41\x63\x63\x65\x73\x73' + - b'\x44\x65\x6e\x69\x65\x64\x45\x72\x72\x6f\x72\x01' + - b'\xd9\x34\x2f\x5f\x5f\x77\x2f\x73\x64\x6b\x2f\x73' + - b'\x64\x6b\x2f\x74\x61\x72\x61\x6e\x74\x6f\x6f\x6c' + - b'\x2d\x32\x2e\x31\x30\x2f\x74\x61\x72\x61\x6e\x74' + - b'\x6f\x6f\x6c\x2f\x73\x72\x63\x2f\x62\x6f\x78\x2f' + - b'\x66\x75\x6e\x63\x2e\x63\x02\xcd\x02\x17\x03\xd9' + - b'\x4e\x45\x78\x65\x63\x75\x74\x65\x20\x61\x63\x63' + - b'\x65\x73\x73\x20\x74\x6f\x20\x66\x75\x6e\x63\x74' + - b'\x69\x6f\x6e\x20\x27\x66\x6f\x72\x62\x69\x64\x64' + - b'\x65\x6e\x5f\x66\x75\x6e\x63\x74\x69\x6f\x6e\x27' + - b'\x20\x69\x73\x20\x64\x65\x6e\x69\x65\x64\x20\x66' + - b'\x6f\x72\x20\x75\x73\x65\x72\x20\x27\x6e\x6f\x5f' + - b'\x67\x72\x61\x6e\x74\x73\x27\x04\x00\x05\x2a\x06' + - b'\x83\xab\x6f\x62\x6a\x65\x63\x74\x5f\x74\x79\x70' + - b'\x65\xa8\x66\x75\x6e\x63\x74\x69\x6f\x6e\xab\x6f' + - b'\x62\x6a\x65\x63\x74\x5f\x6e\x61\x6d\x65\xb2\x66' + - b'\x6f\x72\x62\x69\x64\x64\x65\x6e\x5f\x66\x75\x6e' + - b'\x63\x74\x69\x6f\x6e\xab\x61\x63\x63\x65\x73\x73' + - b'\x5f\x74\x79\x70\x65\xa7\x45\x78\x65\x63\x75\x74' + + 'msgpack': (b'\x81\x00\x91\x87\x00\xb1\x41\x63\x63\x65\x73\x73' + b'\x44\x65\x6e\x69\x65\x64\x45\x72\x72\x6f\x72\x01' + b'\xd9\x34\x2f\x5f\x5f\x77\x2f\x73\x64\x6b\x2f\x73' + b'\x64\x6b\x2f\x74\x61\x72\x61\x6e\x74\x6f\x6f\x6c' + b'\x2d\x32\x2e\x31\x30\x2f\x74\x61\x72\x61\x6e\x74' + b'\x6f\x6f\x6c\x2f\x73\x72\x63\x2f\x62\x6f\x78\x2f' + b'\x66\x75\x6e\x63\x2e\x63\x02\xcd\x02\x17\x03\xd9' + b'\x4e\x45\x78\x65\x63\x75\x74\x65\x20\x61\x63\x63' + b'\x65\x73\x73\x20\x74\x6f\x20\x66\x75\x6e\x63\x74' + b'\x69\x6f\x6e\x20\x27\x66\x6f\x72\x62\x69\x64\x64' + b'\x65\x6e\x5f\x66\x75\x6e\x63\x74\x69\x6f\x6e\x27' + b'\x20\x69\x73\x20\x64\x65\x6e\x69\x65\x64\x20\x66' + b'\x6f\x72\x20\x75\x73\x65\x72\x20\x27\x6e\x6f\x5f' + b'\x67\x72\x61\x6e\x74\x73\x27\x04\x00\x05\x2a\x06' + b'\x83\xab\x6f\x62\x6a\x65\x63\x74\x5f\x74\x79\x70' + b'\x65\xa8\x66\x75\x6e\x63\x74\x69\x6f\x6e\xab\x6f' + b'\x62\x6a\x65\x63\x74\x5f\x6e\x61\x6d\x65\xb2\x66' + b'\x6f\x72\x62\x69\x64\x64\x65\x6e\x5f\x66\x75\x6e' + b'\x63\x74\x69\x6f\x6e\xab\x61\x63\x63\x65\x73\x73' + b'\x5f\x74\x79\x70\x65\xa7\x45\x78\x65\x63\x75\x74' b'\x65'), 'tarantool': "access_denied_error", 'ignore_file_info': True, @@ -167,7 +182,8 @@ def setUp(self): type=b'AccessDeniedError', file=b'/__w/sdk/sdk/tarantool-2.10/tarantool/src/box/func.c', line=535, - message=b"Execute access to function 'forbidden_function' is denied for user 'no_grants'", + message=(b"Execute access to function 'forbidden_function' is denied " + b"for user 'no_grants'"), errno=0, errcode=42, fields={ @@ -176,26 +192,26 @@ def setUp(self): b'access_type': b'Execute', }, ), - 'msgpack': (b'\x81\x00\x91\x87\x00\xb1\x41\x63\x63\x65\x73\x73' + - b'\x44\x65\x6e\x69\x65\x64\x45\x72\x72\x6f\x72\x01' + - b'\xda\x00\x34\x2f\x5f\x5f\x77\x2f\x73\x64\x6b\x2f' + - b'\x73\x64\x6b\x2f\x74\x61\x72\x61\x6e\x74\x6f\x6f' + - b'\x6c\x2d\x32\x2e\x31\x30\x2f\x74\x61\x72\x61\x6e' + - b'\x74\x6f\x6f\x6c\x2f\x73\x72\x63\x2f\x62\x6f\x78' + - b'\x2f\x66\x75\x6e\x63\x2e\x63\x02\xcd\x02\x17\x03' + - b'\xda\x00\x4e\x45\x78\x65\x63\x75\x74\x65\x20\x61' + - b'\x63\x63\x65\x73\x73\x20\x74\x6f\x20\x66\x75\x6e' + - b'\x63\x74\x69\x6f\x6e\x20\x27\x66\x6f\x72\x62\x69' + - b'\x64\x64\x65\x6e\x5f\x66\x75\x6e\x63\x74\x69\x6f' + - b'\x6e\x27\x20\x69\x73\x20\x64\x65\x6e\x69\x65\x64' + - b'\x20\x66\x6f\x72\x20\x75\x73\x65\x72\x20\x27\x6e' + - b'\x6f\x5f\x67\x72\x61\x6e\x74\x73\x27\x04\x00\x05' + - b'\x2a\x06\x83\xab\x6f\x62\x6a\x65\x63\x74\x5f\x74' + - b'\x79\x70\x65\xa8\x66\x75\x6e\x63\x74\x69\x6f\x6e' + - b'\xab\x6f\x62\x6a\x65\x63\x74\x5f\x6e\x61\x6d\x65' + - b'\xb2\x66\x6f\x72\x62\x69\x64\x64\x65\x6e\x5f\x66' + - b'\x75\x6e\x63\x74\x69\x6f\x6e\xab\x61\x63\x63\x65' + - b'\x73\x73\x5f\x74\x79\x70\x65\xa7\x45\x78\x65\x63' + + 'msgpack': (b'\x81\x00\x91\x87\x00\xb1\x41\x63\x63\x65\x73\x73' + b'\x44\x65\x6e\x69\x65\x64\x45\x72\x72\x6f\x72\x01' + b'\xda\x00\x34\x2f\x5f\x5f\x77\x2f\x73\x64\x6b\x2f' + b'\x73\x64\x6b\x2f\x74\x61\x72\x61\x6e\x74\x6f\x6f' + b'\x6c\x2d\x32\x2e\x31\x30\x2f\x74\x61\x72\x61\x6e' + b'\x74\x6f\x6f\x6c\x2f\x73\x72\x63\x2f\x62\x6f\x78' + b'\x2f\x66\x75\x6e\x63\x2e\x63\x02\xcd\x02\x17\x03' + b'\xda\x00\x4e\x45\x78\x65\x63\x75\x74\x65\x20\x61' + b'\x63\x63\x65\x73\x73\x20\x74\x6f\x20\x66\x75\x6e' + b'\x63\x74\x69\x6f\x6e\x20\x27\x66\x6f\x72\x62\x69' + b'\x64\x64\x65\x6e\x5f\x66\x75\x6e\x63\x74\x69\x6f' + b'\x6e\x27\x20\x69\x73\x20\x64\x65\x6e\x69\x65\x64' + b'\x20\x66\x6f\x72\x20\x75\x73\x65\x72\x20\x27\x6e' + b'\x6f\x5f\x67\x72\x61\x6e\x74\x73\x27\x04\x00\x05' + b'\x2a\x06\x83\xab\x6f\x62\x6a\x65\x63\x74\x5f\x74' + b'\x79\x70\x65\xa8\x66\x75\x6e\x63\x74\x69\x6f\x6e' + b'\xab\x6f\x62\x6a\x65\x63\x74\x5f\x6e\x61\x6d\x65' + b'\xb2\x66\x6f\x72\x62\x69\x64\x64\x65\x6e\x5f\x66' + b'\x75\x6e\x63\x74\x69\x6f\x6e\xab\x61\x63\x63\x65' + b'\x73\x73\x5f\x74\x79\x70\x65\xa7\x45\x78\x65\x63' b'\x75\x74\x65'), 'tarantool': "access_denied_error", 'ignore_file_info': True, @@ -219,12 +235,12 @@ def setUp(self): errcode=0, ), ), - 'msgpack': (b'\x81\x00\x92\x86\x00\xab\x43\x6c\x69\x65\x6e\x74' + - b'\x45\x72\x72\x6f\x72\x01\xa4\x65\x76\x61\x6c\x02' + - b'\x03\x03\xad\x55\x6e\x6b\x6e\x6f\x77\x6e\x20\x65' + - b'\x72\x72\x6f\x72\x04\x00\x05\x00\x86\x00\xab\x43' + - b'\x6c\x69\x65\x6e\x74\x45\x72\x72\x6f\x72\x01\xa4' + - b'\x65\x76\x61\x6c\x02\x02\x03\xad\x55\x6e\x6b\x6e' + + 'msgpack': (b'\x81\x00\x92\x86\x00\xab\x43\x6c\x69\x65\x6e\x74' + b'\x45\x72\x72\x6f\x72\x01\xa4\x65\x76\x61\x6c\x02' + b'\x03\x03\xad\x55\x6e\x6b\x6e\x6f\x77\x6e\x20\x65' + b'\x72\x72\x6f\x72\x04\x00\x05\x00\x86\x00\xab\x43' + b'\x6c\x69\x65\x6e\x74\x45\x72\x72\x6f\x72\x01\xa4' + b'\x65\x76\x61\x6c\x02\x02\x03\xad\x55\x6e\x6b\x6e' b'\x6f\x77\x6e\x20\x65\x72\x72\x6f\x72\x04\x00\x05\x00'), 'tarantool': "chained_error", 'ignore_file_info': False, @@ -248,23 +264,21 @@ def setUp(self): errcode=0, ), ), - 'msgpack': (b'\x81\x00\x92\x86\x00\xab\x43\x6c\x69\x65\x6e\x74' + - b'\x45\x72\x72\x6f\x72\x01\xa4\x65\x76\x61\x6c\x02' + - b'\x03\x03\xad\x55\x6e\x6b\x6e\x6f\x77\x6e\x20\x65' + - b'\x72\x72\x6f\x72\x04\x00\x05\x00\x86\x00\xab\x43' + - b'\x6c\x69\x65\x6e\x74\x45\x72\x72\x6f\x72\x01\xa4' + - b'\x65\x76\x61\x6c\x02\x02\x03\xad\x55\x6e\x6b\x6e' + + 'msgpack': (b'\x81\x00\x92\x86\x00\xab\x43\x6c\x69\x65\x6e\x74' + b'\x45\x72\x72\x6f\x72\x01\xa4\x65\x76\x61\x6c\x02' + b'\x03\x03\xad\x55\x6e\x6b\x6e\x6f\x77\x6e\x20\x65' + b'\x72\x72\x6f\x72\x04\x00\x05\x00\x86\x00\xab\x43' + b'\x6c\x69\x65\x6e\x74\x45\x72\x72\x6f\x72\x01\xa4' + b'\x65\x76\x61\x6c\x02\x02\x03\xad\x55\x6e\x6b\x6e' b'\x6f\x77\x6e\x20\x65\x72\x72\x6f\x72\x04\x00\x05\x00'), 'tarantool': "chained_error", 'ignore_file_info': False, } } - def test_msgpack_decode(self): - for name in self.cases.keys(): + for name, case in self.cases.items(): with self.subTest(msg=name): - case = self.cases[name] conn = getattr(self, case['conn']) self.assertEqual( @@ -277,9 +291,8 @@ def test_msgpack_decode(self): @skip_or_run_error_ext_type_test def test_tarantool_decode(self): - for name in self.cases.keys(): + for name, case in self.cases.items(): with self.subTest(msg=name): - case = self.cases[name] conn = getattr(self, case['conn']) self.adm(f""" @@ -314,18 +327,23 @@ def test_tarantool_decode(self): self.assertEqual(err.message, expected_err.message) self.assertEqual(err.errno, expected_err.errno) self.assertEqual(err.errcode, expected_err.errcode) - self.assertEqual(err.fields, expected_err.fields) + expected_fields = expected_err.fields + if expected_fields is None: + expected_fields = {} + actual_fields = err.fields + if actual_fields is None: + actual_fields = {} + self.assertGreaterEqual(actual_fields.items(), + expected_fields.items()) err = err.prev expected_err = expected_err.prev self.assertEqual(err, expected_err) - def test_msgpack_encode(self): - for name in self.cases.keys(): + for name, case in self.cases.items(): with self.subTest(msg=name): - case = self.cases[name] conn = getattr(self, case['conn']) self.assertEqual(packer_default(case['python'], conn._packer_factory()), @@ -333,9 +351,8 @@ def test_msgpack_encode(self): @skip_or_run_error_ext_type_test def test_tarantool_encode(self): - for name in self.cases.keys(): + for name, case in self.cases.items(): with self.subTest(msg=name): - case = self.cases[name] conn = getattr(self, case['conn']) conn.insert( @@ -388,10 +405,9 @@ def test_tarantool_encode(self): self.assertSequenceEqual(conn.eval(lua_eval), [True]) - @classmethod - def tearDownClass(self): - self.conn_encoding_utf8.close() - self.conn_encoding_none.close() - self.srv.stop() - self.srv.clean() + def tearDownClass(cls): + cls.conn_encoding_utf8.close() + cls.conn_encoding_none.close() + cls.srv.stop() + cls.srv.clean() diff --git a/test/suites/test_execute.py b/test/suites/test_execute.py index f90b2fc3..46e85ce7 100644 --- a/test/suites/test_execute.py +++ b/test/suites/test_execute.py @@ -1,12 +1,18 @@ +""" +This module tests API for running SQL on a server. +""" +# pylint: disable=missing-class-docstring,missing-function-docstring,duplicate-code + import sys import unittest import tarantool from .lib.tarantool_server import TarantoolServer from .lib.skip import skip_or_run_sql_test +from .utils import assert_admin_success -class TestSuite_Execute(unittest.TestCase): +class TestSuiteExecute(unittest.TestCase): ddl = 'create table %s (id INTEGER PRIMARY KEY AUTOINCREMENT, ' \ 'name varchar(20))' @@ -19,13 +25,21 @@ class TestSuite_Execute(unittest.TestCase): ] @classmethod - def setUpClass(self): + def setUpClass(cls): print(' EXECUTE '.center(70, '='), file=sys.stderr) print('-' * 70, file=sys.stderr) - self.srv = TarantoolServer() - self.srv.script = 'test/suites/box.lua' - self.srv.start() - self.con = tarantool.Connection(self.srv.host, self.srv.args['primary']) + cls.srv = TarantoolServer() + cls.srv.script = 'test/suites/box.lua' + cls.srv.start() + cls.con = tarantool.Connection(cls.srv.host, cls.srv.args['primary']) + + # grant full access to guest + resp = cls.srv.admin(""" + box.schema.user.grant('guest', 'create,read,write,execute', 'universe', + nil, {if_not_exists = true}) + return true + """) + assert_admin_success(resp) @skip_or_run_sql_test def setUp(self): @@ -34,18 +48,14 @@ def setUp(self): self.srv.touch_lock() self.con.flush_schema() - # grant full access to guest - self.srv.admin("box.schema.user.grant('guest', 'create,read,write," - "execute', 'universe')") - @classmethod - def tearDownClass(self): - self.con.close() - self.srv.stop() - self.srv.clean() + def tearDownClass(cls): + cls.con.close() + cls.srv.stop() + cls.srv.clean() def _populate_data(self, table_name): - query = "insert into %s values (:id, :name)" % table_name + query = f"insert into {table_name} values (:id, :name)" for param in self.dml_params: self.con.execute(query, param) @@ -59,7 +69,7 @@ def test_dml_response(self): self.assertEqual(response.affected_row_count, 1) self.assertEqual(response.data, None) - query = "insert into %s values (:id, :name)" % table_name + query = f"insert into {table_name} values (:id, :name)" for num, param in enumerate(self.dml_params, start=1): response = self.con.execute(query, param) @@ -67,7 +77,7 @@ def test_dml_response(self): self.assertEqual(response.affected_row_count, 1) self.assertEqual(response.data, None) - query = "delete from %s where id in (4, 5)" % table_name + query = f"delete from {table_name} where id in (4, 5)" response = self.con.execute(query) self.assertEqual(response.autoincrement_ids, None) self.assertEqual(response.affected_row_count, 2) @@ -78,7 +88,7 @@ def test_dql_response(self): self._create_table(table_name) self._populate_data(table_name) - select_query = "select name from %s where id in (1, 3, 5)" % table_name + select_query = f"select name from {table_name} where id in (1, 3, 5)" response = self.con.execute(select_query) self.assertEqual(response.autoincrement_ids, None) self.assertEqual(response.affected_row_count, None) diff --git a/test/suites/test_interval.py b/test/suites/test_interval.py index 2252ebe8..2b6f6abe 100644 --- a/test/suites/test_interval.py +++ b/test/suites/test_interval.py @@ -1,40 +1,57 @@ +""" +This module tests work with datetime interval type. +""" +# pylint: disable=missing-class-docstring,missing-function-docstring,protected-access,too-many-function-args,duplicate-code + import re import sys import unittest + import msgpack -import warnings -import tarantool -import pandas -import pytz +import tarantool +from tarantool.error import MsgpackError from tarantool.msgpack_ext.packer import default as packer_default from tarantool.msgpack_ext.unpacker import ext_hook as unpacker_ext_hook +from tarantool.msgpack_ext.types.interval import ( + MAX_YEAR_RANGE, + MAX_MONTH_RANGE, + MAX_WEEK_RANGE, + MAX_DAY_RANGE, + MAX_HOUR_RANGE, + MAX_MIN_RANGE, + MAX_SEC_RANGE, + MAX_NSEC_RANGE, +) from .lib.tarantool_server import TarantoolServer from .lib.skip import skip_or_run_datetime_test -from tarantool.error import MsgpackError +from .utils import assert_admin_success + -class TestSuite_Interval(unittest.TestCase): +class TestSuiteInterval(unittest.TestCase): @classmethod - def setUpClass(self): + def setUpClass(cls): print(' INTERVAL EXT TYPE '.center(70, '='), file=sys.stderr) print('-' * 70, file=sys.stderr) - self.srv = TarantoolServer() - self.srv.script = 'test/suites/box.lua' - self.srv.start() + cls.srv = TarantoolServer() + cls.srv.script = 'test/suites/box.lua' + cls.srv.start() - self.adm = self.srv.admin - self.adm(r""" + cls.adm = cls.srv.admin + resp = cls.adm(""" _, datetime = pcall(require, 'datetime') - box.schema.space.create('test') + box.schema.space.create('test', {if_not_exists = true}) box.space['test']:create_index('primary', { type = 'tree', parts = {1, 'string'}, - unique = true}) + unique = true, + if_not_exists = true}) box.schema.user.create('test', {password = 'test', if_not_exists = true}) - box.schema.user.grant('test', 'read,write,execute', 'universe') + box.schema.user.grant('test', 'read,write,execute', 'universe', + nil, {if_not_exists = true}) local function add(arg1, arg2) return arg1 + arg2 @@ -45,19 +62,26 @@ def setUpClass(self): return arg1 - arg2 end rawset(_G, 'sub', sub) + + return true """) + assert_admin_success(resp) - self.con = tarantool.Connection(self.srv.host, self.srv.args['primary'], - user='test', password='test') + cls.con = tarantool.Connection(cls.srv.host, cls.srv.args['primary'], + user='test', password='test') def setUp(self): # prevent a remote tarantool from clean our session if self.srv.is_started(): self.srv.touch_lock() - self.adm("box.space['test']:truncate()") + resp = self.adm(""" + box.space['test']:truncate() + return true + """) + assert_admin_success(resp) - def test_Interval_positional_init(self): + def test_interval_positional_init(self): self.assertRaisesRegex( TypeError, re.escape('__init__() takes 1 positional argument but 2 were given'), lambda: tarantool.Interval(1)) @@ -67,101 +91,408 @@ def test_Interval_positional_init(self): 'python': tarantool.Interval(year=1), 'msgpack': (b'\x02\x00\x01\x08\x01'), 'tarantool': r"datetime.interval.new({year=1})", + 'str': 'tarantool.Interval(year=1, month=0, week=0, day=0, hour=0, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', }, 'big_year': { 'python': tarantool.Interval(year=1000), 'msgpack': (b'\x02\x00\xcd\x03\xe8\x08\x01'), 'tarantool': r"datetime.interval.new({year=1000})", + 'str': 'tarantool.Interval(year=1000, month=0, week=0, day=0, hour=0, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', }, 'date': { 'python': tarantool.Interval(year=1, month=2, day=3), 'msgpack': (b'\x04\x00\x01\x01\x02\x03\x03\x08\x01'), 'tarantool': r"datetime.interval.new({year=1, month=2, day=3})", + 'str': 'tarantool.Interval(year=1, month=2, week=0, day=3, hour=0, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', }, 'big_month_date': { 'python': tarantool.Interval(year=1, month=100000, day=3), 'msgpack': (b'\x04\x00\x01\x01\xce\x00\x01\x86\xa0\x03\x03\x08\x01'), 'tarantool': r"datetime.interval.new({year=1, month=100000, day=3})", + 'str': 'tarantool.Interval(year=1, month=100000, week=0, day=3, hour=0, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', }, 'time': { 'python': tarantool.Interval(hour=1, minute=2, sec=3), 'msgpack': (b'\x04\x04\x01\x05\x02\x06\x03\x08\x01'), 'tarantool': r"datetime.interval.new({hour=1, min=2, sec=3})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=1, ' + 'minute=2, sec=3, nsec=0, adjust=Adjust.NONE)', }, 'big_seconds_time': { 'python': tarantool.Interval(hour=1, minute=2, sec=3000), 'msgpack': (b'\x04\x04\x01\x05\x02\x06\xcd\x0b\xb8\x08\x01'), 'tarantool': r"datetime.interval.new({hour=1, min=2, sec=3000})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=1, ' + 'minute=2, sec=3000, nsec=0, adjust=Adjust.NONE)', }, 'datetime': { 'python': tarantool.Interval(year=1, month=2, day=3, hour=1, minute=2, sec=3000), 'msgpack': (b'\x07\x00\x01\x01\x02\x03\x03\x04\x01\x05\x02\x06\xcd\x0b\xb8\x08\x01'), - 'tarantool': r"datetime.interval.new({year=1, month=2, day=3, hour=1, min=2, sec=3000})", + 'tarantool': r"datetime.interval.new({year=1, month=2, day=3, hour=1, " + r"min=2, sec=3000})", + 'str': 'tarantool.Interval(year=1, month=2, week=0, day=3, hour=1, ' + 'minute=2, sec=3000, nsec=0, adjust=Adjust.NONE)', }, 'nanoseconds': { 'python': tarantool.Interval(nsec=10000000), 'msgpack': (b'\x02\x07\xce\x00\x98\x96\x80\x08\x01'), 'tarantool': r"datetime.interval.new({nsec=10000000})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + 'minute=0, sec=0, nsec=10000000, adjust=Adjust.NONE)', }, 'datetime_with_nanoseconds': { 'python': tarantool.Interval(year=1, month=2, day=3, hour=1, minute=2, sec=3000, nsec=10000000), - 'msgpack': (b'\x08\x00\x01\x01\x02\x03\x03\x04\x01\x05\x02\x06\xcd\x0b\xb8\x07\xce' + + 'msgpack': (b'\x08\x00\x01\x01\x02\x03\x03\x04\x01\x05\x02\x06\xcd\x0b\xb8\x07\xce' b'\x00\x98\x96\x80\x08\x01'), - 'tarantool': r"datetime.interval.new({year=1, month=2, day=3, hour=1, " + + 'tarantool': r"datetime.interval.new({year=1, month=2, day=3, hour=1, " r"min=2, sec=3000, nsec=10000000})", + 'str': 'tarantool.Interval(year=1, month=2, week=0, day=3, hour=1, ' + 'minute=2, sec=3000, nsec=10000000, adjust=Adjust.NONE)', }, 'datetime_none_adjust': { 'python': tarantool.Interval(year=1, month=2, day=3, hour=1, minute=2, sec=3000, nsec=10000000, adjust=tarantool.IntervalAdjust.NONE), - 'msgpack': (b'\x08\x00\x01\x01\x02\x03\x03\x04\x01\x05\x02\x06\xcd\x0b\xb8\x07\xce' + + 'msgpack': (b'\x08\x00\x01\x01\x02\x03\x03\x04\x01\x05\x02\x06\xcd\x0b\xb8\x07\xce' b'\x00\x98\x96\x80\x08\x01'), - 'tarantool': r"datetime.interval.new({year=1, month=2, day=3, hour=1, " + + 'tarantool': r"datetime.interval.new({year=1, month=2, day=3, hour=1, " r"min=2, sec=3000, nsec=10000000, adjust='none'})", + 'str': 'tarantool.Interval(year=1, month=2, week=0, day=3, hour=1, ' + 'minute=2, sec=3000, nsec=10000000, adjust=Adjust.NONE)', }, 'datetime_excess_adjust': { 'python': tarantool.Interval(year=1, month=2, day=3, hour=1, minute=2, sec=3000, nsec=10000000, adjust=tarantool.IntervalAdjust.EXCESS), - 'msgpack': (b'\x07\x00\x01\x01\x02\x03\x03\x04\x01\x05\x02\x06\xcd\x0b\xb8\x07\xce' + + 'msgpack': (b'\x07\x00\x01\x01\x02\x03\x03\x04\x01\x05\x02\x06\xcd\x0b\xb8\x07\xce' b'\x00\x98\x96\x80'), - 'tarantool': r"datetime.interval.new({year=1, month=2, day=3, hour=1, " + + 'tarantool': r"datetime.interval.new({year=1, month=2, day=3, hour=1, " r"min=2, sec=3000, nsec=10000000, adjust='excess'})", + 'str': 'tarantool.Interval(year=1, month=2, week=0, day=3, hour=1, ' + 'minute=2, sec=3000, nsec=10000000, adjust=Adjust.EXCESS)', }, 'datetime_last_adjust': { 'python': tarantool.Interval(year=1, month=2, day=3, hour=1, minute=2, sec=3000, nsec=10000000, adjust=tarantool.IntervalAdjust.LAST), - 'msgpack': (b'\x08\x00\x01\x01\x02\x03\x03\x04\x01\x05\x02\x06\xcd\x0b\xb8\x07\xce' + + 'msgpack': (b'\x08\x00\x01\x01\x02\x03\x03\x04\x01\x05\x02\x06\xcd\x0b\xb8\x07\xce' b'\x00\x98\x96\x80\x08\x02'), - 'tarantool': r"datetime.interval.new({year=1, month=2, day=3, hour=1, " + + 'tarantool': r"datetime.interval.new({year=1, month=2, day=3, hour=1, " r"min=2, sec=3000, nsec=10000000, adjust='last'})", + 'str': 'tarantool.Interval(year=1, month=2, week=0, day=3, hour=1, ' + 'minute=2, sec=3000, nsec=10000000, adjust=Adjust.LAST)', }, 'all_zeroes': { 'python': tarantool.Interval(adjust=tarantool.IntervalAdjust.EXCESS), 'msgpack': (b'\x00'), 'tarantool': r"datetime.interval.new({adjust='excess'})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.EXCESS)', + }, + 'weeks': { + 'python': tarantool.Interval(week=3), + 'msgpack': (b'\x02\x02\x03\x08\x01'), + 'tarantool': r"datetime.interval.new({week=3})", + 'str': 'tarantool.Interval(year=0, month=0, week=3, day=0, hour=0, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'date_with_week': { + 'python': tarantool.Interval(year=1, month=2, week=3, day=4), + 'msgpack': (b'\x05\x00\x01\x01\x02\x02\x03\x03\x04\x08\x01'), + 'tarantool': r"datetime.interval.new({year=1, month=2, week=3, day=4})", + 'str': 'tarantool.Interval(year=1, month=2, week=3, day=4, hour=0, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'datetime_with_week': { + 'python': tarantool.Interval(year=1, month=2, week=3, day=4, hour=1, minute=2, + sec=3000, nsec=10000000), + 'msgpack': (b'\x09\x00\x01\x01\x02\x02\x03\x03\x04\x04\x01\x05\x02\x06\xcd\x0b\xb8' + b'\x07\xce\x00\x98\x96\x80\x08\x01'), + 'tarantool': r"datetime.interval.new({year=1, month=2, week=3, day=4, hour=1, " + r"min=2, sec=3000, nsec=10000000})", + 'str': 'tarantool.Interval(year=1, month=2, week=3, day=4, hour=1, ' + 'minute=2, sec=3000, nsec=10000000, adjust=Adjust.NONE)', + }, + 'min_year_interval': { + 'python': tarantool.Interval(year=-int(MAX_YEAR_RANGE)), + 'msgpack': (b'\x02\x00\xd2\xff\x4c\x91\x8b\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{year=-{int(MAX_YEAR_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{year=-{int(MAX_YEAR_RANGE)} + 1}}) - " + r"datetime.interval.new({year=1})", + 'str': f'tarantool.Interval(year=-{int(MAX_YEAR_RANGE)}, month=0, week=0, day=0, ' + 'hour=0, minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'max_year_interval': { + 'python': tarantool.Interval(year=int(MAX_YEAR_RANGE)), + 'msgpack': (b'\x02\x00\xce\x00\xb3\x6e\x75\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{year={int(MAX_YEAR_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{year={int(MAX_YEAR_RANGE)} - 1}}) + " + r"datetime.interval.new({year=1})", + 'str': f'tarantool.Interval(year={int(MAX_YEAR_RANGE)}, month=0, week=0, day=0, ' + 'hour=0, minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'min_month_interval': { + 'python': tarantool.Interval(month=-int(MAX_MONTH_RANGE)), + 'msgpack': (b'\x02\x01\xd2\xf7\x96\xd2\x84\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{month=-{int(MAX_MONTH_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{month=-{int(MAX_MONTH_RANGE)} + 1}}) - " + r"datetime.interval.new({month=1})", + 'str': f'tarantool.Interval(year=0, month=-{int(MAX_MONTH_RANGE)}, week=0, day=0, ' + 'hour=0, minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'max_month_interval': { + 'python': tarantool.Interval(month=int(MAX_MONTH_RANGE)), + 'msgpack': (b'\x02\x01\xce\x08\x69\x2d\x7c\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{month={int(MAX_MONTH_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{month={int(MAX_MONTH_RANGE)} - 1}}) + " + r"datetime.interval.new({month=1})", + 'str': f'tarantool.Interval(year=0, month={int(MAX_MONTH_RANGE)}, week=0, day=0, ' + 'hour=0, minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'min_week_interval': { + 'python': tarantool.Interval(week=-int(MAX_WEEK_RANGE)), + 'msgpack': (b'\x02\x02\xd2\xdb\x6d\x85\xa8\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{week=-{int(MAX_WEEK_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{week=-{int(MAX_WEEK_RANGE)} + 1}}) - " + r"datetime.interval.new({week=1})", + 'str': f'tarantool.Interval(year=0, month=0, week=-{int(MAX_WEEK_RANGE)}, day=0, ' + 'hour=0, minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'max_week_interval': { + 'python': tarantool.Interval(week=int(MAX_WEEK_RANGE)), + 'msgpack': (b'\x02\x02\xce\x24\x92\x7a\x58\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{week={int(MAX_WEEK_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{week={int(MAX_WEEK_RANGE)} - 1}}) + " + r"datetime.interval.new({week=1})", + 'str': f'tarantool.Interval(year=0, month=0, week={int(MAX_WEEK_RANGE)}, day=0, ' + 'hour=0, minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'min_day_interval': { + 'python': tarantool.Interval(day=-int(MAX_DAY_RANGE)), + 'msgpack': (b'\x02\x03\xd3\xff\xff\xff\xfe\xff\xfe\xa7\x92\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{day=-{int(MAX_DAY_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{day=-{int(MAX_DAY_RANGE)} + 1}}) - " + r"datetime.interval.new({day=1})", + 'str': f'tarantool.Interval(year=0, month=0, week=0, day=-{int(MAX_DAY_RANGE)}, ' + 'hour=0, minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + 'tarantool_8887_issue': True, + }, + 'max_day_interval': { + 'python': tarantool.Interval(day=int(MAX_DAY_RANGE)), + 'msgpack': (b'\x02\x03\xcf\x00\x00\x00\x01\x00\x01\x58\x6e\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{day={int(MAX_DAY_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{day={int(MAX_DAY_RANGE)} - 1}}) + " + r"datetime.interval.new({day=1})", + 'str': f'tarantool.Interval(year=0, month=0, week=0, day={int(MAX_DAY_RANGE)}, hour=0, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + 'tarantool_8887_issue': True, + }, + 'min_int32_day_interval': { + 'python': tarantool.Interval(day=-2147483648), + 'msgpack': (b'\x02\x03\xd2\x80\x00\x00\x00\x08\x01'), + 'tarantool': r"datetime.interval.new({day=-2147483648})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=-2147483648, hour=0, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'max_int32_day_interval': { + 'python': tarantool.Interval(day=2147483647), + 'msgpack': (b'\x02\x03\xce\x7f\xff\xff\xff\x08\x01'), + 'tarantool': r"datetime.interval.new({day=2147483647})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=2147483647, hour=0, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'min_hour_interval': { + 'python': tarantool.Interval(hour=-int(MAX_HOUR_RANGE)), + 'msgpack': (b'\x02\x04\xd3\xff\xff\xff\xe7\xff\xdf\xb5\xaa\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{hour=-{int(MAX_HOUR_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{hour=-{int(MAX_HOUR_RANGE)} + 1}}) - " + r"datetime.interval.new({hour=1})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, ' + f'hour=-{int(MAX_HOUR_RANGE)}, minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + 'tarantool_8887_issue': True, + }, + 'max_hour_interval': { + 'python': tarantool.Interval(hour=int(MAX_HOUR_RANGE)), + 'msgpack': (b'\x02\x04\xcf\x00\x00\x00\x18\x00\x20\x4a\x56\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{hour={int(MAX_HOUR_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{hour={int(MAX_HOUR_RANGE)} - 1}}) + " + r"datetime.interval.new({hour=1})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, ' + f'hour={int(MAX_HOUR_RANGE)}, minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + 'tarantool_8887_issue': True, + }, + 'min_int32_hour_interval': { + 'python': tarantool.Interval(hour=-2147483648), + 'msgpack': (b'\x02\x04\xd2\x80\x00\x00\x00\x08\x01'), + 'tarantool': r"datetime.interval.new({hour=-2147483648})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=-2147483648, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'max_int32_hour_interval': { + 'python': tarantool.Interval(hour=2147483647), + 'msgpack': (b'\x02\x04\xce\x7f\xff\xff\xff\x08\x01'), + 'tarantool': r"datetime.interval.new({hour=2147483647})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=2147483647, ' + 'minute=0, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'min_minute_interval': { + 'python': tarantool.Interval(minute=-int(MAX_MIN_RANGE)), + 'msgpack': (b'\x02\x05\xd3\xff\xff\xfa\x5f\xf8\x6e\x93\xd8\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{min=-{int(MAX_MIN_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{min=-{int(MAX_MIN_RANGE)} + 1}}) - " + r"datetime.interval.new({min=1})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + f'minute=-{int(MAX_MIN_RANGE)}, sec=0, nsec=0, adjust=Adjust.NONE)', + 'tarantool_8887_issue': True, + }, + 'max_minute_interval': { + 'python': tarantool.Interval(minute=int(MAX_MIN_RANGE)), + 'msgpack': (b'\x02\x05\xcf\x00\x00\x05\xa0\x07\x91\x6c\x28\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{min={int(MAX_MIN_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{min={int(MAX_MIN_RANGE)} - 1}}) + " + r"datetime.interval.new({min=1})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + f'minute={int(MAX_MIN_RANGE)}, sec=0, nsec=0, adjust=Adjust.NONE)', + 'tarantool_8887_issue': True, + }, + 'min_int32_minute_interval': { + 'python': tarantool.Interval(minute=-2147483648), + 'msgpack': (b'\x02\x05\xd2\x80\x00\x00\x00\x08\x01'), + 'tarantool': r"datetime.interval.new({min=-2147483648})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + 'minute=-2147483648, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'max_int32_minute_interval': { + 'python': tarantool.Interval(minute=2147483647), + 'msgpack': (b'\x02\x05\xce\x7f\xff\xff\xff\x08\x01'), + 'tarantool': r"datetime.interval.new({min=2147483647})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + 'minute=2147483647, sec=0, nsec=0, adjust=Adjust.NONE)', + }, + 'min_sec_interval': { + 'python': tarantool.Interval(sec=-int(MAX_SEC_RANGE)), + 'msgpack': (b'\x02\x06\xd3\xff\xfe\xae\x7e\x39\xea\xa6\xa0\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{sec=-{int(MAX_SEC_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{sec=-{int(MAX_SEC_RANGE)} + 1}}) - " + r"datetime.interval.new({sec=1})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + f'minute=0, sec=-{int(MAX_SEC_RANGE)}, nsec=0, adjust=Adjust.NONE)', + 'tarantool_8887_issue': True, + }, + 'max_sec_interval': { + 'python': tarantool.Interval(sec=int(MAX_SEC_RANGE)), + 'msgpack': (b'\x02\x06\xcf\x00\x01\x51\x81\xc6\x15\x59\x60\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{sec={int(MAX_SEC_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{sec={int(MAX_SEC_RANGE)} - 1}}) + " + r"datetime.interval.new({sec=1})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + f'minute=0, sec={int(MAX_SEC_RANGE)}, nsec=0, adjust=Adjust.NONE)', + 'tarantool_8887_issue': True, + }, + 'min_int32_sec_interval': { + 'python': tarantool.Interval(sec=-2147483648), + 'msgpack': (b'\x02\x06\xd2\x80\x00\x00\x00\x08\x01'), + 'tarantool': r"datetime.interval.new({sec=-2147483648})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + 'minute=0, sec=-2147483648, nsec=0, adjust=Adjust.NONE)', + }, + 'max_int32_sec_interval': { + 'python': tarantool.Interval(sec=2147483647), + 'msgpack': (b'\x02\x06\xce\x7f\xff\xff\xff\x08\x01'), + 'tarantool': r"datetime.interval.new({sec=2147483647})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + 'minute=0, sec=2147483647, nsec=0, adjust=Adjust.NONE)', + }, + 'min_nsec_interval': { + 'python': tarantool.Interval(nsec=-int(MAX_NSEC_RANGE)), + 'msgpack': (b'\x02\x07\xd2\x80\x00\x00\x01\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{nsec=-{int(MAX_NSEC_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{nsec=-{int(MAX_NSEC_RANGE)} + 1}}) - " + r"datetime.interval.new({nsec=1})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + f'minute=0, sec=0, nsec=-{int(MAX_NSEC_RANGE)}, adjust=Adjust.NONE)', + }, + 'max_nsec_interval': { + 'python': tarantool.Interval(nsec=int(MAX_NSEC_RANGE)), + 'msgpack': (b'\x02\x07\xce\x7f\xff\xff\xff\x08\x01'), + # Reason why Tarantool datetime is so weird here: + # https://github.com/tarantool/tarantool/issues/8878 + # Replace with f"datetime.interval.new({{nsec={int(MAX_NSEC_RANGE)}}}) + # after fix. + 'tarantool': f"datetime.interval.new({{nsec={int(MAX_NSEC_RANGE)} - 1}}) + " + r"datetime.interval.new({nsec=1})", + 'str': 'tarantool.Interval(year=0, month=0, week=0, day=0, hour=0, ' + f'minute=0, sec=0, nsec={int(MAX_NSEC_RANGE)}, adjust=Adjust.NONE)', }, } def test_msgpack_decode(self): - for name in self.cases.keys(): + for name, case in self.cases.items(): with self.subTest(msg=name): - case = self.cases[name] - - self.assertEqual(unpacker_ext_hook( - 6, - case['msgpack'], - self.con._unpacker_factory(), - ), - case['python']) + self.assertEqual( + unpacker_ext_hook( + 6, + case['msgpack'], + self.con._unpacker_factory(), + ), + case['python']) @skip_or_run_datetime_test def test_tarantool_decode(self): - for name in self.cases.keys(): + for name, case in self.cases.items(): with self.subTest(msg=name): - case = self.cases[name] + if ('tarantool_8887_issue' in case) and (case['tarantool_8887_issue'] is True): + self.skipTest('See https://github.com/tarantool/tarantool/issues/8887') self.adm(f"box.space['test']:replace{{'{name}', {case['tarantool']}, 'field'}}") @@ -169,24 +500,23 @@ def test_tarantool_decode(self): [[name, case['python'], 'field']]) def test_msgpack_encode(self): - for name in self.cases.keys(): + for name, case in self.cases.items(): with self.subTest(msg=name): - case = self.cases[name] - self.assertEqual(packer_default(case['python']), msgpack.ExtType(code=6, data=case['msgpack'])) @skip_or_run_datetime_test def test_tarantool_encode(self): - for name in self.cases.keys(): + for name, case in self.cases.items(): with self.subTest(msg=name): - case = self.cases[name] + if ('tarantool_8887_issue' in case) and (case['tarantool_8887_issue'] is True): + self.skipTest('See https://github.com/tarantool/tarantool/issues/8887') self.con.insert('test', [name, case['python'], 'field']) lua_eval = f""" local interval = {case['tarantool']} - + local tuple = box.space['test']:get('{name}') assert(tuple ~= nil) @@ -200,6 +530,11 @@ def test_tarantool_encode(self): self.assertSequenceEqual(self.adm(lua_eval), [True]) + def test_class_string(self): + for name, case in self.cases.items(): + with self.subTest(msg=name): + self.assertEqual(str(case['python']), case['str']) + self.assertEqual(repr(case['python']), case['str']) def test_unknown_field_decode(self): case = b'\x01\x09\xce\x00\x98\x96\x80' @@ -213,6 +548,86 @@ def test_unknown_adjust_decode(self): MsgpackError, '3 is not a valid Adjust', lambda: unpacker_ext_hook(6, case, self.con._unpacker_factory())) + out_of_range_cases = { + 'year_too_small': { + 'kwargs': {'year': -int(MAX_YEAR_RANGE + 1)}, + 'range': MAX_YEAR_RANGE, + }, + 'year_too_large': { + 'kwargs': {'year': int(MAX_YEAR_RANGE + 1)}, + 'range': MAX_YEAR_RANGE, + }, + 'month_too_small': { + 'kwargs': {'month': -int(MAX_MONTH_RANGE + 1)}, + 'range': MAX_MONTH_RANGE, + }, + 'month_too_big': { + 'kwargs': {'month': int(MAX_MONTH_RANGE + 1)}, + 'range': MAX_MONTH_RANGE, + }, + 'week_too_small': { + 'kwargs': {'week': -int(MAX_WEEK_RANGE + 1)}, + 'range': MAX_WEEK_RANGE, + }, + 'week_too_big': { + 'kwargs': {'week': int(MAX_WEEK_RANGE + 1)}, + 'range': MAX_WEEK_RANGE, + }, + 'day_too_small': { + 'kwargs': {'day': -int(MAX_DAY_RANGE + 1)}, + 'range': MAX_DAY_RANGE, + }, + 'day_too_big': { + 'kwargs': {'day': int(MAX_DAY_RANGE + 1)}, + 'range': MAX_DAY_RANGE, + }, + 'hour_too_small': { + 'kwargs': {'hour': -int(MAX_HOUR_RANGE + 1)}, + 'range': MAX_HOUR_RANGE, + }, + 'hour_too_big': { + 'kwargs': {'hour': int(MAX_HOUR_RANGE + 1)}, + 'range': MAX_HOUR_RANGE, + }, + 'minute_too_small': { + 'kwargs': {'minute': -int(MAX_MIN_RANGE + 1)}, + 'range': MAX_MIN_RANGE, + }, + 'minute_too_big': { + 'kwargs': {'minute': int(MAX_MIN_RANGE + 1)}, + 'range': MAX_MIN_RANGE, + }, + 'sec_too_small': { + 'kwargs': {'sec': -int(MAX_SEC_RANGE + 1)}, + 'range': MAX_SEC_RANGE, + }, + 'sec_too_big': { + 'kwargs': {'sec': int(MAX_SEC_RANGE + 1)}, + 'range': MAX_SEC_RANGE, + }, + 'nsec_too_small': { + 'kwargs': {'nsec': -int(MAX_NSEC_RANGE + 1)}, + 'range': MAX_NSEC_RANGE, + }, + 'nsec_too_big': { + 'kwargs': {'nsec': int(MAX_NSEC_RANGE + 1)}, + 'range': MAX_NSEC_RANGE, + }, + } + + def test_out_of_range(self): + # pylint: disable=cell-var-from-loop,unnecessary-lambda + + for name, case in self.out_of_range_cases.items(): + with self.subTest(msg=name): + name = next(iter(case['kwargs'])) + val = case['kwargs'][name] + self.assertRaisesRegex( + ValueError, re.escape( + f"value {val} of {name} is out of " + f"allowed range [{-case['range']}, {case['range']}]" + ), + lambda: tarantool.Interval(**case['kwargs'])) arithmetic_cases = { 'year': { @@ -241,9 +656,9 @@ def test_unknown_adjust_decode(self): }, 'datetime_with_nsec': { 'arg_1': tarantool.Interval(year=1, month=2, day=3, hour=1, minute=2, - sec=3000, nsec=10000000), + sec=3000, nsec=10000000), 'arg_2': tarantool.Interval(year=2, month=1, day=31, hour=-3, minute=0, - sec=1000, nsec=9876543), + sec=1000, nsec=9876543), 'res_add': tarantool.Interval(year=3, month=3, day=34, hour=-2, minute=2, sec=4000, nsec=19876543), 'res_sub': tarantool.Interval(year=-1, month=1, day=-28, hour=4, minute=2, @@ -269,43 +684,72 @@ def test_unknown_adjust_decode(self): 'res_add': tarantool.Interval(year=3, adjust=tarantool.IntervalAdjust.LAST), 'res_sub': tarantool.Interval(year=1, adjust=tarantool.IntervalAdjust.LAST), }, + 'weeks': { + 'arg_1': tarantool.Interval(week=2), + 'arg_2': tarantool.Interval(week=1), + 'res_add': tarantool.Interval(week=3), + 'res_sub': tarantool.Interval(week=1), + }, + 'date_with_week': { + 'arg_1': tarantool.Interval(year=1, month=2, week=3, day=4), + 'arg_2': tarantool.Interval(year=4, month=3, week=2, day=1), + 'res_add': tarantool.Interval(year=5, month=5, week=5, day=5), + 'res_sub': tarantool.Interval(year=-3, month=-1, week=1, day=3), + }, + 'datetime_with_week': { + 'arg_1': tarantool.Interval(year=1, month=2, week=3, day=4, hour=1, minute=2, + sec=3000, nsec=10000000), + 'arg_2': tarantool.Interval(year=2, month=1, week=-1, day=31, hour=-3, minute=0, + sec=1000, nsec=9876543), + 'res_add': tarantool.Interval(year=3, month=3, week=2, day=35, hour=-2, minute=2, + sec=4000, nsec=19876543), + 'res_sub': tarantool.Interval(year=-1, month=1, week=4, day=-27, hour=4, minute=2, + sec=2000, nsec=123457), + }, } def test_python_interval_addition(self): - for name in self.arithmetic_cases.keys(): + for name, case in self.arithmetic_cases.items(): with self.subTest(msg=name): - case = self.arithmetic_cases[name] - self.assertEqual(case['arg_1'] + case['arg_2'], case['res_add']) def test_python_interval_subtraction(self): - for name in self.arithmetic_cases.keys(): + for name, case in self.arithmetic_cases.items(): with self.subTest(msg=name): - case = self.arithmetic_cases[name] - self.assertEqual(case['arg_1'] - case['arg_2'], case['res_sub']) @skip_or_run_datetime_test def test_tarantool_interval_addition(self): - for name in self.arithmetic_cases.keys(): + for name, case in self.arithmetic_cases.items(): with self.subTest(msg=name): - case = self.arithmetic_cases[name] - self.assertSequenceEqual(self.con.call('add', case['arg_1'], case['arg_2']), [case['res_add']]) @skip_or_run_datetime_test def test_tarantool_interval_subtraction(self): - for name in self.arithmetic_cases.keys(): + for name, case in self.arithmetic_cases.items(): with self.subTest(msg=name): - case = self.arithmetic_cases[name] - self.assertSequenceEqual(self.con.call('sub', case['arg_1'], case['arg_2']), [case['res_sub']]) + def test_addition_overflow(self): + self.assertRaisesRegex( + ValueError, re.escape( + f"value {int(MAX_YEAR_RANGE) + 1} of year is out of " + f"allowed range [{-MAX_YEAR_RANGE}, {MAX_YEAR_RANGE}]" + ), + lambda: tarantool.Interval(year=int(MAX_YEAR_RANGE)) + tarantool.Interval(year=1)) + + def test_subtraction_overflow(self): + self.assertRaisesRegex( + ValueError, re.escape( + f"value {-int(MAX_YEAR_RANGE) - 1} of year is out of " + f"allowed range [{-MAX_YEAR_RANGE}, {MAX_YEAR_RANGE}]" + ), + lambda: tarantool.Interval(year=-int(MAX_YEAR_RANGE)) - tarantool.Interval(year=1)) @classmethod - def tearDownClass(self): - self.con.close() - self.srv.stop() - self.srv.clean() + def tearDownClass(cls): + cls.con.close() + cls.srv.stop() + cls.srv.clean() diff --git a/test/suites/test_mesh.py b/test/suites/test_mesh.py index ae621bf5..a906597b 100644 --- a/test/suites/test_mesh.py +++ b/test/suites/test_mesh.py @@ -1,3 +1,9 @@ +""" +This module tests work with a cluster of Tarantool servers through +MeshConnection. +""" +# pylint: disable=missing-class-docstring,missing-function-docstring + import sys import unittest import warnings @@ -10,46 +16,57 @@ ClusterDiscoveryWarning, ) from .lib.tarantool_server import TarantoolServer +from .utils import assert_admin_success def create_server(_id): srv = TarantoolServer() srv.script = 'test/suites/box.lua' srv.start() - srv.admin("box.schema.user.create('test', {password = 'test', " + - "if_not_exists = true})") - srv.admin("box.schema.user.grant('test', 'execute', 'universe')") + resp = srv.admin(f""" + box.schema.user.create('test', {{password = 'test', if_not_exists = true}}) + box.schema.user.grant('test', 'execute', 'universe', + nil, {{if_not_exists = true}}) + + function srv_id() return {_id} end - # Create srv_id function (for testing purposes). - srv.admin("function srv_id() return %s end" % _id) + return true + """) + assert_admin_success(resp) return srv @unittest.skipIf(sys.platform.startswith("win"), 'Mesh tests on windows platform are not supported') -class TestSuite_Mesh(unittest.TestCase): +class TestSuiteMesh(unittest.TestCase): + # pylint: disable=too-many-instance-attributes + def define_cluster_function(self, func_name, servers): addresses = [(srv.host, srv.args['primary']) for srv in servers] - addresses_lua = ",".join("'%s:%d'" % address for address in addresses) - func_body = """ - function %s() - return {%s} + addresses_lua = ",".join(f"'{address[0]}:{address[1]}'" for address in addresses) + func_body = f""" + function {func_name}() + return {{{addresses_lua}}} end - """ % (func_name, addresses_lua) + return true + """ for srv in self.servers: - srv.admin(func_body) + resp = srv.admin(func_body) + assert_admin_success(resp) def define_custom_cluster_function(self, func_name, retval): - func_body = """ - function %s() - return %s + func_body = f""" + function {func_name}() + return {retval} end - """ % (func_name, retval) + return true + """ for srv in self.servers: - srv.admin(func_body) + resp = srv.admin(func_body) + assert_admin_success(resp) @classmethod - def setUpClass(self): + def setUpClass(cls): print(' MESH '.center(70, '='), file=sys.stderr) print('-' * 70, file=sys.stderr) @@ -91,7 +108,11 @@ def assert_srv_id(con, srv_id): # Start instance#1, stop instance#2 -- response from # instance#1 again. self.srv.start() - self.srv.admin('function srv_id() return 1 end') + resp = self.srv.admin(""" + function srv_id() return 1 end + return true + """) + assert_admin_success(resp) self.srv2.stop() assert_srv_id(con, 1) @@ -114,7 +135,6 @@ def test_01_contructor(self): # Verify that a bad address given at initialization leads # to an error. bad_addrs = [ - {"port": 1234}, # no host {"host": "localhost"}, # no port {"host": "localhost", "port": "1234"}, # port is str ] @@ -158,11 +178,11 @@ def test_02_discovery_bad_address(self): # Verify that a cluster discovery (that is triggered # by ping) give one or two warnings. - with warnings.catch_warnings(record=True) as ws: + with warnings.catch_warnings(record=True) as warns: con.ping() - self.assertTrue(len(ws) in (1, 2)) - for w in ws: - self.assertIs(w.category, ClusterDiscoveryWarning) + self.assertTrue(len(warns) in (1, 2)) + for warn in warns: + self.assertIs(warn.category, ClusterDiscoveryWarning) # Verify that incorrect or empty result was discarded. self.assertEqual(len(con.strategy.addrs), 1) @@ -173,7 +193,7 @@ def test_02_discovery_bad_address(self): def test_03_discovery_bad_good_addresses(self): func_name = 'bad_and_good_addresses' - retval = "{'localhost:', '%s:%d'}" % (self.host_2, self.port_2) + retval = f"{{'localhost:', '{self.host_2}:{self.port_2}'}}" self.define_custom_cluster_function(func_name, retval) con = tarantool.MeshConnection(self.host_1, self.port_1, user='test', password='test') @@ -182,10 +202,10 @@ def test_03_discovery_bad_good_addresses(self): # Verify that a cluster discovery (that is triggered # by ping) give one warning. - with warnings.catch_warnings(record=True) as ws: + with warnings.catch_warnings(record=True) as warns: con.ping() - self.assertEqual(len(ws), 1) - self.assertIs(ws[0].category, ClusterDiscoveryWarning) + self.assertEqual(len(warns), 1) + self.assertIs(warns[0].category, ClusterDiscoveryWarning) # Verify that only second address was accepted. self.assertEqual(len(con.strategy.addrs), 1) diff --git a/test/suites/test_package.py b/test/suites/test_package.py index a136c958..8d13858a 100644 --- a/test/suites/test_package.py +++ b/test/suites/test_package.py @@ -1,30 +1,34 @@ +""" +This module tests package features. +""" +# pylint: disable=missing-class-docstring,missing-function-docstring + import os import sys import unittest +import tarantool + if sys.version_info >= (3, 8): from importlib import metadata else: import importlib_metadata as metadata -import tarantool - def is_test_pure_install(): env = os.getenv("TEST_PURE_INSTALL") if env: env = env.upper() - return env == "1" or env == "TRUE" + return env in ("1", "TRUE") return False -class TestSuite_Package(unittest.TestCase): +class TestSuitePackage(unittest.TestCase): @classmethod - def setUpClass(self): + def setUpClass(cls): print(' PACKAGE '.center(70, '='), file=sys.stderr) print('-' * 70, file=sys.stderr) - def test_version(self): if is_test_pure_install(): self.assertEqual(tarantool.__version__, metadata.version('tarantool')) diff --git a/test/suites/test_pool.py b/test/suites/test_pool.py index f5e27f16..e15bcbbf 100644 --- a/test/suites/test_pool.py +++ b/test/suites/test_pool.py @@ -1,3 +1,9 @@ +""" +This module tests work with a cluster of Tarantool servers through +ConnectionPool. +""" +# pylint: disable=missing-class-docstring,missing-function-docstring,too-many-public-methods,too-many-locals,duplicate-code,bad-option-value,no-self-use + import sys import time import unittest @@ -7,7 +13,6 @@ from tarantool.error import ( ClusterConnectWarning, DatabaseError, - NetworkError, NetworkWarning, PoolTolopogyError, PoolTolopogyWarning, @@ -15,63 +20,79 @@ from .lib.skip import skip_or_run_sql_test from .lib.tarantool_server import TarantoolServer +from .utils import assert_admin_success def create_server(_id): srv = TarantoolServer() srv.script = 'test/suites/box.lua' srv.start() - srv.admin("box.schema.user.create('test', {password = 'test', " + - "if_not_exists = true})") - srv.admin("box.schema.user.grant('test', 'execute', 'universe')") - srv.admin("box.schema.space.create('test')") - srv.admin(r"box.space.test:format({" - +r" { name = 'pk', type = 'string' }," + - r" { name = 'id', type = 'number', is_nullable = true }" + - r"})") - srv.admin(r"box.space.test:create_index('pk'," + - r"{ unique = true," + - r" parts = {{field = 1, type = 'string'}}})") - srv.admin(r"box.space.test:create_index('id'," + - r"{ unique = true," + - r" parts = {{field = 2, type = 'number', is_nullable=true}}})") - srv.admin("box.schema.user.grant('test', 'read,write', 'space', 'test')") - srv.admin("json = require('json')") - - # Create srv_id function (for testing purposes). - srv.admin("function srv_id() return %s end" % _id) + resp = srv.admin(f""" + box.schema.user.create('test', {{password = 'test', if_not_exists = true}}) + box.schema.user.grant('test', 'execute', 'universe', + nil, {{if_not_exists = true}}) + box.schema.space.create('test', {{if_not_exists = true}}) + box.space.test:format({{ + {{ name = 'pk', type = 'string' }}, + {{ name = 'id', type = 'number', is_nullable = true }} + }}) + box.space.test:create_index('pk', + {{ + unique = true, + parts = {{ + {{field = 1, type = 'string'}} + }}, + if_not_exists = true, + }}) + box.space.test:create_index('id', + {{ + unique = true, + parts = {{ + {{field = 2, type = 'number', is_nullable=true}} + }}, + if_not_exists = true, + }}) + box.schema.user.grant('test', 'read,write', 'space', 'test', {{if_not_exists = true}}) + json = require('json') + + function srv_id() return {_id} end + + return true + """) + assert_admin_success(resp) return srv @unittest.skipIf(sys.platform.startswith("win"), 'Pool tests on windows platform are not supported') -class TestSuite_Pool(unittest.TestCase): +class TestSuitePool(unittest.TestCase): def set_ro(self, srv, read_only): if read_only: - req = r'box.cfg{read_only = true}' + req = r'box.cfg{read_only = true}; return true' else: - req = r'box.cfg{read_only = false}' + req = r'box.cfg{read_only = false}; return true' - srv.admin(req) + resp = srv.admin(req) + assert_admin_success(resp) def set_cluster_ro(self, read_only_list): assert len(self.servers) == len(read_only_list) - for i in range(len(self.servers)): - self.set_ro(self.servers[i], read_only_list[i]) + for i, server in enumerate(self.servers): + self.set_ro(server, read_only_list[i]) def retry(self, func, count=5, timeout=0.5): for i in range(count): try: func() - except Exception as e: + except Exception as exc: # pylint: disable=bad-option-value,broad-exception-caught,broad-except if i + 1 == count: - raise e + raise exc time.sleep(timeout) @classmethod - def setUpClass(self): + def setUpClass(cls): print(' POOL '.center(70, '='), file=sys.stderr) print('-' * 70, file=sys.stderr) @@ -79,6 +100,7 @@ def setUp(self): # Create five servers and extract helpful fields for tests. self.servers = [] self.addrs = [] + self.pool = None self.servers_count = 5 for i in range(self.servers_count): srv = create_server(i) @@ -108,8 +130,9 @@ def test_00_basic(self): def test_01_roundrobin(self): self.set_cluster_ro([False, False, True, False, True]) - RW_ports = set([str(self.addrs[0]['port']), str(self.addrs[1]['port']), str(self.addrs[3]['port'])]) - RO_ports = set([str(self.addrs[2]['port']), str(self.addrs[4]['port'])]) + rw_ports = set([str(self.addrs[0]['port']), str(self.addrs[1]['port']), + str(self.addrs[3]['port'])]) + ro_ports = set([str(self.addrs[2]['port']), str(self.addrs[4]['port'])]) all_ports = set() for addr in self.addrs: all_ports.add(str(addr['port'])) @@ -126,79 +149,79 @@ def get_port(self, mode): return resp.data[0] # Expect ANY iterate through all instances. - ANY_ports_result = set() - for i in range(len(self.servers)): - ANY_ports_result.add(get_port(self, tarantool.Mode.ANY)) + any_ports_result = set() + for _ in range(len(self.servers)): + any_ports_result.add(get_port(self, tarantool.Mode.ANY)) - self.assertSetEqual(ANY_ports_result, all_ports) + self.assertSetEqual(any_ports_result, all_ports) # Expect RW iterate through all RW instances. - RW_ports_result = set() - for i in range(len(self.servers)): - RW_ports_result.add(get_port(self, tarantool.Mode.RW)) + rw_ports_result = set() + for _ in range(len(self.servers)): + rw_ports_result.add(get_port(self, tarantool.Mode.RW)) - self.assertSetEqual(RW_ports_result, RW_ports) + self.assertSetEqual(rw_ports_result, rw_ports) # Expect RO iterate through all RO instances. - RO_ports_result = set() - for i in range(len(self.servers)): - RO_ports_result.add(get_port(self, tarantool.Mode.RO)) + ro_ports_result = set() + for _ in range(len(self.servers)): + ro_ports_result.add(get_port(self, tarantool.Mode.RO)) - self.assertSetEqual(RO_ports_result, RO_ports) + self.assertSetEqual(ro_ports_result, ro_ports) # Expect PREFER_RW iterate through all RW instances if there is at least one. - PREFER_RW_ports_result = set() - for i in range(len(self.servers)): - PREFER_RW_ports_result.add(get_port(self, tarantool.Mode.PREFER_RW)) + prefer_rw_ports_result = set() + for _ in range(len(self.servers)): + prefer_rw_ports_result.add(get_port(self, tarantool.Mode.PREFER_RW)) - self.assertSetEqual(PREFER_RW_ports_result, RW_ports) + self.assertSetEqual(prefer_rw_ports_result, rw_ports) # Expect PREFER_RO iterate through all RO instances if there is at least one. - PREFER_RO_ports_result = set() - for i in range(len(self.servers)): - PREFER_RO_ports_result.add(get_port(self, tarantool.Mode.PREFER_RO)) + prefer_ro_ports_result = set() + for _ in range(len(self.servers)): + prefer_ro_ports_result.add(get_port(self, tarantool.Mode.PREFER_RO)) - self.assertSetEqual(PREFER_RO_ports_result, RO_ports) + self.assertSetEqual(prefer_ro_ports_result, ro_ports) # Setup cluster with no RW. self.set_cluster_ro([True, True, True, True, True]) # Expect RW to fail if there are no RW. - def expect_RW_to_fail_if_there_are_no_RW(): + def expect_rw_to_fail_if_there_are_no_rw(): with self.assertRaises(PoolTolopogyError): self.pool.eval('return box.cfg.listen', mode=tarantool.Mode.RW) - self.retry(func=expect_RW_to_fail_if_there_are_no_RW) + self.retry(func=expect_rw_to_fail_if_there_are_no_rw) # Expect PREFER_RW iterate through all instances if there are no RW. - def expect_PREFER_RW_iterate_through_all_instances_if_there_are_no_RW(): - PREFER_RW_ports_result_all_ro = set() - for i in range(len(self.servers)): - PREFER_RW_ports_result_all_ro.add(get_port(self, tarantool.Mode.PREFER_RW)) + def expect_prefer_rw_iterate_through_all_instances_if_there_are_no_rw(): + prefer_rw_ports_result_all_ro = set() + for _ in range(len(self.servers)): + prefer_rw_ports_result_all_ro.add(get_port(self, tarantool.Mode.PREFER_RW)) + + self.assertSetEqual(prefer_rw_ports_result_all_ro, all_ports) - self.assertSetEqual(PREFER_RW_ports_result_all_ro, all_ports) - - self.retry(func=expect_PREFER_RW_iterate_through_all_instances_if_there_are_no_RW) + self.retry(func=expect_prefer_rw_iterate_through_all_instances_if_there_are_no_rw) # Setup cluster with no RO. self.set_cluster_ro([False, False, False, False, False]) # Expect RO to fail if there are no RO. - def expect_RO_to_fail_if_there_are_no_RO(): + def expect_ro_to_fail_if_there_are_no_ro(): with self.assertRaises(PoolTolopogyError): self.pool.eval('return box.cfg.listen', mode=tarantool.Mode.RO) - self.retry(func=expect_RO_to_fail_if_there_are_no_RO) + self.retry(func=expect_ro_to_fail_if_there_are_no_ro) # Expect PREFER_RO iterate through all instances if there are no RO. - def expect_PREFER_RO_iterate_through_all_instances_if_there_are_no_RO(): - PREFER_RO_ports_result_all_rw = set() - for i in range(len(self.servers)): - PREFER_RO_ports_result_all_rw.add(get_port(self, tarantool.Mode.PREFER_RO)) + def expect_prefer_ro_iterate_through_all_instances_if_there_are_no_ro(): + prefer_ro_ports_result_all_rw = set() + for _ in range(len(self.servers)): + prefer_ro_ports_result_all_rw.add(get_port(self, tarantool.Mode.PREFER_RO)) - self.assertSetEqual(PREFER_RO_ports_result_all_rw, all_ports) + self.assertSetEqual(prefer_ro_ports_result_all_rw, all_ports) - self.retry(func=expect_PREFER_RO_iterate_through_all_instances_if_there_are_no_RO) + self.retry(func=expect_prefer_ro_iterate_through_all_instances_if_there_are_no_ro) def test_02_exception_raise(self): self.set_cluster_ro([False, False, True, False, True]) @@ -222,8 +245,7 @@ def test_03_insert(self): self.pool.insert('test', ['test_03_insert_1', 1]), [['test_03_insert_1', 1]]) self.assertSequenceEqual( - self.pool.insert('test', ['test_03_insert_2', 2], - mode=tarantool.Mode.RW), + self.pool.insert('test', ['test_03_insert_2', 2], mode=tarantool.Mode.RW), [['test_03_insert_2', 2]]) conn_2 = tarantool.connect( @@ -294,8 +316,9 @@ def test_05_upsert(self): [['test_05_upsert', 3]]) self.assertSequenceEqual( - self.pool.upsert('test', ['test_05_upsert', 3], - [('+', 1, 1)], mode=tarantool.Mode.RW), []) + self.pool.upsert('test', ['test_05_upsert', 3], [('+', 1, 1)], + mode=tarantool.Mode.RW), + []) self.assertSequenceEqual( conn_1.select('test', 'test_05_upsert'), [['test_05_upsert', 4]]) @@ -327,8 +350,8 @@ def test_06_update(self): [['test_06_update_1', 4]]) self.assertSequenceEqual( - self.pool.update('test', ('test_06_update_2',), - [('=', 1, 10)], mode=tarantool.Mode.RW), + self.pool.update('test', ('test_06_update_2',), [('=', 1, 10)], + mode=tarantool.Mode.RW), [['test_06_update_2', 10]]) self.assertSequenceEqual( conn_4.select('test', 'test_06_update_2'), @@ -354,7 +377,7 @@ def test_07_replace(self): self.assertSequenceEqual( self.pool.replace('test', ['test_07_replace', 4], - mode=tarantool.Mode.RW), + mode=tarantool.Mode.RW), [['test_07_replace', 4]]) self.assertSequenceEqual( conn_4.select('test', 'test_07_replace'), @@ -394,22 +417,17 @@ def test_08_select(self): self.pool.select('test', 'test_08_select'), [['test_08_select', 3]]) self.assertSequenceEqual( - self.pool.select('test', ['test_08_select'], - mode=tarantool.Mode.ANY), + self.pool.select('test', ['test_08_select'], mode=tarantool.Mode.ANY), [['test_08_select', 3]]) self.assertSequenceEqual( - self.pool.select('test', 3, index='id', - mode=tarantool.Mode.RO), + self.pool.select('test', 3, index='id', mode=tarantool.Mode.RO), [['test_08_select', 3]]) self.assertSequenceEqual( - self.pool.select('test', [3], index='id', - mode=tarantool.Mode.PREFER_RW), + self.pool.select('test', [3], index='id', mode=tarantool.Mode.PREFER_RW), [['test_08_select', 3]]) def test_09_ping(self): - self.pool = tarantool.ConnectionPool(addrs=self.addrs, - user='test', - password='test') + self.pool = tarantool.ConnectionPool(addrs=self.addrs, user='test', password='test') with self.assertRaisesRegex(ValueError, "Please, specify 'mode' keyword argument"): self.pool.ping() @@ -473,7 +491,7 @@ def test_12_execute(self): resp = self.pool.execute( 'insert into "test" values (:pk, :id)', - { 'pk': 'test_12_execute_2', 'id': 2}, + {'pk': 'test_12_execute_2', 'id': 2}, mode=tarantool.Mode.RW) self.assertEqual(resp.affected_row_count, 1) self.assertEqual(resp.data, None) @@ -509,12 +527,12 @@ def test_13_failover(self): self.servers[0].stop() self.set_ro(self.servers[1], False) - def expect_RW_request_execute_on_new_master(): + def expect_rw_request_execute_on_new_master(): self.assertSequenceEqual( self.pool.eval('return box.cfg.listen', mode=tarantool.Mode.RW), - [ str(self.addrs[1]['port']) ]) + [str(self.addrs[1]['port'])]) - self.retry(func=expect_RW_request_execute_on_new_master) + self.retry(func=expect_rw_request_execute_on_new_master) def test_14_cluster_with_instances_dead_in_runtime_is_ok(self): warnings.simplefilter('ignore', category=ClusterConnectWarning) @@ -544,10 +562,10 @@ def test_15_cluster_with_dead_instances_on_start_is_ok(self): self.servers[0].start() - def ping_RW(): + def ping_rw(): self.pool.ping(mode=tarantool.Mode.RW) - self.retry(func=ping_RW) + self.retry(func=ping_rw) def test_16_is_closed(self): self.set_cluster_ro([False, False, True, False, True]) @@ -564,7 +582,7 @@ def test_16_is_closed(self): self.assertEqual(self.pool.is_closed(), True) def tearDown(self): - if hasattr(self, 'pool'): + if self.pool: self.pool.close() for srv in self.servers: diff --git a/test/suites/test_protocol.py b/test/suites/test_protocol.py index 6a548e4f..004f0ea0 100644 --- a/test/suites/test_protocol.py +++ b/test/suites/test_protocol.py @@ -1,30 +1,41 @@ +""" +This module tests connection authentication. +""" +# pylint: disable=missing-class-docstring,missing-function-docstring,protected-access,duplicate-code + import sys -import pkg_resources import unittest import uuid -import tarantool -from tarantool.utils import greeting_decode, version_id - -from .lib.tarantool_server import TarantoolServer +import pkg_resources +import tarantool from tarantool.const import ( IPROTO_FEATURE_STREAMS, IPROTO_FEATURE_TRANSACTIONS, IPROTO_FEATURE_ERROR_EXTENSION, IPROTO_FEATURE_WATCHERS, + IPROTO_FEATURE_PAGINATION, + IPROTO_FEATURE_SPACE_AND_INDEX_NAMES, + IPROTO_FEATURE_WATCH_ONCE, ) +from tarantool.error import NetworkError +from tarantool.utils import greeting_decode, version_id + +from .lib.tarantool_server import TarantoolServer +from .lib.skip import skip_or_run_iproto_basic_features_test + -class TestSuite_Protocol(unittest.TestCase): +class TestSuiteProtocol(unittest.TestCase): @classmethod - def setUpClass(self): + def setUpClass(cls): print(' PROTOCOL '.center(70, '='), file=sys.stderr) print('-' * 70, file=sys.stderr) - self.srv = TarantoolServer() - self.srv.script = 'test/suites/box.lua' - self.srv.start() - self.con = tarantool.Connection(self.srv.host, self.srv.args['primary']) - self.adm = self.srv.admin + cls.srv = TarantoolServer() + cls.srv.script = 'test/suites/box.lua' + cls.srv.start() + cls.con = tarantool.Connection(cls.srv.host, cls.srv.args['primary']) + cls.adm = cls.srv.admin def setUp(self): # prevent a remote tarantool from clean our session @@ -33,7 +44,7 @@ def setUp(self): def test_00_greeting_1_6(self): buf = "Tarantool 1.6.6 \n" + \ - "AtQnb9SAIaKazZZy9lJKvK3urtbjCEJndhRVbslSPGc= \n"; + "AtQnb9SAIaKazZZy9lJKvK3urtbjCEJndhRVbslSPGc= \n" greeting = greeting_decode(buf.encode()) self.assertEqual(greeting.version_id, version_id(1, 6, 6)) self.assertEqual(greeting.protocol, "Binary") @@ -42,7 +53,7 @@ def test_00_greeting_1_6(self): def test_01_greeting_1_6_with_tag(self): buf = "Tarantool 1.6.6-232-gcf47324 \n" + \ - "AtQnb9SAIaKazZZy9lJKvK3urtbjCEJndhRVbslSPGc= \n"; + "AtQnb9SAIaKazZZy9lJKvK3urtbjCEJndhRVbslSPGc= \n" greeting = greeting_decode(buf.encode()) self.assertEqual(greeting.version_id, version_id(1, 6, 6)) self.assertEqual(greeting.protocol, "Binary") @@ -51,7 +62,7 @@ def test_01_greeting_1_6_with_tag(self): def test_02_greeting_1_6_console(self): buf = "Tarantool 1.6.6-132-g82f5424 (Lua console) \n" + \ - "type 'help' for interactive help \n"; + "type 'help' for interactive help \n" greeting = greeting_decode(buf.encode()) self.assertEqual(greeting.version_id, version_id(1, 6, 6)) self.assertEqual(greeting.protocol, "Lua console") @@ -60,7 +71,7 @@ def test_02_greeting_1_6_console(self): def test_03_greeting_1_6_7(self): buf = "Tarantool 1.6.7 (Binary) 52dc2837-8001-48fe-bdce-c493c04599ce \n" + \ - "Z+2F+VRlyK1nKT82xQtxqEggMtkTK5RtPYf27JryRas= \n"; + "Z+2F+VRlyK1nKT82xQtxqEggMtkTK5RtPYf27JryRas= \n" greeting = greeting_decode(buf.encode()) self.assertEqual(greeting.version_id, version_id(1, 6, 7)) self.assertEqual(greeting.protocol, "Binary") @@ -85,10 +96,38 @@ def test_04_protocol(self): self.assertEqual(self.con._features[IPROTO_FEATURE_STREAMS], False) self.assertEqual(self.con._features[IPROTO_FEATURE_TRANSACTIONS], False) self.assertEqual(self.con._features[IPROTO_FEATURE_WATCHERS], False) + self.assertEqual(self.con._features[IPROTO_FEATURE_PAGINATION], False) + self.assertEqual(self.con._features[IPROTO_FEATURE_SPACE_AND_INDEX_NAMES], False) + self.assertEqual(self.con._features[IPROTO_FEATURE_WATCH_ONCE], False) - @classmethod - def tearDownClass(self): - self.con.close() - self.srv.stop() - self.srv.clean() + @skip_or_run_iproto_basic_features_test + def test_protocol_requirement(self): + try: + con = tarantool.Connection(self.srv.host, self.srv.args['primary'], + required_protocol_version=3, + required_features=[IPROTO_FEATURE_STREAMS, + IPROTO_FEATURE_TRANSACTIONS, + IPROTO_FEATURE_ERROR_EXTENSION]) + con.close() + except Exception as exc: # pylint: disable=bad-option-value,broad-exception-caught,broad-except + self.fail(f'Connection create have raised Exception: {repr(exc)}') + + def test_protocol_version_requirement_fail(self): + with self.assertRaisesRegex(NetworkError, # ConfigurationError is wrapped in NetworkError + 'protocol version 100500 is required'): + con = tarantool.Connection(self.srv.host, self.srv.args['primary'], + required_protocol_version=100500) + con.close() + def test_protocol_features_requirement_fail(self): + with self.assertRaisesRegex(NetworkError, # ConfigurationError is wrapped in NetworkError + 'Server missing protocol features with id 100500, 500100'): + con = tarantool.Connection(self.srv.host, self.srv.args['primary'], + required_features=[100500, 500100]) + con.close() + + @classmethod + def tearDownClass(cls): + cls.con.close() + cls.srv.stop() + cls.srv.clean() diff --git a/test/suites/test_push.py b/test/suites/test_push.py index 618a56f5..24a1ae7d 100644 --- a/test/suites/test_push.py +++ b/test/suites/test_push.py @@ -1,97 +1,92 @@ +""" +This module tests work with IPROTO pushed messages. +""" +# pylint: disable=missing-class-docstring,missing-function-docstring,duplicate-code + import sys import unittest import tarantool from .lib.tarantool_server import TarantoolServer +from .utils import assert_admin_success def create_server(): srv = TarantoolServer() srv.script = 'test/suites/box.lua' srv.start() - srv.admin("box.schema.user.create('test', {password = 'test', " + - "if_not_exists = true})") - srv.admin("box.schema.user.grant('test', 'read,write,execute', 'universe')") - - # Create server_function (for testing purposes). - srv.admin(""" - function server_function() - x = {0,0} - while x[1] < 3 do - x[1] = x[1] + 1 - box.session.push(x) + resp = srv.admin(""" + box.schema.user.create('test', {password = 'test', if_not_exists = true}) + box.schema.user.grant('test', 'read,write,execute', 'universe', + nil, {if_not_exists = true}) + + function server_function() + x = {0,0} + while x[1] < 3 do + x[1] = x[1] + 1 + box.session.push(x) + end + return x end - return x - end - """) - # Create tester space and on_replace trigger (for testing purposes). - srv.admin(""" - box.schema.create_space( - 'tester', { - format = { - {name = 'id', type = 'unsigned'}, - {name = 'name', type = 'string'}, - } - }) - """) - srv.admin(""" - box.space.tester:create_index( - 'primary_index', { - parts = { - {field = 1, type = 'unsigned'}, - } - }) - """) - srv.admin(""" - box.space.tester:create_index( - 'primary_index', { - parts = { - {field = 1, type = 'unsigned'}, - } - }) - """) - srv.admin(""" - function on_replace_callback() - x = {0,0} - while x[1] < 300 do - x[1] = x[1] + 100 - box.session.push(x) + box.schema.create_space( + 'tester', { + format = { + {name = 'id', type = 'unsigned'}, + {name = 'name', type = 'string'}, + } + }) + + box.space.tester:create_index( + 'primary_index', { + parts = { + {field = 1, type = 'unsigned'}, + }, + if_not_exists = true, + }) + + function on_replace_callback() + x = {0,0} + while x[1] < 300 do + x[1] = x[1] + 100 + box.session.push(x) + end end - end - """) - srv.admin(""" - box.space.tester:on_replace( - on_replace_callback - ) + + box.space.tester:on_replace( + on_replace_callback + ) + + return true """) + assert_admin_success(resp) return srv # Callback for on_push arg (for testing purposes). -def push_callback(data, on_push_ctx=[]): +def push_callback(data, on_push_ctx): data[0][1] = data[0][1] + 1 on_push_ctx.append(data) -class TestSuite_Push(unittest.TestCase): +class TestSuitePush(unittest.TestCase): @classmethod - def setUpClass(self): + def setUpClass(cls): print(' PUSH '.center(70, '='), file=sys.stderr) print('-' * 70, file=sys.stderr) # Create server and extract helpful fields for tests. - self.srv = create_server() - self.host = self.srv.host - self.port = self.srv.args['primary'] + cls.srv = create_server() + cls.host = cls.srv.host + cls.port = cls.srv.args['primary'] def setUp(self): # Open connection, connection pool and mesh connection to instance. - self.conn = tarantool.Connection(host=self.host, port=self.port, + self.conn = tarantool.Connection(host=self.host, port=self.port, user='test', password='test') - self.conn_pool = tarantool.ConnectionPool([{'host':self.host, 'port':self.port}], - user='test', password='test') - self.mesh_conn = tarantool.MeshConnection(host=self.host, port=self.port, + self.conn_pool = tarantool.ConnectionPool([{'host': self.host, 'port': self.port}], + user='test', password='test') + self.mesh_conn = tarantool.MeshConnection(host=self.host, port=self.port, user='test', password='test') push_test_cases = { @@ -199,11 +194,10 @@ def setUp(self): } def test_00_00_push_via_connection(self): - for case_name in self.push_test_cases.keys(): - with self.subTest(name=case_name): + for name, case in self.push_test_cases.items(): + with self.subTest(name=name): callback_res = [] - case = self.push_test_cases[case_name] - testing_function = getattr(self.conn, case_name) + testing_function = getattr(self.conn, name) case['input']['kwargs']['on_push_ctx'] = callback_res resp = testing_function( *case['input']['args'], @@ -214,11 +208,10 @@ def test_00_00_push_via_connection(self): self.assertEqual(resp.data[0], case['output']['resp']) def test_00_01_push_via_mesh_connection(self): - for case_name in self.push_test_cases.keys(): - with self.subTest(name=case_name): + for name, case in self.push_test_cases.items(): + with self.subTest(name=name): callback_res = [] - case = self.push_test_cases[case_name] - testing_function = getattr(self.mesh_conn, case_name) + testing_function = getattr(self.mesh_conn, name) case['input']['kwargs']['on_push_ctx'] = callback_res resp = testing_function( *case['input']['args'], @@ -229,11 +222,10 @@ def test_00_01_push_via_mesh_connection(self): self.assertEqual(resp.data[0], case['output']['resp']) def test_00_02_push_via_connection_pool(self): - for case_name in self.push_test_cases.keys(): - with self.subTest(name=case_name): + for name, case in self.push_test_cases.items(): + with self.subTest(name=name): callback_res = [] - case = self.push_test_cases[case_name] - testing_function = getattr(self.conn_pool, case_name) + testing_function = getattr(self.conn_pool, name) case['input']['kwargs']['on_push_ctx'] = callback_res resp = testing_function( *case['input']['args'], @@ -251,7 +243,7 @@ def tearDown(self): self.mesh_conn.close() @classmethod - def tearDownClass(self): + def tearDownClass(cls): # Stop instance. - self.srv.stop() - self.srv.clean() + cls.srv.stop() + cls.srv.clean() diff --git a/test/suites/test_reconnect.py b/test/suites/test_reconnect.py index fc41cb57..f7811fab 100644 --- a/test/suites/test_reconnect.py +++ b/test/suites/test_reconnect.py @@ -1,3 +1,8 @@ +""" +This module tests basic reconnect behavior. +""" +# pylint: disable=missing-class-docstring,missing-function-docstring + import sys import unittest import warnings @@ -5,13 +10,13 @@ from .lib.tarantool_server import TarantoolServer -class TestSuite_Reconnect(unittest.TestCase): +class TestSuiteReconnect(unittest.TestCase): @classmethod - def setUpClass(self): + def setUpClass(cls): print(' RECONNECT '.center(70, '='), file=sys.stderr) print('-' * 70, file=sys.stderr) - self.srv = TarantoolServer() - self.srv.script = 'test/suites/box.lua' + cls.srv = TarantoolServer() + cls.srv.script = 'test/suites/box.lua' def setUp(self): # prevent a remote tarantool from clean our session @@ -80,5 +85,5 @@ def test_03_connect_after_close(self): self.srv.stop() @classmethod - def tearDownClass(self): - self.srv.clean() + def tearDownClass(cls): + cls.srv.clean() diff --git a/test/suites/test_schema.py b/test/suites/test_schema.py index 09017b2a..c90ddbff 100644 --- a/test/suites/test_schema.py +++ b/test/suites/test_schema.py @@ -1,7 +1,18 @@ +""" +This module tests space and index schema fetch. +""" +# pylint: disable=missing-class-docstring,missing-function-docstring,fixme,too-many-public-methods,too-many-branches,too-many-statements,duplicate-code + import sys import unittest +import pkg_resources + import tarantool +from tarantool.error import NotSupportedError + from .lib.tarantool_server import TarantoolServer +from .lib.skip import skip_or_run_constraints_test +from .utils import assert_admin_success # FIXME: I'm quite sure that there is a simpler way to count @@ -16,9 +27,11 @@ def _bind(self, obj, method_name): self._obj = obj self._method_name = method_name self._saved_method = getattr(obj, method_name) + def wrapper(_, *args, **kwargs): self._call_count += 1 return self._saved_method(*args, **kwargs) + bound_wrapper = wrapper.__get__(obj.__class__, obj) setattr(obj, method_name, bound_wrapper) @@ -30,31 +43,74 @@ def call_count(self): return self._call_count -class TestSuite_Schema_Abstract(unittest.TestCase): +class TestSuiteSchemaAbstract(unittest.TestCase): # Define 'encoding' field in a concrete class. + encoding = None @classmethod - def setUpClass(self): - params = 'connection.encoding: {}'.format(repr(self.encoding)) - print(' SCHEMA ({}) '.format(params).center(70, '='), file=sys.stderr) + def setUpClass(cls): + params = f'connection.encoding: {repr(cls.encoding)}' + print(f' SCHEMA ({params}) '.center(70, '='), file=sys.stderr) print('-' * 70, file=sys.stderr) - self.srv = TarantoolServer() - self.srv.script = 'test/suites/box.lua' - self.srv.start() - self.con = tarantool.Connection(self.srv.host, self.srv.args['primary'], - encoding=self.encoding) - self.sch = self.con.schema - - # The relevant test cases mainly target Python 2, where - # a user may want to pass a string literal as a space or - # an index name and don't bother whether all symbols in it - # are ASCII. - self.unicode_space_name_literal = '∞' - self.unicode_index_name_literal = '→' - - self.unicode_space_name_u = u'∞' - self.unicode_index_name_u = u'→' - self.unicode_space_id, self.unicode_index_id = self.srv.admin(""" + cls.srv = TarantoolServer() + cls.srv.script = 'test/suites/box.lua' + cls.srv.start() + resp = cls.srv.admin(""" + box.schema.user.create('test', {password = 'test', if_not_exists = true}) + box.schema.user.grant('test', 'read,write,execute', 'universe', + nil, {if_not_exists = true}) + + function server_function() return 2+2 end + + box.schema.create_space( + 'tester', { + format = { + {name = 'id', type = 'unsigned'}, + {name = 'name', type = 'string', is_nullable = true}, + }, + if_not_exists = true, + }) + + box.space.tester:create_index( + 'primary_index', { + parts = { + {field = 1, type = 'unsigned'}, + }, + if_not_exists = true, + }) + + box.space.tester:insert({1, nil}) + + return true + """) + assert_admin_success(resp) + + cls.con = tarantool.Connection(cls.srv.host, cls.srv.args['primary'], + encoding=cls.encoding, user='test', password='test') + cls.con_schema_disable = tarantool.Connection(cls.srv.host, cls.srv.args['primary'], + encoding=cls.encoding, fetch_schema=False, + user='test', password='test') + if not sys.platform.startswith("win"): + # Schema fetch disable tests via mesh and pool connection + # are not supported on windows platform. + cls.mesh_con_schema_disable = tarantool.MeshConnection( + host=cls.srv.host, + port=cls.srv.args['primary'], + fetch_schema=False, + user='test', password='test') + cls.pool_con_schema_disable = tarantool.ConnectionPool( + [{ + 'host': cls.srv.host, + 'port': cls.srv.args['primary'] + }], + user='test', password='test', + fetch_schema=False) + cls.sch = cls.con.schema + + cls.unicode_space_name_literal = '∞' + cls.unicode_index_name_literal = '→' + + cls.unicode_space_id, cls.unicode_index_id = cls.srv.admin(""" do local space = box.schema.create_space('\\xe2\\x88\\x9e') local index = space:create_index('\\xe2\\x86\\x92') @@ -62,6 +118,36 @@ def setUpClass(self): end """) + if cls.srv.admin.tnt_version >= pkg_resources.parse_version('2.10.0'): + resp = cls.srv.admin(""" + box.schema.create_space( + 'constr_tester_1', { + format = { + { name = 'id', type = 'unsigned' }, + { name = 'payload', type = 'number' }, + } + }) + box.space.constr_tester_1:create_index('I1', { parts = {'id'} }) + + box.space.constr_tester_1:replace({1, 999}) + + box.schema.create_space( + 'constr_tester_2', { + format = { + { name = 'id', type = 'unsigned' }, + { name = 'table1_id', type = 'unsigned', + foreign_key = { fk_video = { space = 'constr_tester_1', field = 'id' } }, + }, + { name = 'payload', type = 'number' }, + } + }) + box.space.constr_tester_2:create_index('I1', { parts = {'id'} }) + box.space.constr_tester_2:create_index('I2', { parts = {'table1_id'} }) + + return true + """) + assert_admin_success(resp) + def setUp(self): # prevent a remote tarantool from clean our session if self.srv.is_started(): @@ -88,47 +174,33 @@ def fetch_count(self): def verify_unicode_space(self, space): self.assertEqual(space.sid, self.unicode_space_id) - self.assertEqual(space.name, self.unicode_space_name_u) + self.assertEqual(space.name, self.unicode_space_name_literal) self.assertEqual(space.arity, 1) def verify_unicode_index(self, index): - self.assertEqual(index.space.name, self.unicode_space_name_u) + self.assertEqual(index.space.name, self.unicode_space_name_literal) self.assertEqual(index.iid, self.unicode_index_id) - self.assertEqual(index.name, self.unicode_index_name_u) + self.assertEqual(index.name, self.unicode_index_name_literal) self.assertEqual(len(index.parts), 1) - def test_00_authenticate(self): - self.assertIsNone(self.srv.admin("box.schema.user.create('test', { password = 'test' })")) - self.assertIsNone(self.srv.admin("box.schema.user.grant('test', 'read,write', 'space', '_space')")) - self.assertIsNone(self.srv.admin("box.schema.user.grant('test', 'read,write', 'space', '_index')")) - self.assertEqual(self.con.authenticate('test', 'test')._data, None) - def test_01_space_bad(self): - with self.assertRaisesRegex(tarantool.SchemaError, - 'There\'s no space.*'): + with self.assertRaisesRegex(tarantool.SchemaError, 'There\'s no space.*'): self.sch.get_space(0) - with self.assertRaisesRegex(tarantool.SchemaError, - 'There\'s no space.*'): + with self.assertRaisesRegex(tarantool.SchemaError, 'There\'s no space.*'): self.sch.get_space(0) - with self.assertRaisesRegex(tarantool.SchemaError, - 'There\'s no space.*'): + with self.assertRaisesRegex(tarantool.SchemaError, 'There\'s no space.*'): self.sch.get_space('bad_name') def test_02_index_bad(self): - with self.assertRaisesRegex(tarantool.SchemaError, - 'There\'s no space.*'): + with self.assertRaisesRegex(tarantool.SchemaError, 'There\'s no space.*'): self.sch.get_index(0, 'primary') - with self.assertRaisesRegex(tarantool.SchemaError, - 'There\'s no space.*'): + with self.assertRaisesRegex(tarantool.SchemaError, 'There\'s no space.*'): self.sch.get_index('bad_space', 'primary') - with self.assertRaisesRegex(tarantool.SchemaError, - 'There\'s no index.*'): + with self.assertRaisesRegex(tarantool.SchemaError, 'There\'s no index.*'): self.sch.get_index(280, 'bad_index') - with self.assertRaisesRegex(tarantool.SchemaError, - 'There\'s no index.*'): + with self.assertRaisesRegex(tarantool.SchemaError, 'There\'s no index.*'): self.sch.get_index(280, 'bad_index') - with self.assertRaisesRegex(tarantool.SchemaError, - 'There\'s no index.*'): + with self.assertRaisesRegex(tarantool.SchemaError, 'There\'s no index.*'): self.sch.get_index(280, 3) def test_03_01_space_name__(self): @@ -329,8 +401,8 @@ def test_06_index_cached(self): (self.unicode_space_id, self.unicode_index_name_literal), (self.unicode_space_id, self.unicode_index_id), ) - for s, i in cases: - index = self.sch.get_index(s, i) + for space, index_id in cases: + index = self.sch.get_index(space, index_id) self.verify_unicode_index(index) # Verify that no schema fetches occurs. @@ -341,16 +413,228 @@ def test_07_schema_version_update(self): self.srv.admin("box.schema.create_space('ttt22')") self.assertEqual(len(self.con.select('_space')), _space_len + 1) - @classmethod - def tearDownClass(self): - self.con.close() - self.srv.stop() - self.srv.clean() - + # For schema fetch disable testing purposes. + testing_methods = { + 'unavailable': { + 'replace': { + 'input': ['tester', (1, None)], + 'output': [[1, None]], + }, + 'delete': { + 'input': ['tester', 1], + 'output': [[1, None]], + }, + 'insert': { + 'input': ['tester', (1, None)], + 'output': [[1, None]], + }, + 'upsert': { + 'input': ['tester', (1, None), []], + 'output': [], + }, + 'update': { + 'input': ['tester', 1, []], + 'output': [[1, None]], + }, + 'select': { + 'input': ['tester', 1], + 'output': [[1, None]], + }, + 'space': { + 'input': ['tester'], + }, + }, + 'available': { + # CRUD methods are also tested with the fetch_schema=False opt, + # see the test_crud.py file. + 'call': { + 'input': ['server_function'], + 'output': [4], + }, + 'eval': { + 'input': ['return 2+2'], + 'output': [4], + }, + 'ping': { + 'input': [], + }, + }, + } + + def _run_test_schema_fetch_disable(self, con, mode=None): + # Enable SQL test case for tarantool 2.* and higher. + if int(str(self.srv.admin.tnt_version)[0]) > 1: + if self.srv.admin.tnt_version >= pkg_resources.parse_version('2.11.0'): + # SEQSCAN keyword is explicitly allowing to use seqscan + # https://github.com/tarantool/tarantool/commit/77648827326ad268ec0ffbcd620c2371b65ef2b4 + # It was introduced in 2.11.0-rc1. If compat.sql_seq_scan_default + # set to "new" (default value since 3.0), returns error + # if trying to scan without keyword. + self.testing_methods['available']['execute'] = { + 'input': ['SELECT * FROM SEQSCAN "tester"'], + 'output': [[1, None]], + } + else: + self.testing_methods['available']['execute'] = { + 'input': ['SELECT * FROM "tester"'], + 'output': [[1, None]], + } + + # Testing the schemaless connection with methods + # that should NOT be available. + if mode is not None: + for addr in con.pool.keys(): + self.assertEqual(con.pool[addr].conn.schema_version, 0) + self.assertEqual(con.pool[addr].conn.schema, None) + else: + self.assertEqual(con.schema_version, 0) + self.assertEqual(con.schema, None) + for method_case in self.testing_methods['unavailable']: + with self.subTest(name=method_case): + if isinstance(con, tarantool.ConnectionPool) and method_case == 'space': + continue + testing_function = getattr(con, method_case) + try: + if mode is not None: + _ = testing_function( + *self.testing_methods['unavailable'][method_case]['input'], + mode=mode) + else: + _ = testing_function( + *self.testing_methods['unavailable'][method_case]['input']) + except NotSupportedError as exc: + self.assertEqual(exc.message, 'This method is not available in ' + 'connection opened with fetch_schema=False') + # Testing the schemaless connection with methods + # that should be available. + for method_case in self.testing_methods['available']: + with self.subTest(name=method_case): + testing_function = getattr(con, method_case) + if mode is not None: + resp = testing_function( + *self.testing_methods['available'][method_case]['input'], + mode=mode) + else: + resp = testing_function( + *self.testing_methods['available'][method_case]['input']) + if method_case == 'ping': + self.assertEqual(isinstance(resp, float), True) + else: + self.assertEqual( + resp.data, + self.testing_methods['available'][method_case]['output']) + + # Turning the same connection into schemaful. + if mode is not None: + for addr in con.pool.keys(): + con.pool[addr].conn.update_schema(con.pool[addr].conn.schema_version) + else: + con.update_schema(con.schema_version) + + # Testing the schemaful connection with methods + # that should NOW be available. + for method_case in self.testing_methods['unavailable']: + with self.subTest(name=method_case): + if isinstance(con, tarantool.ConnectionPool) and method_case == 'space': + continue + testing_function = getattr(con, method_case) + if mode is not None: + resp = testing_function( + *self.testing_methods['unavailable'][method_case]['input'], + mode=mode) + else: + resp = testing_function( + *self.testing_methods['unavailable'][method_case]['input']) + if method_case == 'space': + self.assertEqual(isinstance(resp, tarantool.space.Space), True) + else: + self.assertEqual( + resp.data, + self.testing_methods['unavailable'][method_case]['output']) + # Testing the schemaful connection with methods + # that should have remained available. + for method_case in self.testing_methods['available']: + with self.subTest(name=method_case): + testing_function = getattr(con, method_case) + if mode is not None: + resp = testing_function( + *self.testing_methods['available'][method_case]['input'], + mode=mode) + else: + resp = testing_function( + *self.testing_methods['available'][method_case]['input']) + if method_case == 'ping': + self.assertEqual(isinstance(resp, float), True) + else: + self.assertEqual( + resp.data, + self.testing_methods['available'][method_case]['output']) + if mode is not None: + self.assertNotEqual(con.pool[addr].conn.schema_version, 1) + self.assertNotEqual(con.pool[addr].conn.schema, None) + else: + self.assertNotEqual(con.schema_version, 1) + self.assertNotEqual(con.schema, None) + + def test_08_schema_fetch_disable_via_connection(self): + self._run_test_schema_fetch_disable(self.con_schema_disable) + + @unittest.skipIf( + sys.platform.startswith("win"), + 'Schema fetch disable tests via mesh connection on windows platform are not supported') + def test_08_schema_fetch_disable_via_mesh_connection(self): + self._run_test_schema_fetch_disable(self.mesh_con_schema_disable) + + @unittest.skipIf( + sys.platform.startswith("win"), + 'Schema fetch disable tests via connection pool on windows platform are not supported') + def test_08_schema_fetch_disable_via_connection_pool(self): + self._run_test_schema_fetch_disable(self.pool_con_schema_disable, + mode=tarantool.Mode.ANY) + + @skip_or_run_constraints_test + def test_09_foreign_key_info_fetched_to_schema(self): + self.assertIn('foreign_key', self.sch.get_space('constr_tester_2').format['table1_id']) + + @skip_or_run_constraints_test + def test_10_foreign_key_valid_replace(self): + self.assertSequenceEqual( + self.con.replace('constr_tester_2', [1, 1, 623]), + [[1, 1, 623]]) + + @skip_or_run_constraints_test + def test_11_foreign_key_invalid_replace(self): + with self.assertRaisesRegex(tarantool.DatabaseError, + 'foreign tuple was not found'): + self.con.replace('constr_tester_2', [2, 999, 623]) -class TestSuite_Schema_UnicodeConnection(TestSuite_Schema_Abstract): + @classmethod + def tearDownClass(cls): + # We need to drop spaces with foreign keys with predetermined order, + # otherwise remote server clean() will fail to clean up resources. + if cls.srv.admin.tnt_version >= pkg_resources.parse_version('2.10.0'): + resp = cls.srv.admin(""" + box.space.constr_tester_2:drop() + box.space.constr_tester_1:drop() + + return true + """) + assert_admin_success(resp) + + cls.con.close() + cls.con_schema_disable.close() + if not sys.platform.startswith("win"): + # Schema fetch disable tests via mesh and pool connection + # are not supported on windows platform. + cls.mesh_con_schema_disable.close() + cls.pool_con_schema_disable.close() + cls.srv.stop() + cls.srv.clean() + + +class TestSuiteSchemaUnicodeConnection(TestSuiteSchemaAbstract): encoding = 'utf-8' -class TestSuite_Schema_BinaryConnection(TestSuite_Schema_Abstract): +class TestSuiteSchemaBinaryConnection(TestSuiteSchemaAbstract): encoding = None diff --git a/test/suites/test_socket_fd.py b/test/suites/test_socket_fd.py new file mode 100644 index 00000000..5cf94777 --- /dev/null +++ b/test/suites/test_socket_fd.py @@ -0,0 +1,200 @@ +""" +This module tests work with connection over socket fd. +""" +import os.path +# pylint: disable=missing-class-docstring,missing-function-docstring + +import socket +import sys +import unittest + +import tarantool +from .lib.skip import skip_or_run_box_session_new_tests +from .lib.tarantool_server import TarantoolServer, find_port +from .utils import assert_admin_success + + +def find_python(): + for _dir in os.environ["PATH"].split(os.pathsep): + exe = os.path.join(_dir, "python") + if os.access(exe, os.X_OK): + return os.path.abspath(exe) + raise RuntimeError("Can't find python executable in " + os.environ["PATH"]) + + +class TestSuiteSocketFD(unittest.TestCase): + EVAL_USER = "return box.session.user()" + + @classmethod + def setUpClass(cls): + print(' SOCKET FD '.center(70, '='), file=sys.stderr) + print('-' * 70, file=sys.stderr) + + cls.srv = TarantoolServer() + cls.srv.script = 'test/suites/box.lua' + cls.srv.start() + cls.tcp_port = find_port() + + # Start tcp server to test work with blocking sockets. + # pylint: disable=consider-using-f-string + resp = cls.srv.admin(""" + local socket = require('socket') + + box.cfg{} + box.schema.user.create('test', {password = 'test', if_not_exists = true}) + box.schema.user.grant('test', 'read,write,execute,create', 'universe', + nil, {if_not_exists = true}) + box.schema.user.grant('guest', 'execute', 'universe', + nil, {if_not_exists = true}) + + socket.tcp_server('0.0.0.0', %d, function(s) + if not s:nonblock(true) then + s:close() + return + end + box.session.new({ + type = 'binary', + fd = s:fd(), + user = 'test', + }) + s:detach() + end) + + box.schema.create_space('test', { + format = {{type='unsigned', name='id'}}, + if_not_exists = true, + }) + box.space.test:create_index('primary') + + return true + """ % cls.tcp_port) + assert_admin_success(resp) + + @skip_or_run_box_session_new_tests + def setUp(self): + # Prevent a remote tarantool from clean our session. + if self.srv.is_started(): + self.srv.touch_lock() + + def _get_tt_sock(self): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect((self.srv.host, self.tcp_port)) + return sock + + def test_01_incorrect_params(self): + cases = { + "host and socket_fd": { + "args": {"host": "123", "socket_fd": 3}, + "msg": "specifying both socket_fd and host/port is not allowed", + }, + "port and socket_fd": { + "args": {"port": 14, "socket_fd": 3}, + "msg": "specifying both socket_fd and host/port is not allowed", + }, + "empty": { + "args": {}, + "msg": r"host/port.* port.* or socket_fd", + }, + "only host": { + "args": {"host": "localhost"}, + "msg": "when specifying host, it is also necessary to specify port", + }, + } + + for name, case in cases.items(): + with self.subTest(msg=name): + with self.assertRaisesRegex(tarantool.Error, case["msg"]): + tarantool.Connection(**case["args"]) + + def test_02_socket_fd_connect(self): + sock = self._get_tt_sock() + conn = tarantool.connect(None, None, socket_fd=sock.fileno()) + sock.detach() + try: + self.assertSequenceEqual(conn.eval(self.EVAL_USER), ["test"]) + finally: + conn.close() + + def test_03_socket_fd_re_auth(self): + sock = self._get_tt_sock() + conn = tarantool.connect(None, None, socket_fd=sock.fileno(), user="guest") + sock.detach() + try: + self.assertSequenceEqual(conn.eval(self.EVAL_USER), ["guest"]) + finally: + conn.close() + + @unittest.skipIf(sys.platform.startswith("win"), + "Skip on Windows since it uses remote server") + def test_04_tarantool_made_socket(self): + python_exe = find_python() + cwd = os.getcwd() + side_script_path = os.path.join(cwd, "test", "suites", "sidecar.py") + + # pylint: disable=consider-using-f-string + ret_code, err = self.srv.admin(""" + local socket = require('socket') + local popen = require('popen') + local os = require('os') + local s1, s2 = socket.socketpair('AF_UNIX', 'SOCK_STREAM', 0) + + --[[ Tell sidecar which fd use to connect. --]] + os.setenv('SOCKET_FD', tostring(s2:fd())) + + --[[ Tell sidecar where find `tarantool` module. --]] + os.setenv('PYTHONPATH', (os.getenv('PYTHONPATH') or '') .. ':' .. '%s') + + box.session.new({ + type = 'binary', + fd = s1:fd(), + user = 'test', + }) + s1:detach() + + local ph, err = popen.new({'%s', '%s'}, { + stdout = popen.opts.PIPE, + stderr = popen.opts.PIPE, + inherit_fds = {s2:fd()}, + }) + + if err ~= nil then + return 1, err + end + + ph:wait() + + local status_code = ph:info().status.exit_code + local stderr = ph:read({stderr=true}):rstrip() + return status_code, stderr + """ % (cwd, python_exe, side_script_path)) + self.assertIsNone(err, err) + self.assertEqual(ret_code, 0) + + def test_05_socket_fd_pool(self): + sock = self._get_tt_sock() + pool = tarantool.ConnectionPool( + addrs=[{'host': None, 'port': None, 'socket_fd': sock.fileno()}] + ) + sock.detach() + try: + self.assertSequenceEqual(pool.eval(self.EVAL_USER, mode=tarantool.Mode.ANY), ["test"]) + finally: + pool.close() + + def test_06_socket_fd_mesh(self): + sock = self._get_tt_sock() + mesh = tarantool.MeshConnection( + host=None, + port=None, + socket_fd=sock.fileno() + ) + sock.detach() + try: + self.assertSequenceEqual(mesh.eval(self.EVAL_USER), ["test"]) + finally: + mesh.close() + + @classmethod + def tearDownClass(cls): + cls.srv.stop() + cls.srv.clean() diff --git a/test/suites/test_ssl.py b/test/suites/test_ssl.py index 36e172de..b7d1da97 100644 --- a/test/suites/test_ssl.py +++ b/test/suites/test_ssl.py @@ -1,3 +1,8 @@ +""" +This module tests connection through SSL. +""" +# pylint: disable=missing-class-docstring,missing-function-docstring,too-many-branches,too-many-locals,bad-option-value,no-self-use + import os import sys import unittest @@ -24,11 +29,13 @@ def is_test_ssl(): env = os.getenv("TEST_TNT_SSL") if env: env = env.upper() - return env == "1" or env == "TRUE" + return env in ("1", "TRUE") return False class SslTestCase: + # pylint: disable=too-few-public-methods,too-many-instance-attributes,too-many-arguments,too-many-positional-arguments + def __init__(self, name="", ok=False, @@ -69,37 +76,38 @@ def __init__(self, self.client_password_file = client_password_file self.client_auth_type = client_auth_type + @unittest.skipIf(not is_test_ssl(), "TEST_TNT_SSL is not set.") -class TestSuite_Ssl(unittest.TestCase): +class TestSuiteSsl(unittest.TestCase): @classmethod - def setUpClass(self): + def setUpClass(cls): print(' SSL '.center(70, '='), file=sys.stderr) print('-' * 70, file=sys.stderr) test_suites_dir = os.path.dirname(__file__) test_data_dir = os.path.join(test_suites_dir, "..", "data") - self.cert_file = os.path.join(test_data_dir, "localhost.crt") - self.invalidhost_cert_file = os.path.join(test_data_dir, - "invalidhost.crt") - self.key_file = os.path.join(test_data_dir, "localhost.key") - self.key_enc_file = os.path.join(test_data_dir, "localhost.enc.key") - self.ca_file = os.path.join(test_data_dir, "ca.crt") - self.empty_file = os.path.join(test_data_dir, "empty") - self.password = "mysslpassword" - self.invalid_password = "notmysslpassword" - self.password_file = os.path.join(test_data_dir, "passwords") - self.invalid_password_file = os.path.join(test_data_dir, "invalidpasswords") - self.invalid_file = "any_invalid_path" + cls.cert_file = os.path.join(test_data_dir, "localhost.crt") + cls.invalidhost_cert_file = os.path.join(test_data_dir, + "invalidhost.crt") + cls.key_file = os.path.join(test_data_dir, "localhost.key") + cls.key_enc_file = os.path.join(test_data_dir, "localhost.enc.key") + cls.ca_file = os.path.join(test_data_dir, "ca.crt") + cls.empty_file = os.path.join(test_data_dir, "empty") + cls.password = "mysslpassword" + cls.invalid_password = "notmysslpassword" + cls.password_file = os.path.join(test_data_dir, "passwords") + cls.invalid_password_file = os.path.join(test_data_dir, "invalidpasswords") + cls.invalid_file = "any_invalid_path" # Extract the version for skips. - self.tnt_version = None - self.srv = TarantoolServer() - self.srv.script = 'test/suites/box.lua' - self.srv.start() - fetch_tarantool_version(self) - self.srv.stop() - self.srv.clean() - self.srv = None + cls.tnt_version = None + cls.srv = TarantoolServer() + cls.srv.script = 'test/suites/box.lua' + cls.srv.start() + fetch_tarantool_version(cls) + cls.srv.stop() + cls.srv.clean() + cls.srv = None def stop_srv(self, srv): if srv: @@ -450,13 +458,13 @@ def test_single(self): for t in testcases: with self.subTest(msg=t.name): if t.server_password is not None \ - or t.server_password_file is not None \ - or t.client_password is not None \ - or t.server_password_file is not None: + or t.server_password_file is not None \ + or t.client_password is not None \ + or t.server_password_file is not None: skip_or_run_ssl_password_test_call(self) if t.server_auth_type is not None \ - or t.client_auth_type is not None: + or t.client_auth_type is not None: skip_or_run_auth_type_test_call(self) srv = TarantoolServer( @@ -567,18 +575,18 @@ def test_pool(self): cnt = 5 with self.subTest(msg=t.name): if t.server_password is not None \ - or t.server_password_file is not None \ - or t.client_password is not None \ - or t.server_password_file is not None: + or t.server_password_file is not None \ + or t.client_password is not None \ + or t.server_password_file is not None: skip_or_run_ssl_password_test_call(self) if t.server_auth_type is not None \ - or t.client_auth_type is not None: + or t.client_auth_type is not None: skip_or_run_auth_type_test_call(self) addrs = [] servers = [] - for i in range(cnt): + for _ in range(cnt): srv = TarantoolServer( transport=t.server_transport, ssl_key_file=t.server_key_file, @@ -692,13 +700,13 @@ def test_mesh(self): cnt = 5 with self.subTest(msg=t.name): if t.server_password is not None \ - or t.server_password_file is not None \ - or t.client_password is not None \ - or t.server_password_file is not None: + or t.server_password_file is not None \ + or t.client_password is not None \ + or t.server_password_file is not None: skip_or_run_ssl_password_test_call(self) if t.server_auth_type is not None \ - or t.client_auth_type is not None: + or t.client_auth_type is not None: skip_or_run_auth_type_test_call(self) addrs = [] @@ -721,7 +729,7 @@ def test_mesh(self): srv.admin(""" box.schema.user.grant('test', 'execute,read,write', 'universe') """.replace('\n', ' ')) - srv.admin("function srv_id() return %s end" % i) + srv.admin(f"function srv_id() return {i} end") servers.append(srv) addr = { 'host': srv.host, diff --git a/test/suites/test_uuid.py b/test/suites/test_uuid.py index f81f1117..90943fcb 100644 --- a/test/suites/test_uuid.py +++ b/test/suites/test_uuid.py @@ -1,58 +1,77 @@ +""" +This module tests work with UUID type. +""" +# pylint: disable=missing-class-docstring,missing-function-docstring,duplicate-code + import sys import unittest import uuid + +import pkg_resources import msgpack -import warnings -import tarantool +import tarantool from tarantool.msgpack_ext.packer import default as packer_default from tarantool.msgpack_ext.unpacker import ext_hook as unpacker_ext_hook from .lib.tarantool_server import TarantoolServer -from .lib.skip import skip_or_run_UUID_test -from tarantool.error import MsgpackError, MsgpackWarning +from .lib.skip import skip_or_run_uuid_test +from .utils import assert_admin_success + -class TestSuite_UUID(unittest.TestCase): +class TestSuiteUUID(unittest.TestCase): @classmethod - def setUpClass(self): + def setUpClass(cls): print(' UUID EXT TYPE '.center(70, '='), file=sys.stderr) print('-' * 70, file=sys.stderr) - self.srv = TarantoolServer() - self.srv.script = 'test/suites/box.lua' - self.srv.start() + cls.srv = TarantoolServer() + cls.srv.script = 'test/suites/box.lua' + cls.srv.start() - self.adm = self.srv.admin - self.adm(r""" + cls.adm = cls.srv.admin + resp = cls.adm(""" _, uuid = pcall(require, 'uuid') - box.schema.space.create('test') + box.schema.space.create('test', {if_not_exists = true}) box.space['test']:create_index('primary', { type = 'tree', parts = {1, 'string'}, - unique = true}) + unique = true, + if_not_exists = true}) - pcall(function() - box.schema.space.create('test_pk') + box.schema.user.create('test', {password = 'test', if_not_exists = true}) + box.schema.user.grant('test', 'read,write,execute', 'universe', + nil, {if_not_exists = true}) + + return true + """) + assert_admin_success(resp) + + if cls.srv.admin.tnt_version >= pkg_resources.parse_version('2.4.1'): + resp = cls.adm(""" + box.schema.space.create('test_pk', {if_not_exists = true}) box.space['test_pk']:create_index('primary', { type = 'tree', parts = {1, 'uuid'}, - unique = true}) - end) - - box.schema.user.create('test', {password = 'test', if_not_exists = true}) - box.schema.user.grant('test', 'read,write,execute', 'universe') - """) + unique = true, + if_not_exists = true}) + return true + """) + assert_admin_success(resp) - self.con = tarantool.Connection(self.srv.host, self.srv.args['primary'], - user='test', password='test') + cls.con = tarantool.Connection(cls.srv.host, cls.srv.args['primary'], + user='test', password='test') def setUp(self): # prevent a remote tarantool from clean our session if self.srv.is_started(): self.srv.touch_lock() - self.adm("box.space['test']:truncate()") - + resp = self.adm(""" + box.space['test']:truncate() + return true + """) + assert_admin_success(resp) cases = { 'uuid_1': { @@ -83,38 +102,30 @@ def setUp(self): } def test_msgpack_decode(self): - for name in self.cases.keys(): + for name, case in self.cases.items(): with self.subTest(msg=name): - case = self.cases[name] - self.assertEqual(unpacker_ext_hook(2, case['msgpack']), case['python']) - @skip_or_run_UUID_test + @skip_or_run_uuid_test def test_tarantool_decode(self): - for name in self.cases.keys(): + for name, case in self.cases.items(): with self.subTest(msg=name): - case = self.cases[name] - self.adm(f"box.space['test']:replace{{'{name}', {case['tarantool']}}}") self.assertSequenceEqual(self.con.select('test', name), [[name, case['python']]]) def test_msgpack_encode(self): - for name in self.cases.keys(): + for name, case in self.cases.items(): with self.subTest(msg=name): - case = self.cases[name] - self.assertEqual(packer_default(case['python']), msgpack.ExtType(code=2, data=case['msgpack'])) - @skip_or_run_UUID_test + @skip_or_run_uuid_test def test_tarantool_encode(self): - for name in self.cases.keys(): + for name, case in self.cases.items(): with self.subTest(msg=name): - case = self.cases[name] - self.con.insert('test', [name, case['python']]) lua_eval = f""" @@ -132,17 +143,15 @@ def test_tarantool_encode(self): self.assertSequenceEqual(self.con.eval(lua_eval), [True]) - - @skip_or_run_UUID_test + @skip_or_run_uuid_test def test_primary_key(self): data = [uuid.UUID('ae28d4f6-076c-49dd-8227-7f9fae9592d0'), 'content'] self.assertSequenceEqual(self.con.insert('test_pk', data), [data]) self.assertSequenceEqual(self.con.select('test_pk', data[0]), [data]) - @classmethod - def tearDownClass(self): - self.con.close() - self.srv.stop() - self.srv.clean() + def tearDownClass(cls): + cls.con.close() + cls.srv.stop() + cls.srv.clean() diff --git a/test/suites/utils.py b/test/suites/utils.py new file mode 100644 index 00000000..65e68b4c --- /dev/null +++ b/test/suites/utils.py @@ -0,0 +1,16 @@ +""" +Various test utilities. +""" + + +def assert_admin_success(resp): + """ + Util to assert admin text request response. + It is expected that request last line is `return true`. + If something went wrong on executing, Tarantool throws an error + which would be a part of return values. + """ + + assert isinstance(resp, list), f'got unexpected resp type: {type(resp)}' + assert len(resp) > 0, 'got unexpected empty resp' + assert resp[0] is True, f'got unexpected resp: {resp}'