From 04b857ce5a6ef13325d764c5bc5145fd248c5b30 Mon Sep 17 00:00:00 2001 From: Pranshu Srivastava <37413698+PranshuSrivastava@users.noreply.github.com> Date: Tue, 27 Feb 2024 13:25:41 +0530 Subject: [PATCH 01/18] fix: shortened the test file and added README.md (#7) * fix: shortened the test file and added README.md Signed-off-by: Pranshu Srivastava * fix: calculating coverage from terminal instead of library Signed-off-by: Pranshu Srivastava * fix: made changes according to the comments Signed-off-by: Pranshu Srivastava * fix: typose Signed-off-by: Pranshu Srivastava --------- Signed-off-by: Pranshu Srivastava --- README.md | 80 +++++- __init__.py | 1 + keploy/__init__.py | 5 - keploy/client.py | 0 keploy/constants.py | 7 - keploy/contrib/flask/__init__.py | 62 ---- keploy/contrib/flask/utils.py | 20 -- keploy/keploy.py | 269 ------------------ keploy/mode.py | 21 -- keploy/models.py | 179 ------------ keploy/utils.py | 32 --- keploy/v2/__init__.py | 0 keploy/setup.py => setup.py | 16 +- .../v2/keployCLI.py => src/keploy/Keploy.py | 87 +++--- src/keploy/__init__.py | 1 + 15 files changed, 139 insertions(+), 641 deletions(-) create mode 100644 __init__.py delete mode 100644 keploy/__init__.py delete mode 100644 keploy/client.py delete mode 100644 keploy/constants.py delete mode 100644 keploy/contrib/flask/__init__.py delete mode 100644 keploy/contrib/flask/utils.py delete mode 100644 keploy/keploy.py delete mode 100644 keploy/mode.py delete mode 100644 keploy/models.py delete mode 100644 keploy/utils.py delete mode 100644 keploy/v2/__init__.py rename keploy/setup.py => setup.py (58%) rename keploy/v2/keployCLI.py => src/keploy/Keploy.py (62%) create mode 100644 src/keploy/__init__.py diff --git a/README.md b/README.md index eeabe3c..25d7bf2 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,81 @@ -[![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen?logo=github)](CODE_OF_CONDUCT.md) +[![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen?logo=github)](CODE_OF_CONDUCT.md) [![Slack](.github/slack.svg)](https://join.slack.com/t/keploy/shared_invite/zt-12rfbvc01-o54cOG0X1G6eVJTuI_orSA) [![License](.github/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) -# Keploy -[Keploy](https://keploy.io) is a no-code testing platform that generates tests from API calls. +**Note** :- Issue Creation is disabled on this Repository, please visit [here](https://github.com/keploy/keploy/issues/new/choose) to submit issue. +# Keploy Python-SDK +This is the client SDK for the [Keploy](https://github.com/keploy/keploy) testing platform. With the Python SDK, you can test both your existing unit test cases in Pytest and create new end-to-end test cases for your applications. +The **HTTP mocks/stubs and tests are the same format** and inter-exchangeable. + +## Contents +1. [Installation](#installation) +2. [Usage](#usage) +3. [Community support](#community-support) + +## Installation +1. First you need to install [Python(version 3 and above)](https://www.python.org/downloads/) + +2. Install the Python-SDK and also Python's coverage library via pip. + +```bash +pip install keploy coverage +``` + +3. Install Keploy from [here](https://github.com/keploy/keploy?tab=readme-ov-file#-quick-installation) + +## Usage +Keploy simplifies the testing process by seamlessly generating end-to-end test cases without the need to write unit test files and manage mocks/stubs. + +Add a test file with the following code to the directory with all your existing tests. This will help us to get the coverage of Keploy's API tests along with the other unit tests. We can call this `test_keploy.py` + +```python +from Keploy import run +def test_keploy(): + run("python3 -m coverage run --data-file=.coverage_data.keploy ") +``` + +> Note: If you face any problems with running the coverage library, you can refer to the documentation for the same [here](https://coverage.readthedocs.io/en/7.4.2/cmd.html#execution-coverage-run) + +To ignore the coverage of python libraries which are included in the report by default, you can create a .coveragerc file in the directory where you will ignore the /usr/ directory(only for Linux users). The contents of the file will be as follows: + +```bash +[run] +omit = + /usr/* +``` + +Before starting your application, make sure that the debug mode is set to False in your application, for the coverage library to work properly. + +Now to run this testcase along with your another unit testcases, you can run the command below: + +```bash +keploy test -c "python3 -m coverage run -p --data-file=.coverage.unit -m pytest test_keploy.py " --delay 10 --coverage +``` + +Now, to combine the coverage from the unit tests, and Keploy's API tests, we can use the command below: + +```bash +python3 -m coverage combine +``` + +Make sure to run this command before starting a new test run to avoid getting multiple coverage files. + +Finally, to generate the coverage report for the test run, you can run: + +```bash +python3 -m coverage report +``` + +and if you want the coverage in an html file, you can run: + +```bash +python3 -m coverage html +``` + +Hooray🎉! You've sucessfully got the coverage of your Keploy recorded api tests using Pytest. ## Community support -We'd love to collaborate with you to make Keploy great. To get started: +We'd love to collaborate with you to make Keploy.io great. To get started: * [Slack](https://join.slack.com/t/keploy/shared_invite/zt-12rfbvc01-o54cOG0X1G6eVJTuI_orSA) - Discussions with the community and the team. -* [GitHub](https://github.com/keploy/keploy/issues) - For bug reports and feature requests. - +* [GitHub](https://github.com/keploy/keploy/issues) - For bug reports and feature requests. \ No newline at end of file diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..6daf22c --- /dev/null +++ b/__init__.py @@ -0,0 +1 @@ +from .Keploy import run \ No newline at end of file diff --git a/keploy/__init__.py b/keploy/__init__.py deleted file mode 100644 index 095b5ac..0000000 --- a/keploy/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .keploy import Keploy -from .models import Dependency, AppConfig, ServerConfig, FilterConfig, Config, TestCase, TestCaseRequest, TestReq, HttpReq, HttpResp -from .contrib.flask import KFlask -from .mode import setMode, getMode -from .utils import capture_test diff --git a/keploy/client.py b/keploy/client.py deleted file mode 100644 index e69de29..0000000 diff --git a/keploy/constants.py b/keploy/constants.py deleted file mode 100644 index 692fc89..0000000 --- a/keploy/constants.py +++ /dev/null @@ -1,7 +0,0 @@ -MODE_RECORD = "record" -MODE_TEST = "test" -MODE_OFF = "off" -USE_HTTPS = 'https' -USE_HTTP = 'http' -ALLOWED_DEPENDENCY_TYPES = ['NO_SQL_DB', 'SQL_DB', 'GRPC', 'HTTP_CLIENT'] -ALLOWED_METHODS = ['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'HEAD', 'OPTIONS', 'TRACE'] \ No newline at end of file diff --git a/keploy/contrib/flask/__init__.py b/keploy/contrib/flask/__init__.py deleted file mode 100644 index 3dcc9c5..0000000 --- a/keploy/contrib/flask/__init__.py +++ /dev/null @@ -1,62 +0,0 @@ -# BSD 3-Clause License - -import copy -import io -from typing import Any -from flask import Flask, request -from keploy.constants import MODE_OFF -import keploy as k -from keploy.contrib.flask.utils import get_request_data -from keploy.models import HttpResp -from keploy.utils import capture_test -from werkzeug import Response - -class KFlask(object): - - def __init__(self, keploy:k.Keploy=None, app:Flask=None): - self.app = app - self.keploy = keploy - - if not app: - raise ValueError("Flask app instance not passed, Please initiate flask instance and pass it as keyword argument.") - - if not keploy or k.getMode() == MODE_OFF: - return - - app.wsgi_app = KeployMiddleware(keploy, app.wsgi_app) - - -class KeployMiddleware(object): - def __init__(self, kep, app) -> None: - self.app = app - self.keploy = kep - - def __call__(self, environ, start_response) -> Any: - - if not self.keploy: - return self.app(environ, start_response) - - req = {} - res = {} - - def _start_response(status, response_headers, *args): - nonlocal req - nonlocal res - req = get_request_data(request) - res['header'] = {key: [value] for key,value in response_headers} - res['status_code'] = int(status.split(' ')[0]) - return start_response(status, response_headers, *args) - - def _end_response(resp_body): - nonlocal res - res['body'] = b"".join(resp_body).decode("utf8") - return [res['body'].encode('utf-8')] - - resp = _end_response(self.app(environ, _start_response)) - - if environ.get("HTTP_KEPLOY_TEST_ID", None): - self.keploy.put_resp(environ.get('HTTP_KEPLOY_TEST_ID'), HttpResp(**res)) - else: - capture_test(self.keploy, req, res) - - return resp diff --git a/keploy/contrib/flask/utils.py b/keploy/contrib/flask/utils.py deleted file mode 100644 index 17dbac0..0000000 --- a/keploy/contrib/flask/utils.py +++ /dev/null @@ -1,20 +0,0 @@ - -def get_request_data(request) -> dict: - req_data = {} - - req_data['header'] = {key: [value] for key,value in request.headers.to_wsgi_list()} - req_data['method'] = request.method - req_data['body'] = request.get_data(as_text=True) - # req_data['form_data'] = request.form.to_dict() - # req_data['file_data'] = { k: v[0].read() for k, v in request.files.lists()} - req_data['uri'] = request.url_rule.rule - req_data['url'] = request.path - req_data['base'] = request.url - req_data['params'] = request.args.to_dict() - - protocol = request.environ.get('SERVER_PROTOCOL', None) - if protocol: - req_data['proto_major'] = int(protocol.split(".")[0][-1]) - req_data['proto_minor'] = int(protocol.split(".")[1]) - - return req_data \ No newline at end of file diff --git a/keploy/keploy.py b/keploy/keploy.py deleted file mode 100644 index 87f5ba5..0000000 --- a/keploy/keploy.py +++ /dev/null @@ -1,269 +0,0 @@ -import json -import logging -import http.client -import re -import threading -import time -from typing import Iterable, List, Mapping, Optional, Sequence -from keploy.mode import getMode -from keploy.constants import MODE_TEST, USE_HTTPS - -from keploy.models import Config, Dependency, HttpResp, TestCase, TestCaseRequest, TestReq - - -class Keploy(object): - - def __init__(self, conf:Config) -> None: - - if not isinstance(conf, Config): - raise TypeError("Please provide a valid keploy configuration.") - - logger = logging.getLogger('keploy') - logger.setLevel(logging.DEBUG) - - self._config = conf - self._logger = logger - self._dependencies = {} - self._responses = {} - self._client = None - - if self._config.server.protocol == USE_HTTPS: - self._client = http.client.HTTPSConnection(host=self._config.server.url, port=self._config.server.port) - else: - self._client = http.client.HTTPConnection(host=self._config.server.url, port=self._config.server.port) - - # self._client.connect() - - if getMode() == MODE_TEST: - t = threading.Thread(target=self.test) - t.start() - - - def get_dependencies(self, id: str) -> Optional[Iterable[Dependency]]: - return self._dependencies.get(id, None) - - - def get_resp(self, t_id: str) -> Optional[HttpResp]: - return self._responses.get(t_id, None) - - - def put_resp(self, t_id:str, resp: HttpResp) -> None: - self._responses[t_id] = resp - - - def capture(self, request:TestCaseRequest): - self.put(request) - - - def start(self, total:int) -> Optional[str]: - try: - headers = {'Content-type': 'application/json', 'key': self._config.server.licenseKey} - self._client.request("GET", "/{}/regression/start?app={}&total={}".format(self._config.server.suffix, self._config.app.name, total), None, headers) - - response = self._client.getresponse() - if response.status != 200: - self._logger.error("Error occured while fetching start information. Please try again.") - return - - body = json.loads(response.read().decode()) - if body.get('id', None): - return body['id'] - - self._logger.error("failed to start operation.") - return - except: - self._logger.error("Exception occured while starting the test case run.") - - - def end(self, id:str, status:bool) -> None: - try: - headers = {'Content-type': 'application/json', 'key': self._config.server.licenseKey} - self._client.request("GET", "/{}/regression/end?id={}&status={}".format(self._config.server.suffix, id, status), None, headers) - - response = self._client.getresponse() - if response.status != 200: - self._logger.error("failed to perform end operation.") - - return - except: - self._logger.error("Exception occured while ending the test run.") - - - def put(self, rq: TestCaseRequest): - try: - filters = self._config.app.filters - if filters: - match = re.search(filters.urlRegex, rq.uri) - if match: - return None - - headers = {'Content-type': 'application/json', 'key': self._config.server.licenseKey} - bytes_data = json.dumps(rq, default=lambda o: o.__dict__).encode() - self._client.request("POST", "/{}/regression/testcase".format(self._config.server.suffix), bytes_data, headers) - - response = self._client.getresponse() - if response.status != 200: - self._logger.error("failed to send testcase to backend") - - body = json.loads(response.read().decode()) - if body.get('id', None): - self.denoise(body['id'], rq) - - except: - self._logger.error("Exception occured while storing the request information. Try again.") - - - def denoise(self, id:str, tcase:TestCaseRequest): - try: - unit = TestCase(id=id, captured=tcase.captured, uri=tcase.uri, http_req=tcase.http_req, http_resp={}, deps=tcase.deps) - res = self.simulate(unit) - if not res: - self._logger.error("failed to simulate request while denoising") - return - - headers = {'Content-type': 'application/json', 'key': self._config.server.licenseKey} - bin_data = json.dumps(TestReq(id=id, app_id=self._config.app.name, resp=res), default=lambda o: o.__dict__).encode() - self._client.request("POST", "/{}/regression/denoise".format(self._config.server.suffix), bin_data, headers) - - response = self._client.getresponse() - if response.status != 200: - self._logger.error("failed to de-noise request to backend") - - body = response.read().decode() - - except: - self._logger.error("Error occured while denoising the test case request. Skipping...") - - - def simulate(self, test_case:TestCase) -> Optional[HttpResp]: - try: - self._dependencies[test_case.id] = test_case.deps - - heads = test_case.http_req.header - heads['KEPLOY_TEST_ID'] = [test_case.id] - - cli = http.client.HTTPConnection(self._config.app.host, self._config.app.port) - cli._http_vsn = int(str(test_case.http_req.proto_major) + str(test_case.http_req.proto_minor)) - cli._http_vsn_str = 'HTTP/{}.{}'.format(test_case.http_req.proto_major, test_case.http_req.proto_minor) - - cli.request( - method=test_case.http_req.method, - url=self._config.app.suffix + test_case.http_req.url, - body=json.dumps(test_case.http_req.body).encode(), - headers={key: value[0] for key, value in heads.items()} - ) - - time.sleep(2.0) - # TODO: getting None in case of regular execution. Urgent fix needed. - response = self.get_resp(test_case.id) - if not response or not self._responses.pop(test_case.id, None): - self._logger.error("failed loading the response for testcase.") - return - - self._dependencies.pop(test_case.id, None) - cli.close() - - return response - - except Exception as e: - self._logger.exception("Exception occured in simulation of test case with id: %s" %test_case.id) - - - def check(self, r_id:str, tc: TestCase) -> bool: - try: - resp = self.simulate(tc) - if not resp: - self._logger.error("failed to simulate request on local server.") - return False - - headers = {'Content-type': 'application/json', 'key': self._config.server.licenseKey} - bytes_data = json.dumps(TestReq(id=tc.id, app_id=self._config.app.name, run_id=r_id, resp=resp), default=lambda o: o.__dict__).encode() - self._client.request("POST", "/{}/regression/test".format(self._config.server.suffix), bytes_data, headers) - - response = self._client.getresponse() - if response.status != 200: - self._logger.error("failed to read response from backend") - - body = json.loads(response.read().decode()) - if body.get('pass', False): - return body['pass'] - - return False - - except: - self._logger.exception("[SKIP] Failed to check testcase with id: %s" %tc.id) - return False - - - def get(self, id:str) -> Optional[TestCase]: - try: - headers = {'Content-type': 'application/json', 'key': self._config.server.licenseKey} - self._client.request("GET", "/{}/regression/testcase/{}".format(self._config.server.suffix, id), None, headers) - - response = self._client.getresponse() - if response.status != 200: - self._logger.error("failed to get request.") - - body = json.loads(response.read().decode()) - unit = TestCase(**body) - - return unit - - except: - self._logger.error("Exception occured while fetching the test case with id: %s" %id) - return - - - def fetch(self, offset:int=0, limit:int=25) -> Optional[Sequence[TestCase]]: - try: - test_cases = [] - headers = {'Content-type': 'application/json', 'key': self._config.server.licenseKey} - - while True: - self._client.request("GET", "/{}/regression/testcase?app={}&offset={}&limit={}".format(self._config.server.suffix, self._config.app.name, offset, limit), None, headers) - response = self._client.getresponse() - if response.status != 200: - self._logger.error("Error occured while fetching test cases. Please try again.") - return - - body = json.loads(response.read().decode()) - if body: - for idx, case in enumerate(body): - body[idx] = TestCase(**case) - test_cases.extend(body) - offset += limit - else: - break - return test_cases - - except: - self._logger.exception("Exception occured while fetching test cases.") - return - - - def test(self): - passed = True - time.sleep(self._config.app.delay) - - self._logger.info("Started test operations on the captured test cases.") - cases = self.fetch() - count = len(cases) - - self._logger.info("Total number of test cases to be checked = %d" %count) - run_id = self.start(count) - - if not run_id: - return - - self._logger.info("Started with testing...") - for case in cases: - ok = self.check(run_id, case) - if not ok: - passed = False - self._logger.info("Finished with testing...") - - self._logger.info("Cleaning up things...") - self.end(run_id, passed) - - return passed - diff --git a/keploy/mode.py b/keploy/mode.py deleted file mode 100644 index 4b51eb6..0000000 --- a/keploy/mode.py +++ /dev/null @@ -1,21 +0,0 @@ -from typing import Literal -from keploy.constants import MODE_OFF, MODE_RECORD, MODE_TEST - - -mode = MODE_RECORD - -def isValidMode(mode): - if mode in [MODE_OFF, MODE_RECORD, MODE_TEST]: - return True - return False - -def getMode(): - return mode - -MODES = Literal["off", "record", "test"] -def setMode(m:MODES): - if isValidMode(m): - global mode - mode = m - return - raise Exception("Mode:{} not supported by keploy. Please enter a valid mode.".format(m)) diff --git a/keploy/models.py b/keploy/models.py deleted file mode 100644 index 5a22048..0000000 --- a/keploy/models.py +++ /dev/null @@ -1,179 +0,0 @@ - -from ast import literal_eval -from typing import Any, Iterable, List, Literal, Optional, Mapping, Sequence - -from keploy.constants import ALLOWED_DEPENDENCY_TYPES, ALLOWED_METHODS - - -def getHostPort(h:str = None, p:int = None): - host = '' - suffix = '' - port = 0 - - if not h and not p: - raise ValueError("Invalid host or port values.") - - if h.startswith(("http:", "https:")): - raise ValueError("please pass host name without http:// or https://") - - host = h - i = h.find("/") - if i > 0: - host = h[:i] - suffix = h[i+1:] - - port = p or 80 - i = h.find(":") - if i > 0: - if p and str(p) != host[i+1:]: - raise ValueError("2 Ports found. Please pass the port as a function argumet only.") - port = int(host[i+1:]) - host = host[:i] - - return (host, port, suffix.strip('/')) - - -class FilterConfig(object): - def __init__(self, regex: str) -> None: - self.urlRegex = regex - - -class AppConfig(object): - def __init__(self, name:str=None, host:str=None, port:int=None, delay:int=5, timeout:int=60, filters:FilterConfig=None): - self.name = name or 'test-keploy' - self.host = host - self.port = port - self.delay = delay - self.timeout = timeout - self.filters = filters - self.suffix = None - - if not host: - raise ValueError("Host not provided in AppConfig.") - - self.host, self.port, self.suffix = getHostPort(host, port) - - -PROTOCOL = Literal['https', 'http'] -class ServerConfig(object): - def __init__(self, protocol:PROTOCOL='https', host:str='api.keploy.io', port:int=None, licenseKey:str=''): - - self.protocol = protocol - self.url = host - self.port = port - self.licenseKey = licenseKey - self.suffix = '' - - if protocol not in ["http", "https"]: - raise ValueError("Invalid protocol type. Please use from available options.") - - if not licenseKey: - self.url, self.port, self.suffix = getHostPort(self.url, self.port) - - -class Config(object): - def __init__(self, appConfig:AppConfig, serverConfig:ServerConfig) -> None: - - if not isinstance(appConfig, AppConfig): - raise TypeError("Please provide a valid app configuration.") - - if not isinstance(serverConfig, ServerConfig): - raise TypeError("Please provide a valid server configuration.") - - self.app = appConfig - self.server = serverConfig - - -TYPES= Literal['NO_SQL_DB', 'SQL_DB', 'GRPC', 'HTTP_CLIENT'] -class Dependency(object): - def __init__(self, name:str, type:TYPES, meta:Mapping[str, str]=None, data:List[bytearray]=None) -> None: - self.name = name - self.type = type - self.meta = meta - self.data = data - - if not type in ALLOWED_DEPENDENCY_TYPES: - raise TypeError("Please provide a valid dependency type.") - - -METHODS = Literal['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'HEAD', 'OPTIONS', 'TRACE'] -class HttpReq(object): - def __init__(self, method:METHODS=None, proto_major:int=1, proto_minor:int=1, url:str=None, url_params:Mapping[str, str]=None, header:Mapping[str, Sequence[str]]=None, body:str=None) -> None: - self.method = method - self.proto_major = proto_major - self.proto_minor = proto_minor - self.url = url - self.url_params = url_params - self.header = header - self.body = body - - if method not in ALLOWED_METHODS: - raise ValueError("{} method is not supported. Please try some other method.".format(self.method)) - - -class HttpResp(object): - def __init__(self, status_code:int=None, header:Mapping[str, Sequence[str]]=None, body:str=None) -> None: - self.status_code = status_code - self.header = header - self.body = body - - -class TestCase(object): - def __init__(self, id:str=None, created:int=None, updated:int=None, captured:int=None, - cid:str=None, app_id:str=None, uri:str=None, http_req:HttpReq=None, - http_resp: HttpResp=None, deps:Sequence[Dependency]=None, all_keys:Mapping[str, Sequence[str]]=None, - anchors:Mapping[str, Sequence[str]]=None, noise:Sequence[str]=None ) -> None: - - #TODO: Need to handle the case when the API response is None instead of [] for deps - if not deps: - deps = [] - - self.id = id - self.created = created - self.updated = updated - self.captured = captured - self.cid = cid - self.app_id = app_id - self.uri = uri - self.http_req = http_req - self.http_resp = http_resp - self.deps = [Dependency(**dep) if not isinstance(dep, Dependency) else dep for dep in deps] - self.all_keys = all_keys - self.anchors = anchors - self.noise = noise - - if not isinstance(http_req, HttpReq): - self.http_req = HttpReq(**http_req) - - if not isinstance(http_resp, HttpResp): - self.http_resp = HttpResp(**http_resp) - - -class TestCaseRequest(object): - def __init__(self, captured:int=None, app_id:str=None, uri:str=None, http_req:HttpReq=None, http_resp:HttpResp=None, deps:Iterable[Dependency]=None) -> None: - self.captured = captured - self.app_id = app_id - self.uri = uri - self.http_req = http_req - self.http_resp = http_resp - self.deps = deps - - if not captured: - raise ValueError("Captured time cannot be empty.") - - if not app_id: - raise ValueError("valid App ID is required to link the TestReq object.") - - -class TestReq(object): - def __init__(self, id:str=None, app_id:str=None, run_id:str=None, resp:HttpResp=None) -> None: - self.id = id - self.app_id = app_id - self.run_id = run_id - self.resp = resp - - if not id: - raise ValueError("ID is required in the TestReq object.") - - if not app_id: - raise ValueError("valid App ID is required to link the TestReq object.") \ No newline at end of file diff --git a/keploy/utils.py b/keploy/utils.py deleted file mode 100644 index da68a04..0000000 --- a/keploy/utils.py +++ /dev/null @@ -1,32 +0,0 @@ -from keploy.keploy import Keploy -from keploy.models import Dependency, HttpReq, HttpResp, TestCaseRequest -import time - -def capture_test(k, reqst, resp): - - deps = [ Dependency('demo_dep', 'HTTP_CLIENT', {}, None), ] - - test = TestCaseRequest( - captured=int(time.time()), - app_id=k._config.app.name, - uri=reqst['uri'], - http_req=HttpReq( - method=reqst['method'], - proto_major=reqst['proto_major'], - proto_minor=reqst['proto_minor'], - url=reqst['url'], - url_params=reqst['params'], - header=reqst['header'], - body=reqst['body'] - ), - http_resp=HttpResp( - status_code=resp['status_code'], - header=resp['header'], - body=resp['body'] - ), - deps=deps - ) - - k.capture(test) - - return \ No newline at end of file diff --git a/keploy/v2/__init__.py b/keploy/v2/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/keploy/setup.py b/setup.py similarity index 58% rename from keploy/setup.py rename to setup.py index ec374ea..0290717 100644 --- a/keploy/setup.py +++ b/setup.py @@ -1,20 +1,19 @@ from setuptools import setup, find_packages -VERSION = '0.0.1' +VERSION = '2.0.0-alpha1' DESCRIPTION = 'Keploy' LONG_DESCRIPTION = 'Keploy Python SDK' # Setting up setup( - name="keploy", + name="keploy", version=VERSION, author="Keploy Inc.", - author_email="contact@keploy.io", - description=DESCRIPTION, - long_description=LONG_DESCRIPTION, - packages=find_packages(), - install_requires=[], # add any additional packages that needs to be installed along with your package. - + author_email="hello@keploy.io", + description="Run your unit tests with Keploy", + long_description="This module allows you to run your unit tests along with pytest and get coverage for the same using Keploy", + packages=find_packages(where='src'), + package_dir={'': 'src'}, keywords=['keploy', 'python', 'sdk'], classifiers= [ "Development Status :: 3 - Alpha", @@ -23,5 +22,6 @@ "Programming Language :: Python :: 3", "Operating System :: MacOS :: MacOS X", "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX :: Linux", ] ) \ No newline at end of file diff --git a/keploy/v2/keployCLI.py b/src/keploy/Keploy.py similarity index 62% rename from keploy/v2/keployCLI.py rename to src/keploy/Keploy.py index 56b5a38..f034244 100644 --- a/keploy/v2/keployCLI.py +++ b/src/keploy/Keploy.py @@ -1,16 +1,13 @@ -import json -import os import requests import subprocess -import threading import logging -import coverage -logger = logging.getLogger('KeployCLI') +logging.basicConfig(level=logging.info) + +logger = logging.getLogger('Keploy') GRAPHQL_ENDPOINT = "/query" -HOST = "http://localhost:" -server_port = 6789 +HOST = "http://localhost:6789" user_command_pid = 0 class TestRunStatus: @@ -18,6 +15,56 @@ class TestRunStatus: PASSED = 2 FAILED = 3 +def run(run_cmd): + test_sets = fetch_test_sets() + logger.info("test_sets: ", test_sets) + if len(test_sets) == 0: + raise AssertionError("Failed to fetch test sets. Are you in the right directory?") + # Run for each test set. + for test_set in test_sets: + test_run_id = run_test_set(test_set) + start_user_application(run_cmd) + if test_run_id is None: + logger.error(f"Failed to run test set: {test_set}") + continue + logger.info(f"Running test set: {test_set} with testrun ID: {test_run_id}") + while True: + subprocess.call(["sleep", "2"]) + status = fetch_test_set_status(test_run_id) + if status is None: + logger.error(f"Failed to fetch status for test set: {test_set}") + if status == TestRunStatus.RUNNING: + logger.info(f"Test set: {test_set} is still running") + elif status == TestRunStatus.PASSED: + logger.info(f"Test set: {test_set} passed") + break + elif status == TestRunStatus.FAILED: + logger.error(f"Test set: {test_set} failed") + break + stop_user_application() + # Wait for the user application to stop + subprocess.call(["sleep", "10"]) + stop_test() + + +def stop_test(): + sessions,url, headers = set_http_client() + if url is None or headers is None: + return None + payload = {"query": "{ stopTest }"} + try: + response = requests.post(url, headers=headers, json=payload, timeout=10) + logger.debug(f"Status code received: {response.status_code}") + if response.ok: + res_body = response.json() + logger.debug(f"Response body received: {res_body}") + return res_body.get('data', {}).get('stopTest') + else: + return None + except Exception as e: + logger.error("Error stopping test", e) + return None + def start_user_application(run_cmd): global user_command_pid @@ -27,9 +74,6 @@ def start_user_application(run_cmd): def stop_user_application(): kill_process_by_pid(user_command_pid) - cov = coverage.Coverage() - cov.load() - cov.save() def get_test_run_status(status_str): @@ -42,7 +86,7 @@ def get_test_run_status(status_str): def set_http_client(): try: - url = f"{HOST}{server_port}{GRAPHQL_ENDPOINT}" + url = f"{HOST}{GRAPHQL_ENDPOINT}" logger.debug(f"Connecting to: {url}") headers = { "Content-Type": "application/json; charset=UTF-8", @@ -114,27 +158,6 @@ def run_test_set(test_set_name): logger.error("Error running test set", e) return None -def find_coverage(test_set): - # Ensure the coverage-report directory exists - coverage_report_dir = f'coverage-report/{test_set}' - if not os.path.exists(coverage_report_dir): - os.makedirs(coverage_report_dir) - - cov = coverage.Coverage() - cov.load() - - # Generate text coverage report - text_report_file = f"{coverage_report_dir}/{test_set}_coverage_report.txt" - with open(text_report_file, 'w') as f: - cov.report(file=f) - - # Generate HTML coverage report - html_report_dir = f"{coverage_report_dir}/{test_set}_coverage_html" - cov.html_report(directory=html_report_dir) - - # Log the report generation - logger.info(f"Coverage reports generated in {coverage_report_dir}") - def kill_process_by_pid(pid): try: # Using SIGTERM (signal 15) to gracefully terminate the process diff --git a/src/keploy/__init__.py b/src/keploy/__init__.py new file mode 100644 index 0000000..6daf22c --- /dev/null +++ b/src/keploy/__init__.py @@ -0,0 +1 @@ +from .Keploy import run \ No newline at end of file From db6aa97d09a1968c68c1b0b28b26c472323eff58 Mon Sep 17 00:00:00 2001 From: Pranshu Srivastava <37413698+PranshuSrivastava@users.noreply.github.com> Date: Tue, 5 Mar 2024 14:29:56 +0530 Subject: [PATCH 02/18] docs: handled sigterm (#8) Signed-off-by: Pranshu Srivastava --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 25d7bf2..ccca761 100644 --- a/README.md +++ b/README.md @@ -30,22 +30,23 @@ Keploy simplifies the testing process by seamlessly generating end-to-end test c Add a test file with the following code to the directory with all your existing tests. This will help us to get the coverage of Keploy's API tests along with the other unit tests. We can call this `test_keploy.py` ```python -from Keploy import run +from keploy import run def test_keploy(): run("python3 -m coverage run --data-file=.coverage_data.keploy ") ``` > Note: If you face any problems with running the coverage library, you can refer to the documentation for the same [here](https://coverage.readthedocs.io/en/7.4.2/cmd.html#execution-coverage-run) -To ignore the coverage of python libraries which are included in the report by default, you can create a .coveragerc file in the directory where you will ignore the /usr/ directory(only for Linux users). The contents of the file will be as follows: +To ignore the coverage of python libraries which are included in the report by default, you need to create a `.coveragerc` file in the directory where you will ignore the /usr/ directory(only for Linux users). The contents of the file will be as follows: ```bash [run] omit = /usr/* +sigterm = true ``` -Before starting your application, make sure that the debug mode is set to False in your application, for the coverage library to work properly. +Before starting your application, make sure that the **debug mode is set to False** in your application, for the coverage library to work properly. Now to run this testcase along with your another unit testcases, you can run the command below: From 51dacf1f0f5a77f91b2db6a796f92d0ec0213e3e Mon Sep 17 00:00:00 2001 From: gouravkrosx Date: Mon, 18 Mar 2024 18:09:39 +0530 Subject: [PATCH 03/18] chore: refactored for unit test integration Signed-off-by: gouravkrosx --- src/keploy/Keploy.py | 401 ++++++++++++++++++++++++++++++++----------- 1 file changed, 301 insertions(+), 100 deletions(-) diff --git a/src/keploy/Keploy.py b/src/keploy/Keploy.py index f034244..226d380 100644 --- a/src/keploy/Keploy.py +++ b/src/keploy/Keploy.py @@ -1,88 +1,163 @@ import requests -import subprocess import logging +import threading +import subprocess +import time -logging.basicConfig(level=logging.info) +logging.basicConfig(level=logging.INFO) -logger = logging.getLogger('Keploy') +logger = logging.getLogger("Keploy") GRAPHQL_ENDPOINT = "/query" HOST = "http://localhost:6789" -user_command_pid = 0 + class TestRunStatus: RUNNING = 1 PASSED = 2 FAILED = 3 + APP_HALTED = 4 + USER_ABORT = 5 + APP_FAULT = 6 + INTERNAL_ERR = 7 + + +def get_test_run_status(status_str): + status_mapping = { + "RUNNING": TestRunStatus.RUNNING, + "PASSED": TestRunStatus.PASSED, + "FAILED": TestRunStatus.FAILED, + "APP_HALTED": TestRunStatus.APP_HALTED, + "USER_ABORT": TestRunStatus.USER_ABORT, + "APP_FAULT": TestRunStatus.APP_FAULT, + "INTERNAL_ERR": TestRunStatus.INTERNAL_ERR, + } + return status_mapping.get(status_str) + + +def run(run_cmd, delay=10, debug=False): + # Starting keploy + start_keploy(run_cmd, delay, debug) + + # Delay for keploy to start + time.sleep(5) + + # Fetching test sets + test_sets, err = fetch_test_sets() + if err is not None: + stop_Keploy() # Stopping keploy as error fetching test sets + raise AssertionError(f"error fetching test sets: {err}") -def run(run_cmd): - test_sets = fetch_test_sets() - logger.info("test_sets: ", test_sets) - if len(test_sets) == 0: - raise AssertionError("Failed to fetch test sets. Are you in the right directory?") - # Run for each test set. - for test_set in test_sets: - test_run_id = run_test_set(test_set) - start_user_application(run_cmd) - if test_run_id is None: - logger.error(f"Failed to run test set: {test_set}") - continue - logger.info(f"Running test set: {test_set} with testrun ID: {test_run_id}") - while True: - subprocess.call(["sleep", "2"]) - status = fetch_test_set_status(test_run_id) - if status is None: - logger.error(f"Failed to fetch status for test set: {test_set}") - if status == TestRunStatus.RUNNING: + logger.debug(f"Test sets found: {test_sets}") + if len(test_sets) == 0: + stop_Keploy() # Stopping keploy as no test sets are found + raise AssertionError("No test sets found. Are you in the right directory?") + + # Start hooking for the application + appId, testRunId, err = start_hooks() + if err is not None: + stop_Keploy() + raise AssertionError(f"error starting hooks: {err}") + + # Run for each test set. + for test_set in test_sets: + # Run test set + run_test_set(testRunId, test_set, appId) + # Start user application + start_user_application(appId) + # Wait for keploy to write initial data to report file + time.sleep(delay) # Initial report is written only after delay in keploy + + logger.info(f"Running test set: {test_set} with testrun ID: {testRunId}") + status = None + while True: + time.sleep(2) + status, err = fetch_test_set_status(testRunId, test_set) + if err is not None or status is None: + logger.error( + f"error getting test set status for testRunId: {testRunId}, testSetId: {test_set}. Error: {err}" + ) + break + + match status: + case TestRunStatus.RUNNING: logger.info(f"Test set: {test_set} is still running") - elif status == TestRunStatus.PASSED: - logger.info(f"Test set: {test_set} passed") + case TestRunStatus.PASSED: + break + case TestRunStatus.FAILED: + break + case TestRunStatus.APP_HALTED: + break + case TestRunStatus.USER_ABORT: break - elif status == TestRunStatus.FAILED: - logger.error(f"Test set: {test_set} failed") + case TestRunStatus.APP_FAULT: + break + case TestRunStatus.INTERNAL_ERR: break - stop_user_application() - # Wait for the user application to stop - subprocess.call(["sleep", "10"]) - stop_test() + # Check if the test set status has some internal error + # In all these cases the application couldn't be started properly + if ( + status == None + or status == TestRunStatus.APP_HALTED + or status == TestRunStatus.USER_ABORT + or status == TestRunStatus.APP_FAULT + or status == TestRunStatus.INTERNAL_ERR + ): + logger.error(f"Test set: {test_set} failed with status: {status}") -def stop_test(): - sessions,url, headers = set_http_client() - if url is None or headers is None: - return None - payload = {"query": "{ stopTest }"} - try: - response = requests.post(url, headers=headers, json=payload, timeout=10) - logger.debug(f"Status code received: {response.status_code}") - if response.ok: - res_body = response.json() - logger.debug(f"Response body received: {res_body}") - return res_body.get('data', {}).get('stopTest') - else: - return None - except Exception as e: - logger.error("Error stopping test", e) - return None + if status == TestRunStatus.FAILED: + logger.error(f"Test set: {test_set} failed") + elif status == TestRunStatus.PASSED: + logger.info(f"Test set: {test_set} passed") -def start_user_application(run_cmd): - global user_command_pid + # Stop user application + err = stop_user_application(appId) + if err is not None: + stop_Keploy() + raise AssertionError(f"error stopping user application: {err}") + time.sleep(5) # Wait for the user application to stop + # Stop keploy after running all test sets + stop_Keploy() - command = run_cmd.split(" ") - process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) - user_command_pid = process.pid -def stop_user_application(): - kill_process_by_pid(user_command_pid) +def start_keploy(runCmd, delay, debug): + thread = threading.Thread( + target=run_keploy, + args=( + runCmd, + delay, + debug, + ), + daemon=False, + ) + thread.start() + return thread -def get_test_run_status(status_str): - status_mapping = { - 'RUNNING': TestRunStatus.RUNNING, - 'PASSED': TestRunStatus.PASSED, - 'FAILED': TestRunStatus.FAILED - } - return status_mapping.get(status_str) +def run_keploy(runCmd, delay, debug): + overallCmd = f'sudo -E env "PATH=$PATH" /usr/local/bin/keploy test -c "{runCmd}" --coverage --delay {delay}' + if debug: + overallCmd += " --debug" + + logger.debug(f"Executing command: {overallCmd}") + + command = ["sh", "-c", overallCmd] + process = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + ) + + # Read and print the output + for line in process.stdout: + # logger.info(line, end="") + print(line, end="",flush=True) + + # Wait for the process to finish + process.wait() + def set_http_client(): try: @@ -90,78 +165,204 @@ def set_http_client(): logger.debug(f"Connecting to: {url}") headers = { "Content-Type": "application/json; charset=UTF-8", - "Accept": "application/json" + "Accept": "application/json", } session = requests.Session() return session, url, headers except Exception as e: - logger.error("Error setting up HTTP client", e) + logger.error(f"Error setting up HTTP client", e) return None, None, None + def fetch_test_sets(): - sessions,url, headers = set_http_client() - if url is None or headers is None: - return None + sessions, url, headers = set_http_client() + if sessions is None or url is None or headers is None: + return [], "Failed to set up HTTP client" payload = {"query": "{ testSets }"} try: - response = requests.post(url, headers=headers, json=payload, timeout=10) + response = sessions.post(url, headers=headers, json=payload, timeout=10) logger.debug(f"Status code received: {response.status_code}") if response.ok: res_body = response.json() logger.debug(f"Response body received: {res_body}") - return res_body.get('data', {}).get('testSets') + if res_body.get("data", {}) is None: + return [], res_body.get("errors", {}) + return res_body.get("data", {}).get("testSets"), None + else: + return [], None except Exception as e: logger.error("Error fetching test sets", e) - return None + return [], None + + +def start_hooks(): + session, url, headers = set_http_client() + if session is None or url is None or headers is None: + return None, None, "Failed to set up HTTP client" + + payload = {"query": "mutation StartHooks { startHooks { appId testRunId } }"} -def fetch_test_set_status(test_run_id): try: - session, url, headers = set_http_client() - payload = { - "query": f"{{ testSetStatus(testRunId: \"{test_run_id}\") {{ status }} }}" - } + response = session.post(url, headers=headers, json=payload, timeout=10) + logger.debug(f"Status code received: {response.status_code}") + + if response.ok: + res_body = response.json() + logger.debug(f"Response body received: {res_body}") + if res_body.get("data", {}) is None: + return None, None, res_body.get("errors", {}) + + start_hooks_data = res_body.get("data", {}).get("startHooks") + if start_hooks_data is None: + return None, None, f"Failed to get start Hooks data" - response = session.post(url, headers=headers, json=payload) - logger.debug(f"status code received: {response.status_code}") + appId = start_hooks_data.get("appId") + testRunId = start_hooks_data.get("testRunId") + return appId, testRunId, None + else: + return ( + None, + None, + f"Failed to start hooks. Status code: {response.status_code}", + ) + except Exception as e: + logger.error(f"Error starting hooks: {e}") + return None, None, f"Error starting hooks: {e}" + + +def run_test_set(testRunId, testSetId, appId): + session, url, headers = set_http_client() + if session is None or url is None or headers is None: + return False, "Failed to set up HTTP client" + + payload = { + "query": f'mutation RunTestSet {{ runTestSet(testSetId: "{testSetId}", testRunId: "{testRunId}", appId: {appId}) }}' + } + + try: + response = session.post(url, headers=headers, json=payload, timeout=10) + logger.debug(f"Status code received: {response.status_code}") if response.ok: res_body = response.json() - logger.debug(f"response body received: {res_body}") - status = res_body['data']['testSetStatus']['status'] - return get_test_run_status(status) + logger.debug(f"Response body received: {res_body}") + if res_body.get("data", {}) is None: + return False, res_body.get("errors", {}) + return res_body.get("data", {}).get("runTestSet"), None + else: + return False, f"Failed to run test set. Status code: {response.status_code}" except Exception as e: - logger.error("Error fetching test set status", e) - return None + logger.error(f"Error running test set: {e}") + return False, f"Error running test set: {e}" + + +def start_user_application(appId): + session, url, headers = set_http_client() + if session is None or url is None or headers is None: + return False, "Failed to set up HTTP client" + + payload = {"query": f"mutation StartApp {{ startApp(appId: {appId}) }}"} -def run_test_set(test_set_name): try: - session, url, headers = set_http_client() - payload = { - "query": f"mutation {{ runTestSet(testSet: \"{test_set_name}\") {{ success testRunId message }} }}" - } + response = session.post(url, headers=headers, json=payload, timeout=10) + logger.debug(f"Status code received: {response.status_code}") + + if response.ok: + res_body = response.json() + logger.debug(f"Response body received: {res_body}") + if res_body.get("data", {}) is None: + return False, res_body.get("errors", {}) + return res_body.get("data", {}).get("startApp"), None + else: + return ( + False, + f"Failed to start user application. Status code: {response.status_code}", + ) + except Exception as e: + logger.error(f"Error starting user application: {e}") + return False, f"Error starting user application: {e}" + + +def fetch_test_set_status(testRunId, testSetId): + session, url, headers = set_http_client() + if session is None or url is None or headers is None: + return None, "Failed to set up HTTP client" + + payload = { + "query": f'query GetTestSetStatus {{ testSetStatus(testRunId: "{testRunId}", testSetId: "{testSetId}") {{ status }} }}' + } - response = session.post(url, headers=headers, json=payload, timeout=5) # Timeout set to 5 seconds + try: + response = session.post(url, headers=headers, json=payload, timeout=10) logger.debug(f"Status code received: {response.status_code}") if response.ok: res_body = response.json() logger.debug(f"Response body received: {res_body}") - if 'data' in res_body and 'runTestSet' in res_body['data']: - return res_body['data']['runTestSet']['testRunId'] - else: - logger.error(f"Unexpected response format: {res_body}") - return None + if res_body.get("data", {}) is None: + return None, res_body.get("errors", {}) + test_set_status = res_body.get("data", {}).get("testSetStatus", {}) + status = test_set_status.get("status") + return get_test_run_status(status), None + else: + return ( + None, + f"Failed to fetch test set status. Status code: {response.status_code}", + ) except Exception as e: - logger.error("Error running test set", e) - return None + logger.error(f"Error fetching test set status: {e}") + return None, f"Error fetching test set status: {e}" + + +def stop_user_application(appId): + session, url, headers = set_http_client() + if session is None or url is None or headers is None: + return "Failed to set up HTTP client" + + payload = {"query": f"mutation StopApp {{ stopApp(appId: {appId}) }}"} + + try: + response = session.post(url, headers=headers, json=payload, timeout=10) + logger.debug(f"Status code received: {response.status_code}") + + if response.ok: + res_body = response.json() + logger.debug(f"Response body received: {res_body}") + if res_body.get("data", {}) is None: + return res_body.get("errors", {}) + stop_app_result = res_body.get("data", {}).get("stopApp") + logger.debug(f"stopApp result: {stop_app_result}") + else: + return ( + f"Failed to stop user application. Status code: {response.status_code}" + ) + except Exception as e: + logger.error(f"Error stopping user application: {e}") + return f"Error stopping user application: {e}" + + return None + + +def stop_Keploy(): + session, url, headers = set_http_client() + if session is None or url is None or headers is None: + return "Failed to set up HTTP client" + + payload = {"query": "mutation { stopHooks }"} -def kill_process_by_pid(pid): try: - # Using SIGTERM (signal 15) to gracefully terminate the process - subprocess.run(["kill", "-15", str(pid)]) - logger.debug(f"Killed process with PID: {pid}") + response = session.post(url, headers=headers, json=payload, timeout=10) + logger.debug(f"Status code received: {response.status_code}") + + if response.ok: + res_body = response.json() + logger.debug(f"Response body received: {res_body}") + if res_body.get("data", {}) is None: + return res_body.get("errors", {}) + return res_body.get("data", {}).get("stopHooks") except Exception as e: - logger.error(f"Failed to kill process with PID {pid}", e) \ No newline at end of file + logger.error(f"Error stopping hooks: {e}") + return None From f708462345fc65a1338608f56193424c7258f59e Mon Sep 17 00:00:00 2001 From: gouravkrosx Date: Mon, 18 Mar 2024 18:44:25 +0530 Subject: [PATCH 04/18] chore: add struct for populating options Signed-off-by: gouravkrosx --- src/keploy/Keploy.py | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/src/keploy/Keploy.py b/src/keploy/Keploy.py index 226d380..24fea47 100644 --- a/src/keploy/Keploy.py +++ b/src/keploy/Keploy.py @@ -9,7 +9,9 @@ logger = logging.getLogger("Keploy") GRAPHQL_ENDPOINT = "/query" -HOST = "http://localhost:6789" +# HOST = "http://localhost:6789" +HOST = "localhost" +PORT = 6789 class TestRunStatus: @@ -35,9 +37,20 @@ def get_test_run_status(status_str): return status_mapping.get(status_str) -def run(run_cmd, delay=10, debug=False): +class RunOptions: + def __init__(self, delay=10, debug=False, port=6789): + self.delay = delay + self.debug = debug + self.port = port + + +def run(run_cmd, run_options: RunOptions): + if run_options.port is not 0: + global PORT + PORT = run_options.port + # Starting keploy - start_keploy(run_cmd, delay, debug) + start_keploy(run_cmd, run_options.delay, run_options.debug, PORT) # Delay for keploy to start time.sleep(5) @@ -66,7 +79,9 @@ def run(run_cmd, delay=10, debug=False): # Start user application start_user_application(appId) # Wait for keploy to write initial data to report file - time.sleep(delay) # Initial report is written only after delay in keploy + time.sleep( + run_options.delay + ) # Initial report is written only after delay in keploy logger.info(f"Running test set: {test_set} with testrun ID: {testRunId}") status = None @@ -121,13 +136,14 @@ def run(run_cmd, delay=10, debug=False): stop_Keploy() -def start_keploy(runCmd, delay, debug): +def start_keploy(runCmd, delay, debug, port): thread = threading.Thread( target=run_keploy, args=( runCmd, delay, debug, + port, ), daemon=False, ) @@ -135,8 +151,8 @@ def start_keploy(runCmd, delay, debug): return thread -def run_keploy(runCmd, delay, debug): - overallCmd = f'sudo -E env "PATH=$PATH" /usr/local/bin/keploy test -c "{runCmd}" --coverage --delay {delay}' +def run_keploy(runCmd, delay, debug, port): + overallCmd = f'sudo -E env "PATH=$PATH" /usr/local/bin/keploybin test -c "{runCmd}" --coverage --delay {delay} --port {port}' if debug: overallCmd += " --debug" @@ -153,7 +169,7 @@ def run_keploy(runCmd, delay, debug): # Read and print the output for line in process.stdout: # logger.info(line, end="") - print(line, end="",flush=True) + print(line, end="", flush=True) # Wait for the process to finish process.wait() @@ -161,7 +177,7 @@ def run_keploy(runCmd, delay, debug): def set_http_client(): try: - url = f"{HOST}{GRAPHQL_ENDPOINT}" + url = f"http://{HOST}:{PORT}{GRAPHQL_ENDPOINT}" logger.debug(f"Connecting to: {url}") headers = { "Content-Type": "application/json; charset=UTF-8", @@ -214,7 +230,7 @@ def start_hooks(): logger.debug(f"Response body received: {res_body}") if res_body.get("data", {}) is None: return None, None, res_body.get("errors", {}) - + start_hooks_data = res_body.get("data", {}).get("startHooks") if start_hooks_data is None: return None, None, f"Failed to get start Hooks data" From 8bdf47b80ef9debd6f5f249a15a299df0bbbeb05 Mon Sep 17 00:00:00 2001 From: gouravkrosx Date: Tue, 19 Mar 2024 13:19:06 +0530 Subject: [PATCH 05/18] chore: refactored code and addressed few comments Signed-off-by: gouravkrosx --- src/keploy/Keploy.py | 155 ++++++++++++++++++++++--------------------- 1 file changed, 78 insertions(+), 77 deletions(-) diff --git a/src/keploy/Keploy.py b/src/keploy/Keploy.py index 24fea47..d087ce0 100644 --- a/src/keploy/Keploy.py +++ b/src/keploy/Keploy.py @@ -41,11 +41,15 @@ class RunOptions: def __init__(self, delay=10, debug=False, port=6789): self.delay = delay self.debug = debug - self.port = port + # Ensure port is a positive integer + if isinstance(port, int) and port >= 0: + self.port = port + else: + raise ValueError("Port must be a positive integer.") def run(run_cmd, run_options: RunOptions): - if run_options.port is not 0: + if run_options.port != 0: global PORT PORT = run_options.port @@ -55,85 +59,82 @@ def run(run_cmd, run_options: RunOptions): # Delay for keploy to start time.sleep(5) - # Fetching test sets - test_sets, err = fetch_test_sets() - if err is not None: - stop_Keploy() # Stopping keploy as error fetching test sets - raise AssertionError(f"error fetching test sets: {err}") + try: + # Fetching test sets + test_sets, err = fetch_test_sets() + if err is not None: + raise AssertionError(f"error fetching test sets: {err}") - logger.debug(f"Test sets found: {test_sets}") - if len(test_sets) == 0: - stop_Keploy() # Stopping keploy as no test sets are found - raise AssertionError("No test sets found. Are you in the right directory?") + logger.debug(f"Test sets found: {test_sets}") + if len(test_sets) == 0: + raise AssertionError("No test sets found. Are you in the right directory?") - # Start hooking for the application - appId, testRunId, err = start_hooks() - if err is not None: - stop_Keploy() - raise AssertionError(f"error starting hooks: {err}") - - # Run for each test set. - for test_set in test_sets: - # Run test set - run_test_set(testRunId, test_set, appId) - # Start user application - start_user_application(appId) - # Wait for keploy to write initial data to report file - time.sleep( - run_options.delay - ) # Initial report is written only after delay in keploy - - logger.info(f"Running test set: {test_set} with testrun ID: {testRunId}") - status = None - while True: - time.sleep(2) - status, err = fetch_test_set_status(testRunId, test_set) - if err is not None or status is None: - logger.error( - f"error getting test set status for testRunId: {testRunId}, testSetId: {test_set}. Error: {err}" - ) - break - - match status: - case TestRunStatus.RUNNING: - logger.info(f"Test set: {test_set} is still running") - case TestRunStatus.PASSED: - break - case TestRunStatus.FAILED: - break - case TestRunStatus.APP_HALTED: - break - case TestRunStatus.USER_ABORT: - break - case TestRunStatus.APP_FAULT: - break - case TestRunStatus.INTERNAL_ERR: + # Start hooking for the application + appId, testRunId, err = start_hooks() + if err is not None: + raise AssertionError(f"error starting hooks: {err}") + + # Run for each test set. + for test_set in test_sets: + # Run test set + run_test_set(testRunId, test_set, appId) + # Start user application + start_user_application(appId) + # Wait for keploy to write initial data to report file + time.sleep( + run_options.delay + ) # Initial report is written only after delay in keploy + + logger.info(f"Running test set: {test_set} with testrun ID: {testRunId}") + status = None + while True: + time.sleep(2) + status, err = fetch_test_set_status(testRunId, test_set) + if err is not None or status is None: + logger.error( + f"error getting test set status for testRunId: {testRunId}, testSetId: {test_set}. Error: {err}" + ) break - # Check if the test set status has some internal error - # In all these cases the application couldn't be started properly - if ( - status == None - or status == TestRunStatus.APP_HALTED - or status == TestRunStatus.USER_ABORT - or status == TestRunStatus.APP_FAULT - or status == TestRunStatus.INTERNAL_ERR - ): - logger.error(f"Test set: {test_set} failed with status: {status}") - - if status == TestRunStatus.FAILED: - logger.error(f"Test set: {test_set} failed") - elif status == TestRunStatus.PASSED: - logger.info(f"Test set: {test_set} passed") - - # Stop user application - err = stop_user_application(appId) - if err is not None: - stop_Keploy() - raise AssertionError(f"error stopping user application: {err}") - time.sleep(5) # Wait for the user application to stop - # Stop keploy after running all test sets - stop_Keploy() + match status: + case TestRunStatus.RUNNING: + logger.info(f"Test set: {test_set} is still running") + case TestRunStatus.PASSED: + break + case TestRunStatus.FAILED: + break + case TestRunStatus.APP_HALTED: + break + case TestRunStatus.USER_ABORT: + break + case TestRunStatus.APP_FAULT: + break + case TestRunStatus.INTERNAL_ERR: + break + + # Check if the test set status has some internal error + if ( + status == None + or status == TestRunStatus.APP_HALTED + or status == TestRunStatus.USER_ABORT + or status == TestRunStatus.APP_FAULT + or status == TestRunStatus.INTERNAL_ERR + ): + logger.error(f"Test set: {test_set} failed with status: {status}") + + if status == TestRunStatus.FAILED: + logger.error(f"Test set: {test_set} failed") + elif status == TestRunStatus.PASSED: + logger.info(f"Test set: {test_set} passed") + + # Stop user application + err = stop_user_application(appId) + if err is not None: + raise AssertionError(f"error stopping user application: {err}") + time.sleep(5) # Wait for the user application to stop + finally: + # Stop keploy after running all test sets + stop_Keploy() def start_keploy(runCmd, delay, debug, port): From c805172a4d0c039033d36112572f15de35c80e0c Mon Sep 17 00:00:00 2001 From: gouravkrosx Date: Tue, 19 Mar 2024 14:06:58 +0530 Subject: [PATCH 06/18] chore: resolved PR commits Signed-off-by: gouravkrosx --- src/keploy/Keploy.py | 47 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/src/keploy/Keploy.py b/src/keploy/Keploy.py index d087ce0..77827fd 100644 --- a/src/keploy/Keploy.py +++ b/src/keploy/Keploy.py @@ -3,13 +3,13 @@ import threading import subprocess import time +import os logging.basicConfig(level=logging.INFO) logger = logging.getLogger("Keploy") GRAPHQL_ENDPOINT = "/query" -# HOST = "http://localhost:6789" HOST = "localhost" PORT = 6789 @@ -38,9 +38,14 @@ def get_test_run_status(status_str): class RunOptions: - def __init__(self, delay=10, debug=False, port=6789): - self.delay = delay + def __init__(self, delay=10, debug=False, port=6789, path="."): + if isinstance(delay, int) and delay >= 0: + self.delay = delay + else: + raise ValueError("Delay must be a positive integer.") self.debug = debug + if path != "": + self.path = path # Ensure port is a positive integer if isinstance(port, int) and port >= 0: self.port = port @@ -80,15 +85,28 @@ def run(run_cmd, run_options: RunOptions): run_test_set(testRunId, test_set, appId) # Start user application start_user_application(appId) + + path = os.path.abspath(run_options.path) + report_path = os.path.join( + path, "Keploy", "reports", testRunId, f"{test_set}-report.yaml" + ) + # check if the report file is created + err = check_report_file(report_path, run_options.delay + 10) + if err is not None: + # Stop user application + appErr = stop_user_application(appId) + if appErr is not None: + raise AssertionError(f"error stopping user application: {appErr}") + logger.error( + f"error getting report file: {testRunId}, testSetId: {test_set}. Error: {err}" + ) + continue + # Wait for keploy to write initial data to report file - time.sleep( - run_options.delay - ) # Initial report is written only after delay in keploy logger.info(f"Running test set: {test_set} with testrun ID: {testRunId}") status = None while True: - time.sleep(2) status, err = fetch_test_set_status(testRunId, test_set) if err is not None or status is None: logger.error( @@ -303,6 +321,21 @@ def start_user_application(appId): return False, f"Error starting user application: {e}" +def check_report_file(report_path, timeout): + + logger.debug(f"Checking report file at: {report_path}") + print(f"Checking report file at: {report_path}") + + start_time = time.time() + while time.time() - start_time < timeout: + print(f"Checking Path: {report_path}") + if os.path.exists(report_path): + return None # Report file found + time.sleep(1) # Wait for 1 second before checking again + + return f"Report file not created within {timeout} seconds" + + def fetch_test_set_status(testRunId, testSetId): session, url, headers = set_http_client() if session is None or url is None or headers is None: From e81ad2f8ea8446b6039bbd23f9fc9212f1f026a2 Mon Sep 17 00:00:00 2001 From: gouravkrosx Date: Thu, 28 Mar 2024 12:46:02 +0530 Subject: [PATCH 07/18] chore: remove print statements Signed-off-by: gouravkrosx --- src/keploy/Keploy.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/keploy/Keploy.py b/src/keploy/Keploy.py index 77827fd..514c3a4 100644 --- a/src/keploy/Keploy.py +++ b/src/keploy/Keploy.py @@ -324,11 +324,9 @@ def start_user_application(appId): def check_report_file(report_path, timeout): logger.debug(f"Checking report file at: {report_path}") - print(f"Checking report file at: {report_path}") start_time = time.time() while time.time() - start_time < timeout: - print(f"Checking Path: {report_path}") if os.path.exists(report_path): return None # Report file found time.sleep(1) # Wait for 1 second before checking again From ec7a0a2f51cbf5a821ecd619ccda723b371745c2 Mon Sep 17 00:00:00 2001 From: gouravkrosx Date: Thu, 28 Mar 2024 12:55:58 +0530 Subject: [PATCH 08/18] change to debug level if debug==true Signed-off-by: gouravkrosx --- src/keploy/Keploy.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/keploy/Keploy.py b/src/keploy/Keploy.py index 514c3a4..0500a74 100644 --- a/src/keploy/Keploy.py +++ b/src/keploy/Keploy.py @@ -58,6 +58,9 @@ def run(run_cmd, run_options: RunOptions): global PORT PORT = run_options.port + if run_options.debug: + logger.setLevel(logging.DEBUG) + # Starting keploy start_keploy(run_cmd, run_options.delay, run_options.debug, PORT) From 16d40af0a6636df32e7631d5f182cfc6e47c9052 Mon Sep 17 00:00:00 2001 From: gouravkrosx Date: Thu, 28 Mar 2024 12:57:31 +0530 Subject: [PATCH 09/18] remove unnecessary return statement Signed-off-by: gouravkrosx --- src/keploy/Keploy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/keploy/Keploy.py b/src/keploy/Keploy.py index 0500a74..63cd795 100644 --- a/src/keploy/Keploy.py +++ b/src/keploy/Keploy.py @@ -170,7 +170,7 @@ def start_keploy(runCmd, delay, debug, port): daemon=False, ) thread.start() - return thread + return def run_keploy(runCmd, delay, debug, port): From 7cf2e63f973dca27b0bd54b1489ade24f0a1b893 Mon Sep 17 00:00:00 2001 From: gouravkrosx Date: Thu, 28 Mar 2024 13:06:46 +0530 Subject: [PATCH 10/18] chore: check if keploy is installed or not Signed-off-by: gouravkrosx --- src/keploy/Keploy.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/keploy/Keploy.py b/src/keploy/Keploy.py index 63cd795..1f95390 100644 --- a/src/keploy/Keploy.py +++ b/src/keploy/Keploy.py @@ -174,7 +174,12 @@ def start_keploy(runCmd, delay, debug, port): def run_keploy(runCmd, delay, debug, port): - overallCmd = f'sudo -E env "PATH=$PATH" /usr/local/bin/keploybin test -c "{runCmd}" --coverage --delay {delay} --port {port}' + keployBin = "/usr/local/bin/keploybin" + + if not os.path.exists(keployBin): + print(f"Keploy binary doesn't exist, please install keploy") + + overallCmd = f'sudo -E env "PATH=$PATH" "{keployBin}" test -c "{runCmd}" --coverage --delay {delay} --port {port}' if debug: overallCmd += " --debug" From ea593833849438786b1388636aebbf0f8759db36 Mon Sep 17 00:00:00 2001 From: gouravkrosx Date: Thu, 28 Mar 2024 13:11:22 +0530 Subject: [PATCH 11/18] chore: return if binary doesn't exist Signed-off-by: gouravkrosx --- src/keploy/Keploy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/keploy/Keploy.py b/src/keploy/Keploy.py index 1f95390..d8ef6fb 100644 --- a/src/keploy/Keploy.py +++ b/src/keploy/Keploy.py @@ -178,6 +178,7 @@ def run_keploy(runCmd, delay, debug, port): if not os.path.exists(keployBin): print(f"Keploy binary doesn't exist, please install keploy") + return overallCmd = f'sudo -E env "PATH=$PATH" "{keployBin}" test -c "{runCmd}" --coverage --delay {delay} --port {port}' if debug: From 28ada93513d05b6906b60716dabfd6f10a5afe3d Mon Sep 17 00:00:00 2001 From: gouravkrosx Date: Thu, 28 Mar 2024 19:18:14 +0530 Subject: [PATCH 12/18] chore: remove report file checking Signed-off-by: gouravkrosx --- src/keploy/Keploy.py | 52 +++++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/src/keploy/Keploy.py b/src/keploy/Keploy.py index d8ef6fb..273cbf5 100644 --- a/src/keploy/Keploy.py +++ b/src/keploy/Keploy.py @@ -58,8 +58,8 @@ def run(run_cmd, run_options: RunOptions): global PORT PORT = run_options.port - if run_options.debug: - logger.setLevel(logging.DEBUG) + # if run_options.debug: + # logger.setLevel(logging.DEBUG) # Starting keploy start_keploy(run_cmd, run_options.delay, run_options.debug, PORT) @@ -89,21 +89,23 @@ def run(run_cmd, run_options: RunOptions): # Start user application start_user_application(appId) - path = os.path.abspath(run_options.path) - report_path = os.path.join( - path, "Keploy", "reports", testRunId, f"{test_set}-report.yaml" - ) - # check if the report file is created - err = check_report_file(report_path, run_options.delay + 10) - if err is not None: - # Stop user application - appErr = stop_user_application(appId) - if appErr is not None: - raise AssertionError(f"error stopping user application: {appErr}") - logger.error( - f"error getting report file: {testRunId}, testSetId: {test_set}. Error: {err}" - ) - continue + # path = os.path.abspath(run_options.path) + # report_path = os.path.join( + # path, "Keploy", "reports", testRunId, f"{test_set}-report.yaml" + # ) + # # check if the report file is created + # err = check_report_file(report_path, run_options.delay + 10) + # if err is not None: + # # Stop user application + # appErr = stop_user_application(appId) + # if appErr is not None: + # raise AssertionError(f"error stopping user application: {appErr}") + # logger.error( + # f"error getting report file: {testRunId}, testSetId: {test_set}. Error: {err}" + # ) + # continue + + time.sleep(run_options.delay + 2) # Wait for keploy to write initial data to report file @@ -330,17 +332,17 @@ def start_user_application(appId): return False, f"Error starting user application: {e}" -def check_report_file(report_path, timeout): +# def check_report_file(report_path, timeout): - logger.debug(f"Checking report file at: {report_path}") +# logger.debug(f"Checking report file at: {report_path}") - start_time = time.time() - while time.time() - start_time < timeout: - if os.path.exists(report_path): - return None # Report file found - time.sleep(1) # Wait for 1 second before checking again +# start_time = time.time() +# while time.time() - start_time < timeout: +# if os.path.exists(report_path): +# return None # Report file found +# time.sleep(1) # Wait for 1 second before checking again - return f"Report file not created within {timeout} seconds" +# return f"Report file not created within {timeout} seconds" def fetch_test_set_status(testRunId, testSetId): From 410ffa215f8c9ede051e9f52cb2bfdddf52524a3 Mon Sep 17 00:00:00 2001 From: Gourav kumar <44055698+gouravkrosx@users.noreply.github.com> Date: Tue, 23 Apr 2024 12:14:59 +0530 Subject: [PATCH 13/18] chore: fix imports of python-sdk (#11) Signed-off-by: gouravkrosx --- __init__.py | 2 +- src/keploy/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/__init__.py b/__init__.py index 6daf22c..97a8d47 100644 --- a/__init__.py +++ b/__init__.py @@ -1 +1 @@ -from .Keploy import run \ No newline at end of file +from .Keploy import run, RunOptions diff --git a/src/keploy/__init__.py b/src/keploy/__init__.py index 6daf22c..97a8d47 100644 --- a/src/keploy/__init__.py +++ b/src/keploy/__init__.py @@ -1 +1 @@ -from .Keploy import run \ No newline at end of file +from .Keploy import run, RunOptions From cba4ea699d78aec3da47b68761e7c56e5efa512c Mon Sep 17 00:00:00 2001 From: Akash Kumar <91385321+AkashKumar7902@users.noreply.github.com> Date: Fri, 21 Jun 2024 15:17:26 +0530 Subject: [PATCH 14/18] feat: dedup middleware for django, flask and fastapi (#5) * feat: integrations for django and flask Signed-off-by: Akash Kumar * fix: only generate dedup data in test mode Signed-off-by: Akash Kumar * feat: added middleware for fast api Signed-off-by: Pranshu Srivastava --------- Signed-off-by: Akash Kumar Signed-off-by: Pranshu Srivastava Co-authored-by: Pranshu Srivastava --- keploy/integrations/djangoCov.py | 19 +++++++++++++++++++ keploy/integrations/fastApiCov.py | 20 ++++++++++++++++++++ keploy/integrations/flaskCov.py | 19 +++++++++++++++++++ keploy/integrations/utils.py | 23 +++++++++++++++++++++++ 4 files changed, 81 insertions(+) create mode 100644 keploy/integrations/djangoCov.py create mode 100644 keploy/integrations/fastApiCov.py create mode 100644 keploy/integrations/flaskCov.py create mode 100644 keploy/integrations/utils.py diff --git a/keploy/integrations/djangoCov.py b/keploy/integrations/djangoCov.py new file mode 100644 index 0000000..6d9da22 --- /dev/null +++ b/keploy/integrations/djangoCov.py @@ -0,0 +1,19 @@ +import coverage + +from utils import write_dedup + +class CoverageMiddleware: + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + id = request.headers.get('KEPLOY-TEST-ID') + if id == None: + return self.get_response(request) + cov = coverage.Coverage(cover_pylib=False) + cov.start() + response = self.get_response(request) + cov.stop() + result = cov.get_data() + write_dedup(result, id) + return response \ No newline at end of file diff --git a/keploy/integrations/fastApiCov.py b/keploy/integrations/fastApiCov.py new file mode 100644 index 0000000..a96f538 --- /dev/null +++ b/keploy/integrations/fastApiCov.py @@ -0,0 +1,20 @@ +import coverage +from fastapi import Request +from starlette.middleware.base import BaseHTTPMiddleware + +from utils import write_dedup + +class CoverageMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next): + id = request.headers.get('KEPLOY-TEST-ID') + if id is None: + response = await call_next(request) + return response + + cov = coverage.Coverage(cover_pylib=False) + cov.start() + response = await call_next(request) + cov.stop() + result = cov.get_data() + write_dedup(result, id) + return response diff --git a/keploy/integrations/flaskCov.py b/keploy/integrations/flaskCov.py new file mode 100644 index 0000000..d973dea --- /dev/null +++ b/keploy/integrations/flaskCov.py @@ -0,0 +1,19 @@ +import coverage + +from utils import write_dedup + +class CoverageMiddleware: + def __init__(self, app): + self.app = app + + def __call__(self, environ, start_response): + id = environ.get('KEPLOY_TEST_ID') + if id == None: + return self.app(environ, start_response) + cov = coverage.Coverage(cover_pylib=False) + cov.start() + response = self.app(environ, start_response) + cov.stop() + result = cov.get_data() + write_dedup(result, id) + return response \ No newline at end of file diff --git a/keploy/integrations/utils.py b/keploy/integrations/utils.py new file mode 100644 index 0000000..81f4bd7 --- /dev/null +++ b/keploy/integrations/utils.py @@ -0,0 +1,23 @@ +import yaml + +def write_dedup(result, id): + filePath = '/Users/pranshu/testing/python-fastapi/dedupData.yaml' + existingData = [] + try: + with open(filePath, 'r') as file: + existingData=yaml.safe_load(file) + except: + with open(filePath, 'w') as file: + pass + + yaml_data = { + 'id': id, + 'executedLinesByFile': {} + } + for file in result.measured_files(): + yaml_data['executedLinesByFile'][file] = result.lines(file) + if existingData is None: + existingData=[] + existingData.append(yaml_data) + with open(filePath, 'w') as file: + yaml.dump(existingData, file, sort_keys=False) \ No newline at end of file From 29963203e42062f1d1d84914e9440112312176fe Mon Sep 17 00:00:00 2001 From: Pranshu Srivastava <37413698+PranshuSrivastava@users.noreply.github.com> Date: Tue, 25 Jun 2024 15:37:12 +0530 Subject: [PATCH 15/18] fix: modularized coverage middlewares to be imported directly from keploy and removed incorrect path (#12) Signed-off-by: Pranshu Srivastava --- __init__.py | 3 +++ setup.py | 2 +- src/keploy/__init__.py | 3 +++ {keploy/integrations => src/keploy}/djangoCov.py | 7 ++++--- {keploy/integrations => src/keploy}/fastApiCov.py | 4 ++-- {keploy/integrations => src/keploy}/flaskCov.py | 6 +++--- {keploy/integrations => src/keploy}/utils.py | 5 +++-- 7 files changed, 19 insertions(+), 11 deletions(-) rename {keploy/integrations => src/keploy}/djangoCov.py (83%) rename {keploy/integrations => src/keploy}/fastApiCov.py (86%) rename {keploy/integrations => src/keploy}/flaskCov.py (83%) rename {keploy/integrations => src/keploy}/utils.py (81%) diff --git a/__init__.py b/__init__.py index 97a8d47..13eed87 100644 --- a/__init__.py +++ b/__init__.py @@ -1 +1,4 @@ from .Keploy import run, RunOptions +from .djangoCov import DjangoCoverageMiddleware +from .fastApiCov import FastApiCoverageMiddleware +from .flaskCov import FlaskCoverageMiddleware diff --git a/setup.py b/setup.py index 0290717..f6c2e7c 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -VERSION = '2.0.0-alpha1' +VERSION = '2.0.0-alpha36' DESCRIPTION = 'Keploy' LONG_DESCRIPTION = 'Keploy Python SDK' diff --git a/src/keploy/__init__.py b/src/keploy/__init__.py index 97a8d47..13eed87 100644 --- a/src/keploy/__init__.py +++ b/src/keploy/__init__.py @@ -1 +1,4 @@ from .Keploy import run, RunOptions +from .djangoCov import DjangoCoverageMiddleware +from .fastApiCov import FastApiCoverageMiddleware +from .flaskCov import FlaskCoverageMiddleware diff --git a/keploy/integrations/djangoCov.py b/src/keploy/djangoCov.py similarity index 83% rename from keploy/integrations/djangoCov.py rename to src/keploy/djangoCov.py index 6d9da22..6c0162a 100644 --- a/keploy/integrations/djangoCov.py +++ b/src/keploy/djangoCov.py @@ -1,8 +1,8 @@ import coverage -from utils import write_dedup +from .utils import write_dedup -class CoverageMiddleware: +class DjangoCoverageMiddleware: def __init__(self, get_response): self.get_response = get_response @@ -16,4 +16,5 @@ def __call__(self, request): cov.stop() result = cov.get_data() write_dedup(result, id) - return response \ No newline at end of file + return response + \ No newline at end of file diff --git a/keploy/integrations/fastApiCov.py b/src/keploy/fastApiCov.py similarity index 86% rename from keploy/integrations/fastApiCov.py rename to src/keploy/fastApiCov.py index a96f538..842c8a5 100644 --- a/keploy/integrations/fastApiCov.py +++ b/src/keploy/fastApiCov.py @@ -2,9 +2,9 @@ from fastapi import Request from starlette.middleware.base import BaseHTTPMiddleware -from utils import write_dedup +from .utils import write_dedup -class CoverageMiddleware(BaseHTTPMiddleware): +class FastApiCoverageMiddleware(BaseHTTPMiddleware): async def dispatch(self, request: Request, call_next): id = request.headers.get('KEPLOY-TEST-ID') if id is None: diff --git a/keploy/integrations/flaskCov.py b/src/keploy/flaskCov.py similarity index 83% rename from keploy/integrations/flaskCov.py rename to src/keploy/flaskCov.py index d973dea..4891d51 100644 --- a/keploy/integrations/flaskCov.py +++ b/src/keploy/flaskCov.py @@ -1,8 +1,8 @@ import coverage -from utils import write_dedup +from .utils import write_dedup -class CoverageMiddleware: +class FlaskCoverageMiddleware: def __init__(self, app): self.app = app @@ -16,4 +16,4 @@ def __call__(self, environ, start_response): cov.stop() result = cov.get_data() write_dedup(result, id) - return response \ No newline at end of file + return response diff --git a/keploy/integrations/utils.py b/src/keploy/utils.py similarity index 81% rename from keploy/integrations/utils.py rename to src/keploy/utils.py index 81f4bd7..9c55e61 100644 --- a/keploy/integrations/utils.py +++ b/src/keploy/utils.py @@ -1,7 +1,7 @@ import yaml def write_dedup(result, id): - filePath = '/Users/pranshu/testing/python-fastapi/dedupData.yaml' + filePath = 'dedupData.yaml' existingData = [] try: with open(filePath, 'r') as file: @@ -20,4 +20,5 @@ def write_dedup(result, id): existingData=[] existingData.append(yaml_data) with open(filePath, 'w') as file: - yaml.dump(existingData, file, sort_keys=False) \ No newline at end of file + yaml.dump(existingData, file, sort_keys=False) + \ No newline at end of file From 5247bb27f1ae8cde9b329fcbfdf48bf8505566bd Mon Sep 17 00:00:00 2001 From: Akash Kumar <91385321+AkashKumar7902@users.noreply.github.com> Date: Tue, 23 Jul 2024 17:09:15 +0530 Subject: [PATCH 16/18] fix: fetching keploy-test-id from request header (#13) * fix: fetching keploy-test-id from request header Signed-off-by: Akash Kumar * feat: writing testset along with test name to write function Signed-off-by: Pranshu Srivastava * feat: not calculating dedup coverage when global coverage is being calculated. Signed-off-by: Pranshu Srivastava --------- Signed-off-by: Akash Kumar Signed-off-by: Pranshu Srivastava Co-authored-by: Pranshu Srivastava --- src/keploy/djangoCov.py | 4 ++-- src/keploy/fastApiCov.py | 8 ++++++-- src/keploy/flaskCov.py | 8 +++++--- src/keploy/utils.py | 5 ++--- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/keploy/djangoCov.py b/src/keploy/djangoCov.py index 6c0162a..2ceda30 100644 --- a/src/keploy/djangoCov.py +++ b/src/keploy/djangoCov.py @@ -8,6 +8,7 @@ def __init__(self, get_response): def __call__(self, request): id = request.headers.get('KEPLOY-TEST-ID') + testSet = request.headers.get('KEPLOY-TEST-SET-ID') if id == None: return self.get_response(request) cov = coverage.Coverage(cover_pylib=False) @@ -15,6 +16,5 @@ def __call__(self, request): response = self.get_response(request) cov.stop() result = cov.get_data() - write_dedup(result, id) + write_dedup(result, id, testSet) return response - \ No newline at end of file diff --git a/src/keploy/fastApiCov.py b/src/keploy/fastApiCov.py index 842c8a5..f041453 100644 --- a/src/keploy/fastApiCov.py +++ b/src/keploy/fastApiCov.py @@ -7,14 +7,18 @@ class FastApiCoverageMiddleware(BaseHTTPMiddleware): async def dispatch(self, request: Request, call_next): id = request.headers.get('KEPLOY-TEST-ID') + testSet = request.headers.get('KEPLOY-TEST-SET-ID') if id is None: response = await call_next(request) return response - + global_coverage = coverage.Coverage.current() + if global_coverage is not None: + response = await call_next(request) + return response cov = coverage.Coverage(cover_pylib=False) cov.start() response = await call_next(request) cov.stop() result = cov.get_data() - write_dedup(result, id) + write_dedup(result, id, testSet) return response diff --git a/src/keploy/flaskCov.py b/src/keploy/flaskCov.py index 4891d51..fb9f8c8 100644 --- a/src/keploy/flaskCov.py +++ b/src/keploy/flaskCov.py @@ -1,5 +1,5 @@ import coverage - +from werkzeug.wrappers import Request from .utils import write_dedup class FlaskCoverageMiddleware: @@ -7,7 +7,9 @@ def __init__(self, app): self.app = app def __call__(self, environ, start_response): - id = environ.get('KEPLOY_TEST_ID') + request = Request(environ) + id = request.headers.get("Keploy-Test-Id") + testSet = request.headers.get("Keploy-Test-Set-Id") if id == None: return self.app(environ, start_response) cov = coverage.Coverage(cover_pylib=False) @@ -15,5 +17,5 @@ def __call__(self, environ, start_response): response = self.app(environ, start_response) cov.stop() result = cov.get_data() - write_dedup(result, id) + write_dedup(result, id, testSet) return response diff --git a/src/keploy/utils.py b/src/keploy/utils.py index 9c55e61..435c99e 100644 --- a/src/keploy/utils.py +++ b/src/keploy/utils.py @@ -1,6 +1,6 @@ import yaml -def write_dedup(result, id): +def write_dedup(result, id, testSet=""): filePath = 'dedupData.yaml' existingData = [] try: @@ -11,7 +11,7 @@ def write_dedup(result, id): pass yaml_data = { - 'id': id, + 'id': testSet + "/" + id, 'executedLinesByFile': {} } for file in result.measured_files(): @@ -21,4 +21,3 @@ def write_dedup(result, id): existingData.append(yaml_data) with open(filePath, 'w') as file: yaml.dump(existingData, file, sort_keys=False) - \ No newline at end of file From d68abb7329cdb5597d408b358fe25a287e5b8306 Mon Sep 17 00:00:00 2001 From: Asish Kumar <87874775+officialasishkumar@users.noreply.github.com> Date: Thu, 31 Jul 2025 16:11:18 +0530 Subject: [PATCH 17/18] feat: add support for dedup (#14) * feat: add support for dedup Signed-off-by: Asish Kumar * feat: add more dedup instruction Signed-off-by: Asish Kumar --------- Signed-off-by: Asish Kumar --- .github/workflows/main.yml | 25 +- README.md | 88 +++----- __init__.py | 4 - pyproject.toml | 34 +++ setup.py | 27 --- src/keploy/Keploy.py | 427 ----------------------------------- src/keploy/__init__.py | 4 - src/keploy/djangoCov.py | 20 -- src/keploy/fastApiCov.py | 24 -- src/keploy/flaskCov.py | 21 -- src/keploy/utils.py | 23 -- src/keploy_agent/__init__.py | 161 +++++++++++++ 12 files changed, 243 insertions(+), 615 deletions(-) delete mode 100644 __init__.py create mode 100644 pyproject.toml delete mode 100644 setup.py delete mode 100644 src/keploy/Keploy.py delete mode 100644 src/keploy/__init__.py delete mode 100644 src/keploy/djangoCov.py delete mode 100644 src/keploy/fastApiCov.py delete mode 100644 src/keploy/flaskCov.py delete mode 100644 src/keploy/utils.py create mode 100644 src/keploy_agent/__init__.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b4ea66c..5c5481d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,22 +11,21 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: 3.x - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine + - name: Install build dependencies + run: python -m pip install --upgrade build - - name: Build and publish to PyPI - run: | - python setup.py sdist bdist_wheel - twine upload dist/* - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + - name: Build package + run: python -m build + + - name: Publish package to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/README.md b/README.md index ccca761..28c1e22 100644 --- a/README.md +++ b/README.md @@ -1,82 +1,66 @@ -[![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen?logo=github)](CODE_OF_CONDUCT.md) -[![Slack](.github/slack.svg)](https://join.slack.com/t/keploy/shared_invite/zt-12rfbvc01-o54cOG0X1G6eVJTuI_orSA) -[![License](.github/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +# Keploy Python Coverage Agent -**Note** :- Issue Creation is disabled on this Repository, please visit [here](https://github.com/keploy/keploy/issues/new/choose) to submit issue. +The Keploy Python Coverage Agent is a lightweight, sidecar module designed to integrate with the Keploy integration testing platform (enterprise version). When imported into a Python application, it enables Keploy to capture and report code coverage on a per-test-case basis. -# Keploy Python-SDK -This is the client SDK for the [Keploy](https://github.com/keploy/keploy) testing platform. With the Python SDK, you can test both your existing unit test cases in Pytest and create new end-to-end test cases for your applications. -The **HTTP mocks/stubs and tests are the same format** and inter-exchangeable. +## Installation and Usage -## Contents -1. [Installation](#installation) -2. [Usage](#usage) -3. [Community support](#community-support) +Follow these steps to integrate the coverage agent into your Python project. -## Installation -1. First you need to install [Python(version 3 and above)](https://www.python.org/downloads/) +### Prerequisites -2. Install the Python-SDK and also Python's coverage library via pip. +You must have the `coverage` library installed in your project's Python environment. ```bash -pip install keploy coverage +pip install coverage ``` -3. Install Keploy from [here](https://github.com/keploy/keploy?tab=readme-ov-file#-quick-installation) +### Step 1: Install the Agent -## Usage -Keploy simplifies the testing process by seamlessly generating end-to-end test cases without the need to write unit test files and manage mocks/stubs. +Install the `keploy-agent` package into your virtual environment. If you are developing the agent, you can install it in an editable mode from its source directory: -Add a test file with the following code to the directory with all your existing tests. This will help us to get the coverage of Keploy's API tests along with the other unit tests. We can call this `test_keploy.py` - -```python -from keploy import run -def test_keploy(): - run("python3 -m coverage run --data-file=.coverage_data.keploy ") +```bash +pip install -e /path/to/keploy_agent ``` -> Note: If you face any problems with running the coverage library, you can refer to the documentation for the same [here](https://coverage.readthedocs.io/en/7.4.2/cmd.html#execution-coverage-run) +### Step 2: Integrate into Your Application -To ignore the coverage of python libraries which are included in the report by default, you need to create a `.coveragerc` file in the directory where you will ignore the /usr/ directory(only for Linux users). The contents of the file will be as follows: +To enable coverage tracking, import the `keploy_agent` module at the **very top** of your application's main entry point file (e.g., `app.py`, `main.py`). -```bash -[run] -omit = - /usr/* -sigterm = true -``` +It is crucial that this is one of the first imports, as this ensures the agent is initialized before your application code begins to execute. -Before starting your application, make sure that the **debug mode is set to False** in your application, for the coverage library to work properly. +**Example `app.py`:** -Now to run this testcase along with your another unit testcases, you can run the command below: +```python +import keploy_agent # <-- Add this line at the top +import os +from flask import Flask -```bash -keploy test -c "python3 -m coverage run -p --data-file=.coverage.unit -m pytest test_keploy.py " --delay 10 --coverage -``` +app = Flask(__name__) -Now, to combine the coverage from the unit tests, and Keploy's API tests, we can use the command below: +@app.get("/") +def hello(): + # This function will be tracked by the coverage agent + # when its endpoint is hit during a Keploy test. + return "Hello, World!" -```bash -python3 -m coverage combine +if __name__ == "__main__": + app.run() ``` -Make sure to run this command before starting a new test run to avoid getting multiple coverage files. +### Step 3: Run with Keploy -Finally, to generate the coverage report for the test run, you can run: +Now, you can run your application tests using the Keploy CLI. The agent will automatically connect with Keploy. ```bash -python3 -m coverage report +sudo -E keploy-enterprise test -c "python3 app.py" --language python --dedup ``` -and if you want the coverage in an html file, you can run: +Now you will see `dedupData.yaml` getting created. -```bash -python3 -m coverage html -``` +Run `sudo -E keploy-enterprise dedup` to get the tests which are duplicate in `duplicates.yaml` file -Hooray🎉! You've sucessfully got the coverage of your Keploy recorded api tests using Pytest. +In order to remove the duplicate tests, run the following command: -## Community support -We'd love to collaborate with you to make Keploy.io great. To get started: -* [Slack](https://join.slack.com/t/keploy/shared_invite/zt-12rfbvc01-o54cOG0X1G6eVJTuI_orSA) - Discussions with the community and the team. -* [GitHub](https://github.com/keploy/keploy/issues) - For bug reports and feature requests. \ No newline at end of file +```bash +sudo -E keploy-enterprise dedup --rm +``` diff --git a/__init__.py b/__init__.py deleted file mode 100644 index 13eed87..0000000 --- a/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .Keploy import run, RunOptions -from .djangoCov import DjangoCoverageMiddleware -from .fastApiCov import FastApiCoverageMiddleware -from .flaskCov import FlaskCoverageMiddleware diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..16377cd --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,34 @@ +# pyproject.toml + +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "keploy-agent" +version = "0.1.0" +authors = [ + { name="Keploy Team", email="hello@keploy.io" }, +] +description = "A side-effect agent for collecting Python code coverage during Keploy test runs." +readme = "README.md" +requires-python = ">=3.8" +license = { file="LICENSE" } +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python :: 3", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Intended Audience :: Developers", + "Topic :: Software Development :: Testing", + "Topic :: System :: Monitoring", +] + +# This is where you list the package's dependencies +dependencies = [ + "coverage>=7.0", +] + +[project.urls] +Homepage = "https://github.com/keploy/python-sdk" +"Bug Tracker" = "https://github.com/keploy/python-sdk/issues" diff --git a/setup.py b/setup.py deleted file mode 100644 index f6c2e7c..0000000 --- a/setup.py +++ /dev/null @@ -1,27 +0,0 @@ -from setuptools import setup, find_packages - -VERSION = '2.0.0-alpha36' -DESCRIPTION = 'Keploy' -LONG_DESCRIPTION = 'Keploy Python SDK' - -# Setting up -setup( - name="keploy", - version=VERSION, - author="Keploy Inc.", - author_email="hello@keploy.io", - description="Run your unit tests with Keploy", - long_description="This module allows you to run your unit tests along with pytest and get coverage for the same using Keploy", - packages=find_packages(where='src'), - package_dir={'': 'src'}, - keywords=['keploy', 'python', 'sdk'], - classifiers= [ - "Development Status :: 3 - Alpha", - "Intended Audience :: Education", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 3", - "Operating System :: MacOS :: MacOS X", - "Operating System :: Microsoft :: Windows", - "Operating System :: POSIX :: Linux", - ] -) \ No newline at end of file diff --git a/src/keploy/Keploy.py b/src/keploy/Keploy.py deleted file mode 100644 index 273cbf5..0000000 --- a/src/keploy/Keploy.py +++ /dev/null @@ -1,427 +0,0 @@ -import requests -import logging -import threading -import subprocess -import time -import os - -logging.basicConfig(level=logging.INFO) - -logger = logging.getLogger("Keploy") - -GRAPHQL_ENDPOINT = "/query" -HOST = "localhost" -PORT = 6789 - - -class TestRunStatus: - RUNNING = 1 - PASSED = 2 - FAILED = 3 - APP_HALTED = 4 - USER_ABORT = 5 - APP_FAULT = 6 - INTERNAL_ERR = 7 - - -def get_test_run_status(status_str): - status_mapping = { - "RUNNING": TestRunStatus.RUNNING, - "PASSED": TestRunStatus.PASSED, - "FAILED": TestRunStatus.FAILED, - "APP_HALTED": TestRunStatus.APP_HALTED, - "USER_ABORT": TestRunStatus.USER_ABORT, - "APP_FAULT": TestRunStatus.APP_FAULT, - "INTERNAL_ERR": TestRunStatus.INTERNAL_ERR, - } - return status_mapping.get(status_str) - - -class RunOptions: - def __init__(self, delay=10, debug=False, port=6789, path="."): - if isinstance(delay, int) and delay >= 0: - self.delay = delay - else: - raise ValueError("Delay must be a positive integer.") - self.debug = debug - if path != "": - self.path = path - # Ensure port is a positive integer - if isinstance(port, int) and port >= 0: - self.port = port - else: - raise ValueError("Port must be a positive integer.") - - -def run(run_cmd, run_options: RunOptions): - if run_options.port != 0: - global PORT - PORT = run_options.port - - # if run_options.debug: - # logger.setLevel(logging.DEBUG) - - # Starting keploy - start_keploy(run_cmd, run_options.delay, run_options.debug, PORT) - - # Delay for keploy to start - time.sleep(5) - - try: - # Fetching test sets - test_sets, err = fetch_test_sets() - if err is not None: - raise AssertionError(f"error fetching test sets: {err}") - - logger.debug(f"Test sets found: {test_sets}") - if len(test_sets) == 0: - raise AssertionError("No test sets found. Are you in the right directory?") - - # Start hooking for the application - appId, testRunId, err = start_hooks() - if err is not None: - raise AssertionError(f"error starting hooks: {err}") - - # Run for each test set. - for test_set in test_sets: - # Run test set - run_test_set(testRunId, test_set, appId) - # Start user application - start_user_application(appId) - - # path = os.path.abspath(run_options.path) - # report_path = os.path.join( - # path, "Keploy", "reports", testRunId, f"{test_set}-report.yaml" - # ) - # # check if the report file is created - # err = check_report_file(report_path, run_options.delay + 10) - # if err is not None: - # # Stop user application - # appErr = stop_user_application(appId) - # if appErr is not None: - # raise AssertionError(f"error stopping user application: {appErr}") - # logger.error( - # f"error getting report file: {testRunId}, testSetId: {test_set}. Error: {err}" - # ) - # continue - - time.sleep(run_options.delay + 2) - - # Wait for keploy to write initial data to report file - - logger.info(f"Running test set: {test_set} with testrun ID: {testRunId}") - status = None - while True: - status, err = fetch_test_set_status(testRunId, test_set) - if err is not None or status is None: - logger.error( - f"error getting test set status for testRunId: {testRunId}, testSetId: {test_set}. Error: {err}" - ) - break - - match status: - case TestRunStatus.RUNNING: - logger.info(f"Test set: {test_set} is still running") - case TestRunStatus.PASSED: - break - case TestRunStatus.FAILED: - break - case TestRunStatus.APP_HALTED: - break - case TestRunStatus.USER_ABORT: - break - case TestRunStatus.APP_FAULT: - break - case TestRunStatus.INTERNAL_ERR: - break - - # Check if the test set status has some internal error - if ( - status == None - or status == TestRunStatus.APP_HALTED - or status == TestRunStatus.USER_ABORT - or status == TestRunStatus.APP_FAULT - or status == TestRunStatus.INTERNAL_ERR - ): - logger.error(f"Test set: {test_set} failed with status: {status}") - - if status == TestRunStatus.FAILED: - logger.error(f"Test set: {test_set} failed") - elif status == TestRunStatus.PASSED: - logger.info(f"Test set: {test_set} passed") - - # Stop user application - err = stop_user_application(appId) - if err is not None: - raise AssertionError(f"error stopping user application: {err}") - time.sleep(5) # Wait for the user application to stop - finally: - # Stop keploy after running all test sets - stop_Keploy() - - -def start_keploy(runCmd, delay, debug, port): - thread = threading.Thread( - target=run_keploy, - args=( - runCmd, - delay, - debug, - port, - ), - daemon=False, - ) - thread.start() - return - - -def run_keploy(runCmd, delay, debug, port): - keployBin = "/usr/local/bin/keploybin" - - if not os.path.exists(keployBin): - print(f"Keploy binary doesn't exist, please install keploy") - return - - overallCmd = f'sudo -E env "PATH=$PATH" "{keployBin}" test -c "{runCmd}" --coverage --delay {delay} --port {port}' - if debug: - overallCmd += " --debug" - - logger.debug(f"Executing command: {overallCmd}") - - command = ["sh", "-c", overallCmd] - process = subprocess.Popen( - command, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - universal_newlines=True, - ) - - # Read and print the output - for line in process.stdout: - # logger.info(line, end="") - print(line, end="", flush=True) - - # Wait for the process to finish - process.wait() - - -def set_http_client(): - try: - url = f"http://{HOST}:{PORT}{GRAPHQL_ENDPOINT}" - logger.debug(f"Connecting to: {url}") - headers = { - "Content-Type": "application/json; charset=UTF-8", - "Accept": "application/json", - } - session = requests.Session() - return session, url, headers - except Exception as e: - logger.error(f"Error setting up HTTP client", e) - return None, None, None - - -def fetch_test_sets(): - sessions, url, headers = set_http_client() - if sessions is None or url is None or headers is None: - return [], "Failed to set up HTTP client" - - payload = {"query": "{ testSets }"} - - try: - response = sessions.post(url, headers=headers, json=payload, timeout=10) - logger.debug(f"Status code received: {response.status_code}") - - if response.ok: - res_body = response.json() - logger.debug(f"Response body received: {res_body}") - if res_body.get("data", {}) is None: - return [], res_body.get("errors", {}) - return res_body.get("data", {}).get("testSets"), None - else: - return [], None - except Exception as e: - logger.error("Error fetching test sets", e) - return [], None - - -def start_hooks(): - session, url, headers = set_http_client() - if session is None or url is None or headers is None: - return None, None, "Failed to set up HTTP client" - - payload = {"query": "mutation StartHooks { startHooks { appId testRunId } }"} - - try: - response = session.post(url, headers=headers, json=payload, timeout=10) - logger.debug(f"Status code received: {response.status_code}") - - if response.ok: - res_body = response.json() - logger.debug(f"Response body received: {res_body}") - if res_body.get("data", {}) is None: - return None, None, res_body.get("errors", {}) - - start_hooks_data = res_body.get("data", {}).get("startHooks") - if start_hooks_data is None: - return None, None, f"Failed to get start Hooks data" - - appId = start_hooks_data.get("appId") - testRunId = start_hooks_data.get("testRunId") - return appId, testRunId, None - else: - return ( - None, - None, - f"Failed to start hooks. Status code: {response.status_code}", - ) - except Exception as e: - logger.error(f"Error starting hooks: {e}") - return None, None, f"Error starting hooks: {e}" - - -def run_test_set(testRunId, testSetId, appId): - session, url, headers = set_http_client() - if session is None or url is None or headers is None: - return False, "Failed to set up HTTP client" - - payload = { - "query": f'mutation RunTestSet {{ runTestSet(testSetId: "{testSetId}", testRunId: "{testRunId}", appId: {appId}) }}' - } - - try: - response = session.post(url, headers=headers, json=payload, timeout=10) - logger.debug(f"Status code received: {response.status_code}") - - if response.ok: - res_body = response.json() - logger.debug(f"Response body received: {res_body}") - if res_body.get("data", {}) is None: - return False, res_body.get("errors", {}) - return res_body.get("data", {}).get("runTestSet"), None - else: - return False, f"Failed to run test set. Status code: {response.status_code}" - except Exception as e: - logger.error(f"Error running test set: {e}") - return False, f"Error running test set: {e}" - - -def start_user_application(appId): - session, url, headers = set_http_client() - if session is None or url is None or headers is None: - return False, "Failed to set up HTTP client" - - payload = {"query": f"mutation StartApp {{ startApp(appId: {appId}) }}"} - - try: - response = session.post(url, headers=headers, json=payload, timeout=10) - logger.debug(f"Status code received: {response.status_code}") - - if response.ok: - res_body = response.json() - logger.debug(f"Response body received: {res_body}") - if res_body.get("data", {}) is None: - return False, res_body.get("errors", {}) - return res_body.get("data", {}).get("startApp"), None - else: - return ( - False, - f"Failed to start user application. Status code: {response.status_code}", - ) - except Exception as e: - logger.error(f"Error starting user application: {e}") - return False, f"Error starting user application: {e}" - - -# def check_report_file(report_path, timeout): - -# logger.debug(f"Checking report file at: {report_path}") - -# start_time = time.time() -# while time.time() - start_time < timeout: -# if os.path.exists(report_path): -# return None # Report file found -# time.sleep(1) # Wait for 1 second before checking again - -# return f"Report file not created within {timeout} seconds" - - -def fetch_test_set_status(testRunId, testSetId): - session, url, headers = set_http_client() - if session is None or url is None or headers is None: - return None, "Failed to set up HTTP client" - - payload = { - "query": f'query GetTestSetStatus {{ testSetStatus(testRunId: "{testRunId}", testSetId: "{testSetId}") {{ status }} }}' - } - - try: - response = session.post(url, headers=headers, json=payload, timeout=10) - logger.debug(f"Status code received: {response.status_code}") - - if response.ok: - res_body = response.json() - logger.debug(f"Response body received: {res_body}") - if res_body.get("data", {}) is None: - return None, res_body.get("errors", {}) - test_set_status = res_body.get("data", {}).get("testSetStatus", {}) - status = test_set_status.get("status") - return get_test_run_status(status), None - else: - return ( - None, - f"Failed to fetch test set status. Status code: {response.status_code}", - ) - except Exception as e: - logger.error(f"Error fetching test set status: {e}") - return None, f"Error fetching test set status: {e}" - - -def stop_user_application(appId): - session, url, headers = set_http_client() - if session is None or url is None or headers is None: - return "Failed to set up HTTP client" - - payload = {"query": f"mutation StopApp {{ stopApp(appId: {appId}) }}"} - - try: - response = session.post(url, headers=headers, json=payload, timeout=10) - logger.debug(f"Status code received: {response.status_code}") - - if response.ok: - res_body = response.json() - logger.debug(f"Response body received: {res_body}") - if res_body.get("data", {}) is None: - return res_body.get("errors", {}) - stop_app_result = res_body.get("data", {}).get("stopApp") - logger.debug(f"stopApp result: {stop_app_result}") - else: - return ( - f"Failed to stop user application. Status code: {response.status_code}" - ) - except Exception as e: - logger.error(f"Error stopping user application: {e}") - return f"Error stopping user application: {e}" - - return None - - -def stop_Keploy(): - session, url, headers = set_http_client() - if session is None or url is None or headers is None: - return "Failed to set up HTTP client" - - payload = {"query": "mutation { stopHooks }"} - - try: - response = session.post(url, headers=headers, json=payload, timeout=10) - logger.debug(f"Status code received: {response.status_code}") - - if response.ok: - res_body = response.json() - logger.debug(f"Response body received: {res_body}") - if res_body.get("data", {}) is None: - return res_body.get("errors", {}) - return res_body.get("data", {}).get("stopHooks") - except Exception as e: - logger.error(f"Error stopping hooks: {e}") - return None diff --git a/src/keploy/__init__.py b/src/keploy/__init__.py deleted file mode 100644 index 13eed87..0000000 --- a/src/keploy/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .Keploy import run, RunOptions -from .djangoCov import DjangoCoverageMiddleware -from .fastApiCov import FastApiCoverageMiddleware -from .flaskCov import FlaskCoverageMiddleware diff --git a/src/keploy/djangoCov.py b/src/keploy/djangoCov.py deleted file mode 100644 index 2ceda30..0000000 --- a/src/keploy/djangoCov.py +++ /dev/null @@ -1,20 +0,0 @@ -import coverage - -from .utils import write_dedup - -class DjangoCoverageMiddleware: - def __init__(self, get_response): - self.get_response = get_response - - def __call__(self, request): - id = request.headers.get('KEPLOY-TEST-ID') - testSet = request.headers.get('KEPLOY-TEST-SET-ID') - if id == None: - return self.get_response(request) - cov = coverage.Coverage(cover_pylib=False) - cov.start() - response = self.get_response(request) - cov.stop() - result = cov.get_data() - write_dedup(result, id, testSet) - return response diff --git a/src/keploy/fastApiCov.py b/src/keploy/fastApiCov.py deleted file mode 100644 index f041453..0000000 --- a/src/keploy/fastApiCov.py +++ /dev/null @@ -1,24 +0,0 @@ -import coverage -from fastapi import Request -from starlette.middleware.base import BaseHTTPMiddleware - -from .utils import write_dedup - -class FastApiCoverageMiddleware(BaseHTTPMiddleware): - async def dispatch(self, request: Request, call_next): - id = request.headers.get('KEPLOY-TEST-ID') - testSet = request.headers.get('KEPLOY-TEST-SET-ID') - if id is None: - response = await call_next(request) - return response - global_coverage = coverage.Coverage.current() - if global_coverage is not None: - response = await call_next(request) - return response - cov = coverage.Coverage(cover_pylib=False) - cov.start() - response = await call_next(request) - cov.stop() - result = cov.get_data() - write_dedup(result, id, testSet) - return response diff --git a/src/keploy/flaskCov.py b/src/keploy/flaskCov.py deleted file mode 100644 index fb9f8c8..0000000 --- a/src/keploy/flaskCov.py +++ /dev/null @@ -1,21 +0,0 @@ -import coverage -from werkzeug.wrappers import Request -from .utils import write_dedup - -class FlaskCoverageMiddleware: - def __init__(self, app): - self.app = app - - def __call__(self, environ, start_response): - request = Request(environ) - id = request.headers.get("Keploy-Test-Id") - testSet = request.headers.get("Keploy-Test-Set-Id") - if id == None: - return self.app(environ, start_response) - cov = coverage.Coverage(cover_pylib=False) - cov.start() - response = self.app(environ, start_response) - cov.stop() - result = cov.get_data() - write_dedup(result, id, testSet) - return response diff --git a/src/keploy/utils.py b/src/keploy/utils.py deleted file mode 100644 index 435c99e..0000000 --- a/src/keploy/utils.py +++ /dev/null @@ -1,23 +0,0 @@ -import yaml - -def write_dedup(result, id, testSet=""): - filePath = 'dedupData.yaml' - existingData = [] - try: - with open(filePath, 'r') as file: - existingData=yaml.safe_load(file) - except: - with open(filePath, 'w') as file: - pass - - yaml_data = { - 'id': testSet + "/" + id, - 'executedLinesByFile': {} - } - for file in result.measured_files(): - yaml_data['executedLinesByFile'][file] = result.lines(file) - if existingData is None: - existingData=[] - existingData.append(yaml_data) - with open(filePath, 'w') as file: - yaml.dump(existingData, file, sort_keys=False) diff --git a/src/keploy_agent/__init__.py b/src/keploy_agent/__init__.py new file mode 100644 index 0000000..8383cd9 --- /dev/null +++ b/src/keploy_agent/__init__.py @@ -0,0 +1,161 @@ +import socket +import threading +import os +import json +import logging +import coverage + +# --- Configuration --- +# Set up logging to be informative +logging.basicConfig(level=logging.INFO, format='[Keploy Agent] %(asctime)s - %(levelname)s - %(message)s') + +# Define socket paths, same as the Go implementation +CONTROL_SOCKET_PATH = "/tmp/coverage_control.sock" +DATA_SOCKET_PATH = "/tmp/coverage_data.sock" + +# --- Global State --- +# This lock protects access to the current_test_id +control_lock = threading.Lock() +# Stores the ID of the test case currently being recorded +current_test_id = None + +cov = coverage.Coverage(data_file=None, auto_data=True) + + +def handle_control_request(conn: socket.socket): + """ + Parses commands from Keploy ("START testID", "END testID") sent over the socket. + This runs in its own thread for each connection. + """ + global current_test_id + try: + with conn: + reader = conn.makefile('r') + command = reader.readline() + if not command: + return + + parts = command.strip().split(" ", 1) + if len(parts) != 2: + logging.error(f"Invalid command format: '{command.strip()}'") + return + + action, test_id = parts[0], parts[1] + + with control_lock: + if action == "START": + logging.info(f"Received START for test: {test_id}") + current_test_id = test_id + cov.erase() + cov.start() + + elif action == "END": + if current_test_id != test_id: + logging.warning( + f"Mismatched END command. Expected '{current_test_id}', got '{test_id}'. " + "Skipping coverage report." + ) + return + + logging.info(f"Received END for test: {test_id}. Reporting coverage.") + cov.stop() + cov.save() + + try: + report_coverage(test_id) + except Exception as e: + logging.error(f"Failed to report coverage for test {test_id}: {e}", exc_info=True) + + current_test_id = None + # Acknowledge the command + conn.sendall(b"ACK\n") + + else: + logging.warning(f"Unrecognized command: {action}") + + except Exception as e: + logging.error(f"Error handling control request: {e}", exc_info=True) + + +def report_coverage(test_id: str): + """ + Gathers, processes, and sends the coverage data to the data socket. + """ + data = cov.get_data() + if not data: + logging.warning("Coverage data is empty. No report will be sent.") + return + + executed_lines_by_file = {} + for filename in data.measured_files(): + abs_path = os.path.abspath(filename) + lines = data.lines(filename) + if lines: + executed_lines_by_file[abs_path] = lines + + if not executed_lines_by_file: + logging.warning(f"No covered lines were found for test {test_id}. The report will be empty.") + + payload = { + "id": test_id, + "executedLinesByFile": executed_lines_by_file, + } + + try: + json_data = json.dumps(payload).encode('utf-8') + send_to_data_socket(json_data) + logging.info(f"Successfully sent coverage report for test: {test_id}") + except Exception as e: + logging.error(f"Failed to serialize or send coverage data: {e}", exc_info=True) + + +def send_to_data_socket(data: bytes): + """Connects to the Keploy data socket and writes the JSON payload.""" + try: + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: + s.connect(DATA_SOCKET_PATH) + s.sendall(data) + except Exception as e: + logging.error(f"Could not connect or send to data socket at {DATA_SOCKET_PATH}: {e}") + raise + + +def start_control_server(): + """ + Sets up and runs the Unix socket server that listens for commands from Keploy. + This function runs in a background thread. + """ + if os.path.exists(CONTROL_SOCKET_PATH): + try: + os.remove(CONTROL_SOCKET_PATH) + except OSError as e: + logging.error(f"Failed to remove old control socket: {e}") + return + + server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + server.bind(CONTROL_SOCKET_PATH) + server.listen() + logging.info(f"Control server listening on {CONTROL_SOCKET_PATH}") + + while True: + conn, _ = server.accept() + handler_thread = threading.Thread(target=handle_control_request, args=(conn,)) + handler_thread.start() + + except Exception as e: + logging.error(f"Control server failed: {e}", exc_info=True) + finally: + server.close() + logging.info("Control server shut down.") + + +# --- SIDE-EFFECT ON IMPORT --- +# This is the code that runs automatically when `import keploy_agent` is executed. +logging.info("Initializing...") + +# Start the control server in a background daemon thread. +control_thread = threading.Thread(target=start_control_server, daemon=True) +control_thread.start() + +logging.info("Agent initialized and control server started in the background.") From 3dfce52a2af663cb211e5f212127c8f567f634e4 Mon Sep 17 00:00:00 2001 From: Asish Kumar <87874775+officialasishkumar@users.noreply.github.com> Date: Fri, 1 Aug 2025 17:19:41 +0530 Subject: [PATCH 18/18] Fix/dedup without ext lib (#15) * fix: agent directory Signed-off-by: Asish * fix: only record coverage in the source dir Signed-off-by: Asish * feat: add support for dedup wihtout ext lib Signed-off-by: Asish --------- Signed-off-by: Asish --- requirements.txt | 1 + src/keploy_agent/__init__.py | 284 +++++++++++++++++------------------ 2 files changed, 139 insertions(+), 146 deletions(-) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ + diff --git a/src/keploy_agent/__init__.py b/src/keploy_agent/__init__.py index 8383cd9..121fc71 100644 --- a/src/keploy_agent/__init__.py +++ b/src/keploy_agent/__init__.py @@ -1,161 +1,153 @@ -import socket -import threading -import os -import json -import logging -import coverage - -# --- Configuration --- -# Set up logging to be informative -logging.basicConfig(level=logging.INFO, format='[Keploy Agent] %(asctime)s - %(levelname)s - %(message)s') - -# Define socket paths, same as the Go implementation -CONTROL_SOCKET_PATH = "/tmp/coverage_control.sock" -DATA_SOCKET_PATH = "/tmp/coverage_data.sock" - -# --- Global State --- -# This lock protects access to the current_test_id -control_lock = threading.Lock() -# Stores the ID of the test case currently being recorded -current_test_id = None - -cov = coverage.Coverage(data_file=None, auto_data=True) - - -def handle_control_request(conn: socket.socket): - """ - Parses commands from Keploy ("START testID", "END testID") sent over the socket. - This runs in its own thread for each connection. - """ - global current_test_id +""" +Keploy Python agent – always-on trace, per-test diff, std-lib filtered. +""" + +from __future__ import annotations +import os, sys, json, time, socket, threading, logging, trace +import sysconfig, site, pathlib +from typing import Dict, Tuple + +# ------------------------------------------------ configuration +CONTROL_SOCK = "/tmp/coverage_control.sock" +DATA_SOCK = "/tmp/coverage_data.sock" + +APP_ROOT = pathlib.Path(os.getenv("KEPLOY_APP_SOURCE_DIR", os.getcwd())).resolve() +AGENT_DIR = pathlib.Path(__file__).parent.resolve() +STDLIB = pathlib.Path(sysconfig.get_paths()["stdlib"]).resolve() +SITE_PKGS = [pathlib.Path(p).resolve() for p in site.getsitepackages()] + + +logging.basicConfig(level=logging.INFO, + format='[Keploy Agent] %(asctime)s - %(message)s') + +# ------------------------------------------------ global tracer +_tracer = trace.Trace(count=1, trace=0) +sys.settrace(_tracer.globaltrace) # main thread +threading.settrace(_tracer.globaltrace) # all future threads +logging.info("Global tracer started for every thread") + +# ------------------------------------------------ agent state +lock = threading.Lock() +current_id = None # active test-case id +baseline_counts: Dict[Tuple[str, int], int] = {} # empty after clear +baseline_tids: set[int] = set() + + +# ------------------------------------------------ helpers +def _copy_counts_once() -> dict[tuple[str, int], int]: + while True: + try: return dict(_tracer.results().counts) + except RuntimeError: continue # mutated – retry + +def _stable_snapshot(max_wait: float = .5) -> dict[tuple[str, int], int]: + start = time.time(); snap = _copy_counts_once() + while True: + time.sleep(0.07) + snap2 = _copy_counts_once() + if snap2 == snap or (time.time() - start) >= max_wait: + return snap2 + snap = snap2 + +def _ack(c: socket.socket): + try: c.sendall(b"ACK\n") + except OSError: pass + +def _is_app_file(raw: str, p: pathlib.Path) -> bool: + if " before.get(key, 0): + yield key + +def _emit(test_id: str): + after = _stable_snapshot() + data = {} + for (raw, line) in _diff(after, baseline_counts): + p = pathlib.Path(raw).resolve() + if _is_app_file(raw, p): + data.setdefault(str(p), []).append(line) + + payload = json.dumps({"id": test_id, + "executedLinesByFile": data}, + separators=(",", ":")).encode() + + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: + s.connect(DATA_SOCK); s.sendall(payload) + + logging.info(f"[{test_id}] sent {len(data)} file(s) / " + f"{sum(len(v) for v in data.values())} line(s)") + +# ------------------------------------------------ per-connection handler +def _handle(conn: socket.socket): + global current_id, baseline_counts, baseline_tids + fp = conn.makefile("r") try: - with conn: - reader = conn.makefile('r') - command = reader.readline() - if not command: - return + cmd = fp.readline().strip() + if not cmd: return + action, test_id = cmd.split(" ", 1) - parts = command.strip().split(" ", 1) - if len(parts) != 2: - logging.error(f"Invalid command format: '{command.strip()}'") - return + with lock: + if action == "START": + logging.info(f"START {test_id}") - action, test_id = parts[0], parts[1] - - with control_lock: - if action == "START": - logging.info(f"Received START for test: {test_id}") - current_test_id = test_id - cov.erase() - cov.start() - - elif action == "END": - if current_test_id != test_id: - logging.warning( - f"Mismatched END command. Expected '{current_test_id}', got '{test_id}'. " - "Skipping coverage report." - ) - return - - logging.info(f"Received END for test: {test_id}. Reporting coverage.") - cov.stop() - cov.save() - - try: - report_coverage(test_id) - except Exception as e: - logging.error(f"Failed to report coverage for test {test_id}: {e}", exc_info=True) - - current_test_id = None - # Acknowledge the command - conn.sendall(b"ACK\n") - - else: - logging.warning(f"Unrecognized command: {action}") + _tracer.results().counts.clear() # start from 0 hits + baseline_counts = {} # diff against empty dict + baseline_tids = _current_tids() # remember threads - except Exception as e: - logging.error(f"Error handling control request: {e}", exc_info=True) - - -def report_coverage(test_id: str): - """ - Gathers, processes, and sends the coverage data to the data socket. - """ - data = cov.get_data() - if not data: - logging.warning("Coverage data is empty. No report will be sent.") - return - - executed_lines_by_file = {} - for filename in data.measured_files(): - abs_path = os.path.abspath(filename) - lines = data.lines(filename) - if lines: - executed_lines_by_file[abs_path] = lines - - if not executed_lines_by_file: - logging.warning(f"No covered lines were found for test {test_id}. The report will be empty.") - - payload = { - "id": test_id, - "executedLinesByFile": executed_lines_by_file, - } + current_id = test_id + _ack(conn) + return - try: - json_data = json.dumps(payload).encode('utf-8') - send_to_data_socket(json_data) - logging.info(f"Successfully sent coverage report for test: {test_id}") - except Exception as e: - logging.error(f"Failed to serialize or send coverage data: {e}", exc_info=True) + if action == "END": + logging.info(f"END {test_id}") + if test_id == current_id: + _wait_for_worker_exit(baseline_tids) # <- NEW + time.sleep(0.02) + _emit(test_id) + current_id = None + _ack(conn) + return + logging.warning(f"Unknown command: {action}") + _ack(conn) -def send_to_data_socket(data: bytes): - """Connects to the Keploy data socket and writes the JSON payload.""" - try: - with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: - s.connect(DATA_SOCKET_PATH) - s.sendall(data) except Exception as e: - logging.error(f"Could not connect or send to data socket at {DATA_SOCKET_PATH}: {e}") - raise - - -def start_control_server(): - """ - Sets up and runs the Unix socket server that listens for commands from Keploy. - This function runs in a background thread. - """ - if os.path.exists(CONTROL_SOCKET_PATH): - try: - os.remove(CONTROL_SOCKET_PATH) - except OSError as e: - logging.error(f"Failed to remove old control socket: {e}") - return - - server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + logging.error(f"Handler error: {e}", exc_info=True); _ack(conn) + finally: + try: fp.close(); conn.close() + except Exception: pass + +# ------------------------------------------------ control server +def _server(): + if os.path.exists(CONTROL_SOCK): os.remove(CONTROL_SOCK) + srv = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + srv.bind(CONTROL_SOCK); srv.listen() + logging.info(f"Keploy control server at {CONTROL_SOCK}") try: - server.bind(CONTROL_SOCKET_PATH) - server.listen() - logging.info(f"Control server listening on {CONTROL_SOCKET_PATH}") - while True: - conn, _ = server.accept() - handler_thread = threading.Thread(target=handle_control_request, args=(conn,)) - handler_thread.start() - - except Exception as e: - logging.error(f"Control server failed: {e}", exc_info=True) + c, _ = srv.accept() + threading.Thread(target=_handle, args=(c,), daemon=True).start() finally: - server.close() - logging.info("Control server shut down.") + srv.close() +threading.Thread(target=_server, daemon=False, + name="KeployControlServer").start() +logging.info("Keploy agent ready (always-on tracer, clean start per test)") -# --- SIDE-EFFECT ON IMPORT --- -# This is the code that runs automatically when `import keploy_agent` is executed. -logging.info("Initializing...") -# Start the control server in a background daemon thread. -control_thread = threading.Thread(target=start_control_server, daemon=True) -control_thread.start() +# ---------------------------------------------------------------- thread helper +def _current_tids() -> set[int]: + return {t.ident for t in threading.enumerate() if t.ident is not None} -logging.info("Agent initialized and control server started in the background.") +def _wait_for_worker_exit(baseline: set[int], timeout: float = 1.0): + """Block until no extra threads (vs. baseline) remain, or timeout.""" + deadline = time.time() + timeout + while time.time() < deadline: + if _current_tids() <= baseline: # no extra threads left + return + time.sleep(0.02) # wait 20 ms and retry