diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 01a388787..778992233 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -39,6 +39,10 @@ jobs: - "twine_check" - "daphne" - "no_optional_packages" + - "perf_asgi" + - "perf_hello" + - "perf_media" + - "perf_query" # TODO(kgriffs): Re-enable once hug has a chance to address # breaking changes in Falcon 3.0 # - "hug" @@ -82,22 +86,31 @@ jobs: - name: Set up Python uses: actions/setup-python@v2.1.4 - if: ${{ matrix.toxenv != 'py35' }} + if: ${{ matrix.toxenv != 'py35' && matrix.toxenv != 'perf_asgi' && matrix.toxenv != 'perf_hello' && matrix.toxenv != 'perf_media' && matrix.toxenv != 'perf_query' }} with: python-version: ${{ matrix.python-version }} + - name: Set up Python 3.5.2 + if: ${{ matrix.toxenv == 'py35' }} + run: | + sudo apt-get update + sudo apt-get install -y build-essential curl python3.5 python3.5-dev + python3.5 --version + - name: Set up Python 3.8 uses: actions/setup-python@v2.1.4 if: ${{ matrix.toxenv == 'py35' }} with: python-version: 3.8 - - name: Set up Python 3.5.2 - if: ${{ matrix.toxenv == 'py35' }} + - name: Set up Python 3.8 (Ubuntu 20.04 build) + if: ${{ matrix.toxenv == 'perf_asgi' || matrix.toxenv == 'perf_hello' || matrix.toxenv == 'perf_media' || matrix.toxenv == 'perf_query' }} run: | sudo apt-get update - sudo apt-get install -y build-essential curl python3.5 python3.5-dev - python3.5 --version + sudo apt-get install -y build-essential curl python3.8 python3.8-dev python3.8-distutils python-is-python3 + curl --silent https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py + sudo python3.8 /tmp/get-pip.py + sudo pip install coverage tox - name: Install smoke test dependencies if: ${{ matrix.toxenv == 'py38_smoke' || matrix.toxenv == 'py38_smoke_cython' }} @@ -105,10 +118,20 @@ jobs: sudo apt-get update sudo apt-get install -y libunwind-dev + - name: Install valgrind + if: ${{ matrix.toxenv == 'perf_asgi' || matrix.toxenv == 'perf_hello' || matrix.toxenv == 'perf_media' || matrix.toxenv == 'perf_query' }} + run: | + sudo apt-get update + sudo apt-get install -y valgrind + - name: Install dependencies + if: ${{ matrix.toxenv != 'perf_asgi' && matrix.toxenv != 'perf_hello' && matrix.toxenv != 'perf_media' && matrix.toxenv != 'perf_query' }} run: | python -m pip install --upgrade pip pip install coverage tox + + - name: Print versions + run: | python --version pip --version tox --version diff --git a/.gitignore b/.gitignore index aea685471..f1925ca6a 100644 --- a/.gitignore +++ b/.gitignore @@ -52,6 +52,9 @@ dash # System .DS_Store +# Valgrind artefacts +perf/cachegrind.out.* + # VIM swap files .*.swp diff --git a/perf/BASELINE.yaml b/perf/BASELINE.yaml new file mode 100644 index 000000000..e852da0df --- /dev/null +++ b/perf/BASELINE.yaml @@ -0,0 +1,49 @@ +cpython_38: + asgi: + expected: + cost: 106660 + variation: 0.0001 + points: + - 10000 + - 15000 + - 20000 + - 25000 + tolerance: + - -0.002 + - +0.001 + hello: + expected: + cost: 75950 + variation: 0.0001 + points: + - 10000 + - 15000 + - 20000 + - 25000 + tolerance: + - -0.002 + - +0.001 + media: + expected: + cost: 198740 + variation: 0.0001 + points: + - 5000 + - 7500 + - 10000 + - 12500 + tolerance: + - -0.002 + - +0.001 + query: + expected: + cost: 182580 + variation: 0.0001 + points: + - 5000 + - 7500 + - 10000 + - 12500 + tolerance: + - -0.002 + - +0.001 diff --git a/perf/cachegrind.py b/perf/cachegrind.py new file mode 100644 index 000000000..ae8fd6c47 --- /dev/null +++ b/perf/cachegrind.py @@ -0,0 +1,136 @@ +""" +As per the original author's recommendation, this script was simply copied from +https://github.com/pythonspeed/cachegrind-benchmarking @ 32d26691. + +See also this awesome article by Itamar Turner-Trauring: +https://pythonspeed.com/articles/consistent-benchmarking-in-ci/. + +The original file content follows below. + +------------------------------------------------------------------------------- + +Proof-of-concept: run_with_cachegrind a program under Cachegrind, combining all the various +metrics into one single performance metric. + +Requires Python 3. + +License: https://opensource.org/licenses/MIT + +## Features + +* Disables ASLR. +* Sets consistent cache sizes. +* Calculates a combined performance metric. + +For more information see the detailed write up at: + +https://pythonspeed.com/articles/consistent-benchmarking-in-ci/ + +## Usage + +This script has no compatibility guarnatees, I recommend copying it into your +repository. To use: + +$ python3 cachegrind.py ./yourprogram --yourparam=yourvalues + +If you're benchmarking Python, make sure to set PYTHONHASHSEED to a fixed value +(e.g. `export PYTHONHASHSEED=1234`). Other languages may have similar +requirements to reduce variability. + +The last line printed will be a combined performance metric, but you can tweak +the script to extract more info, or use it as a library. + +Copyright © 2020, Hyphenated Enterprises LLC. +""" + +from typing import List, Dict +from subprocess import check_call, check_output +import sys +from tempfile import NamedTemporaryFile + +ARCH = check_output(["uname", "-m"]).strip() + + +def run_with_cachegrind(args_list: List[str]) -> Dict[str, int]: + """ + Run the the given program and arguments under Cachegrind, parse the + Cachegrind specs. + + For now we just ignore program output, and in general this is not robust. + """ + temp_file = NamedTemporaryFile("r+") + check_call([ + # Disable ASLR: + "setarch", + ARCH, + "-R", + "valgrind", + "--tool=cachegrind", + # Set some reasonable L1 and LL values, based on Haswell. You can set + # your own, important part is that they are consistent across runs, + # instead of the default of copying from the current machine. + "--I1=32768,8,64", + "--D1=32768,8,64", + "--LL=8388608,16,64", + "--cachegrind-out-file=" + temp_file.name, + ] + args_list) + return parse_cachegrind_output(temp_file) + + +def parse_cachegrind_output(temp_file): + # Parse the output file: + lines = iter(temp_file) + for line in lines: + if line.startswith("events: "): + header = line[len("events: "):].strip() + break + for line in lines: + last_line = line + assert last_line.startswith("summary: ") + last_line = last_line[len("summary:"):].strip() + return dict(zip(header.split(), [int(i) for i in last_line.split()])) + + +def get_counts(cg_results: Dict[str, int]) -> Dict[str, int]: + """ + Given the result of run_with_cachegrind(), figure out the parameters we will use for final + estimate. + + We pretend there's no L2 since Cachegrind doesn't currently support it. + + Caveats: we're not including time to process instructions, only time to + access instruction cache(s), so we're assuming time to fetch and run_with_cachegrind + instruction is the same as time to retrieve data if they're both to L1 + cache. + """ + result = {} + d = cg_results + + ram_hits = d["DLmr"] + d["DLmw"] + d["ILmr"] + + l3_hits = d["I1mr"] + d["D1mw"] + d["D1mr"] - ram_hits + + total_memory_rw = d["Ir"] + d["Dr"] + d["Dw"] + l1_hits = total_memory_rw - l3_hits - ram_hits + assert total_memory_rw == l1_hits + l3_hits + ram_hits + + result["l1"] = l1_hits + result["l3"] = l3_hits + result["ram"] = ram_hits + + return result + + +def combined_instruction_estimate(counts: Dict[str, int]) -> int: + """ + Given the result of run_with_cachegrind(), return estimate of total time to run_with_cachegrind. + + Multipliers were determined empirically, but some research suggests they're + a reasonable approximation for cache time ratios. L3 is probably too low, + but then we're not simulating L2... + """ + return counts["l1"] + (5 * counts["l3"]) + (35 * counts["ram"]) + + +if __name__ == "__main__": + print(combined_instruction_estimate(get_counts(run_with_cachegrind(sys.argv[1:])))) diff --git a/perf/conftest.py b/perf/conftest.py new file mode 100644 index 000000000..bdebf02af --- /dev/null +++ b/perf/conftest.py @@ -0,0 +1,123 @@ +# Copyright 2020 by Vytautas Liuolia. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +import pathlib +import platform +import subprocess +import sys + +import numpy +import pytest +import yaml + +HERE = pathlib.Path(__file__).resolve().parent + + +def _platform(): + # TODO(vytas): Add support for Cython, PyPy etc. + label = platform.python_implementation().lower() + version = ''.join(platform.python_version_tuple()[:2]) + return f'{label}_{version}' + + +class Gauge: + GAUGE_ENV = { + 'LC_ALL': 'en_US.UTF-8', + 'LANG': 'en_US.UTF-8', + 'PYTHONHASHSEED': '0', + 'PYTHONIOENCODING': 'utf-8', + } + + def __init__(self, metric): + with open(HERE / 'BASELINE.yaml', encoding='utf-8') as baseline: + config = yaml.safe_load(baseline) + + platform_label = _platform() + platform_spec = config.get(platform_label) + assert platform_spec, ( + f'no performance baseline established for {platform_label} yet', + ) + + self._metric = metric + self._spec = platform_spec[metric] + + def _fit_data(self, iterations, times): + # NOTE(vytas): Least-squares fitting solution straight from + # https://numpy.org/doc/stable/reference/generated/numpy.linalg.lstsq.html + x = numpy.array(iterations, dtype=float) + y = numpy.array(times, dtype=float) + A = numpy.vstack([x, numpy.ones(len(x))]).T + (cost, _), residuals, _, _ = numpy.linalg.lstsq(A, y, rcond=None) + + N = len(times) + rmsd = math.sqrt(residuals / (N - 2)) + cv_rmsd = rmsd / numpy.mean(y) + return (cost, cv_rmsd) + + def _measure_data_point(self, number): + command = ( + sys.executable, + 'cachegrind.py', + sys.executable, + '-m', + f'metrics.{self._metric}', + str(number), + ) + print('\n\nrunning cachegrind:', ' '.join(command), '\n') + output = subprocess.check_output(command, cwd=HERE, env=self.GAUGE_ENV) + output = output.decode().strip() + print(f'\n{output}') + + return int(output.strip()) + + def measure(self): + iterations = self._spec['points'] + + times = [] + for number in iterations: + times.append(self._measure_data_point(number)) + + cost, cv_rmsd = self._fit_data(iterations, times) + print('\nestimated cost per iteration:', cost) + print('estimated CV of RMSD:', cv_rmsd) + + expected_cost = self._spec['expected']['cost'] + expected_variation = self._spec['expected']['variation'] + tolerance = self._spec['tolerance'] + + assert cost > expected_cost / 10, ( + 'estimated cost per iteration is very low; is the metric broken?') + assert cv_rmsd < expected_variation, ( + 'cachegrind results vary too much between iterations') + + assert cost > expected_cost * (1 + min(tolerance)), ( + 'too good! please revise the baseline if you optimized the code') + assert cost < expected_cost * (1 + max(tolerance)), ( + 'performance regression measured!') + + +def pytest_configure(config): + config.addinivalue_line('markers', 'asgi: "asgi" performance metric') + config.addinivalue_line('markers', 'hello: "hello" performance metric') + config.addinivalue_line('markers', 'media: "media" performance metric') + config.addinivalue_line('markers', 'query: "query" performance metric') + + +@pytest.fixture() +def gauge(): + def _method(metric): + Gauge(metric).measure() + + return _method diff --git a/perf/metrics/__init__.py b/perf/metrics/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/perf/metrics/asgi.py b/perf/metrics/asgi.py new file mode 100644 index 000000000..a4c2e3789 --- /dev/null +++ b/perf/metrics/asgi.py @@ -0,0 +1,62 @@ +import timeit + +import falcon.asgi +from .common import get_work_factor + +SCOPE_BOILERPLATE = { + 'asgi': {'version': '3.0', 'spec_version': '2.1'}, + 'headers': [[b'host', b'falconframework.org']], + 'http_version': '1.1', + 'method': 'GET', + 'path': '/', + 'query_string': b'', + 'server': ['falconframework.org', 80], + 'type': 'http', +} + +RECEIVE_EVENT = { + 'type': 'http.request', + 'body': b'', + 'more_body': False, +} + + +class AsyncGreeter: + async def on_get(self, req, resp): + resp.content_type = 'text/plain; charset=utf-8' + resp.text = 'Hello, World!\n' + + +def create_app(): + app = falcon.asgi.App() + app.add_route('/', AsyncGreeter()) + return app + + +def run(app, expected_status, expected_body, number=None): + async def receive(): + return receive_event + + async def send(event): + if event['type'] == 'http.response.start': + assert event['status'] == expected_status + else: + event['body'] == expected_body + + def request(): + try: + app(scope, receive, send).send(None) + except StopIteration: + pass + + scope = SCOPE_BOILERPLATE.copy() + receive_event = RECEIVE_EVENT.copy() + + if number is None: + number = get_work_factor() + + timeit.timeit(request, number=number) + + +if __name__ == '__main__': + run(create_app(), 200, b'Hello, World!') diff --git a/perf/metrics/common.py b/perf/metrics/common.py new file mode 100644 index 000000000..c0b3b7b75 --- /dev/null +++ b/perf/metrics/common.py @@ -0,0 +1,13 @@ +import sys + + +def get_work_factor(): + if len(sys.argv) != 2: + sys.stderr.write(f'{sys.argv[0]}: expected a single int argument.\n') + sys.exit(2) + + try: + return int(sys.argv[1]) + except ValueError: + sys.stderr.write(f'{sys.argv[0]}: expected a single int argument.\n') + sys.exit(2) diff --git a/perf/metrics/hello.py b/perf/metrics/hello.py new file mode 100644 index 000000000..a63184734 --- /dev/null +++ b/perf/metrics/hello.py @@ -0,0 +1,23 @@ +import falcon +from .wsgi import run + + +class Greeter: + def on_get(self, req, resp): + resp.content_type = 'text/plain; charset=utf-8' + resp.text = 'Hello, World!\n' + + +def create_app(): + app = falcon.App() + app.add_route('/', Greeter()) + return app + + +if __name__ == '__main__': + run( + create_app(), + {}, + '200 OK', + b'Hello, World!\n', + ) diff --git a/perf/metrics/media.py b/perf/metrics/media.py new file mode 100644 index 000000000..ad44bf57c --- /dev/null +++ b/perf/metrics/media.py @@ -0,0 +1,35 @@ +import io + +import falcon +from .wsgi import run + + +class Items: + def on_post(self, req, resp): + item = req.get_media() + item['id'] = itemid = 'bar001337' + + resp.location = f'{req.path}/{itemid}' + resp.media = item + resp.status = falcon.HTTP_CREATED + + +def create_app(): + app = falcon.App() + app.add_route('/items', Items()) + return app + + +if __name__ == '__main__': + run( + create_app(), + { + 'CONTENT_LENGTH': len(b'{"foo": "bar"}'), + 'CONTENT_TYPE': 'application/json', + 'PATH_INFO': '/items', + 'REQUEST_METHOD': 'POST', + 'wsgi.input': io.BytesIO(b'{"foo": "bar"}'), + }, + '201 Created', + b'{"foo": "bar", "id": "bar001337"}', + ) diff --git a/perf/metrics/query.py b/perf/metrics/query.py new file mode 100644 index 000000000..837da99e1 --- /dev/null +++ b/perf/metrics/query.py @@ -0,0 +1,33 @@ +import falcon +from .wsgi import run + + +class QueryParams: + def on_get(self, req, resp): + resp.set_header('X-Falcon', req.get_header('X-Falcon')) + resp.status = req.get_param_as_int('resp_status') + resp.text = req.get_param('framework') + + +def create_app(): + app = falcon.App() + app.add_route('/path', QueryParams()) + return app + + +if __name__ == '__main__': + run( + create_app(), + { + 'HTTP_X_FRAMEWORK': 'falcon', + 'HTTP_X_FALCON': 'peregrine', + 'PATH_INFO': '/path', + 'QUERY_STRING': ( + 'flag1&flag2=&flag3&framework=falcon&resp_status=204&' + 'fruit=apple&flag4=true&fruit=orange&status=%F0%9F%8E%89&' + 'fruit=banana' + ), + }, + '204 No Content', + b'', + ) diff --git a/perf/metrics/wsgi.py b/perf/metrics/wsgi.py new file mode 100644 index 000000000..4b02550e4 --- /dev/null +++ b/perf/metrics/wsgi.py @@ -0,0 +1,49 @@ +import io +import sys +import timeit + +from .common import get_work_factor + +ENVIRON_BOILERPLATE = { + 'HTTP_HOST': 'falconframework.org', + 'PATH_INFO': '/', + 'QUERY_STRING': '', + 'RAW_URI': '/', + 'REMOTE_ADDR': '127.0.0.1', + 'REMOTE_PORT': '61337', + 'REQUEST_METHOD': 'GET', + 'SCRIPT_NAME': '', + 'SCRIPT_NAME': '', + 'SERVER_NAME': 'falconframework.org', + 'SERVER_PORT': '80', + 'SERVER_PROTOCOL': 'HTTP/1.1', + 'SERVER_SOFTWARE': 'falcon/3.0', + 'wsgi.errors': sys.stderr, + 'wsgi.input': io.BytesIO(), + 'wsgi.multiprocess': False, + 'wsgi.multithread': False, + 'wsgi.run_once': False, + 'wsgi.url_scheme': 'http', + 'wsgi.version': (1, 0), +} + + +def run(app, environ, expected_status, expected_body, number=None): + def start_response(status, headers, exc_info=None): + assert status == expected_status + + def request_simple(): + assert b''.join(app(environ, start_response)) == expected_body + + def request_with_payload(): + stream.seek(0) + assert b''.join(app(environ, start_response)) == expected_body + + environ = dict(ENVIRON_BOILERPLATE, **environ) + stream = environ['wsgi.input'] + request = request_with_payload if stream.getvalue() else request_simple + + if number is None: + number = get_work_factor() + + timeit.timeit(request, number=number) diff --git a/perf/test_correctness.py b/perf/test_correctness.py new file mode 100644 index 000000000..6af0fae4d --- /dev/null +++ b/perf/test_correctness.py @@ -0,0 +1,69 @@ +import pytest + +from falcon.testing import TestClient +from metrics import asgi +from metrics import hello +from metrics import media +from metrics import query + + +@pytest.mark.asgi +def test_asgi(): + client = TestClient(asgi.create_app()) + + resp = client.simulate_get('/') + assert resp.status_code == 200 + assert resp.headers.get('Content-Type') == 'text/plain; charset=utf-8' + assert resp.text == 'Hello, World!\n' + + +@pytest.mark.hello +def test_hello(): + client = TestClient(hello.create_app()) + + resp = client.simulate_get('/') + assert resp.status_code == 200 + assert resp.headers.get('Content-Type') == 'text/plain; charset=utf-8' + assert resp.text == 'Hello, World!\n' + + +@pytest.mark.media +def test_media(): + client = TestClient(media.create_app()) + + resp1 = client.simulate_post('/items', json={'foo': 'bar'}) + assert resp1.status_code == 201 + assert resp1.headers.get('Content-Type') == 'application/json' + assert resp1.headers.get('Location') == '/items/bar001337' + assert resp1.json == {'foo': 'bar', 'id': 'bar001337'} + + resp2 = client.simulate_post('/items', json={'apples': 'oranges'}) + assert resp2.status_code == 201 + assert resp2.headers.get('Content-Type') == 'application/json' + assert resp2.headers.get('Location') == '/items/bar001337' + assert resp2.json == {'apples': 'oranges', 'id': 'bar001337'} + + +@pytest.mark.query +def test_query(): + client = TestClient(query.create_app()) + + resp1 = client.simulate_get( + '/path?flag1&flag2=&flag3&framework=falcon&resp_status=204&' + 'fruit=apple&flag4=true&fruit=orange&status=%F0%9F%8E%89&' + 'fruit=banana', + headers={'X-Framework': 'falcon', 'X-Falcon': 'peregrine'}, + ) + assert resp1.status_code == 204 + assert resp1.headers.get('X-Falcon') == 'peregrine' + assert resp1.text == '' + + resp2 = client.simulate_get( + '/path?flag1&flag2=&flag3&framework=falcon&resp_status=200&' + 'fruit=apple&flag4=true&fruit=orange&status=%F0%9F%8E%89&' + 'fruit=banana', + headers={'X-Framework': 'falcon', 'X-Falcon': 'peregrine'}, + ) + assert resp2.status_code == 200 + assert resp2.headers.get('X-Falcon') == 'peregrine' + assert resp2.text == 'falcon' diff --git a/perf/test_performance.py b/perf/test_performance.py new file mode 100644 index 000000000..ecb9f524a --- /dev/null +++ b/perf/test_performance.py @@ -0,0 +1,21 @@ +import pytest + + +@pytest.mark.asgi +def test_asgi_metric(gauge): + gauge('asgi') + + +@pytest.mark.hello +def test_hello_metric(gauge): + gauge('hello') + + +@pytest.mark.media +def test_media_metric(gauge): + gauge('media') + + +@pytest.mark.query +def test_query_metric(gauge): + gauge('query') diff --git a/requirements/perf b/requirements/perf new file mode 100644 index 000000000..f666b9159 --- /dev/null +++ b/requirements/perf @@ -0,0 +1,3 @@ +numpy +pytest +PyYAML diff --git a/tox.ini b/tox.ini index 275ace256..bedb6f4f9 100644 --- a/tox.ini +++ b/tox.ini @@ -203,11 +203,11 @@ basepython = python3.8 commands = flake8 \ --max-complexity=15 \ - --exclude=.ecosystem,.eggs,.tox,.venv,build,dist,docs,examples,falcon/bench/nuts \ + --exclude=.ecosystem,.eggs,.tox,.venv,build,dist,docs,examples,falcon/bench/nuts,perf/cachegrind.py \ --ignore=F403,W504 \ --max-line-length=99 \ --import-order-style=google \ - --application-import-names=falcon,examples \ + --application-import-names=falcon,examples,metrics \ --builtins=ignore,attr,defined \ [] @@ -219,7 +219,7 @@ basepython = python3.8 commands = flake8 \ --docstring-convention=pep257 \ - --exclude=.ecosystem,.eggs,.tox,.venv,build,dist,docs,examples,tests,falcon/vendor,falcon/bench/nuts \ + --exclude=.ecosystem,.eggs,.tox,.venv,build,dist,docs,examples,perf,tests,falcon/vendor,falcon/bench/nuts \ --select=D205,D212,D400,D401,D403,D404 \ [] @@ -425,3 +425,37 @@ commands = python -c "import sys; sys.path.pop(0); from falcon.cyutil import misc, reader, uri" pip install -r{toxinidir}/requirements/tests pytest -q tests [] + +# -------------------------------------------------------------------- +# Performance testing gates +# -------------------------------------------------------------------- + +[testenv:perf] +basepython = python3.8 +deps = -r{toxinidir}/requirements/perf +commands = + pytest {toxinidir}/perf -s -v + +[testenv:perf_asgi] +basepython = python3.8 +deps = -r{toxinidir}/requirements/perf +commands = + pytest {toxinidir}/perf -m asgi -s -v + +[testenv:perf_hello] +basepython = python3.8 +deps = -r{toxinidir}/requirements/perf +commands = + pytest {toxinidir}/perf -m hello -s -v + +[testenv:perf_media] +basepython = python3.8 +deps = -r{toxinidir}/requirements/perf +commands = + pytest {toxinidir}/perf -m media -s -v + +[testenv:perf_query] +basepython = python3.8 +deps = -r{toxinidir}/requirements/perf +commands = + pytest {toxinidir}/perf -m query -s -v