diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..b35560d --- /dev/null +++ b/Makefile @@ -0,0 +1,3 @@ +style: + ruff check --fix + ruff format diff --git a/README.md b/README.md index 2711a4c..9d77cd6 100644 --- a/README.md +++ b/README.md @@ -3,42 +3,11 @@ `httpx-pycurl` provides an `httpx` transport that executes requests with `pycurl`. It combines the [goodness of curl](https://everything.curl.dev/) with the familiar `httpx` API, including support for `http/2` and even non-http -protocols built into `curl`. On my machine, `AsyncPyCurlTransport` performs -better than `httpx`'s default `AsyncHttpTransport`, taking approximately 75% of -the time to fetch 60 files from a local `nginx` test server. - -`httpx-pycurl` is in early development, but it passes most `httpx` tests and has -good performance. A `niquests`-derived test uses `asyncio.gather()` to make 1000 -http/2 requests to `https://httpbingo.org/get`. `httpx-pycurl` is about as fast. - -``` -# First run: -Fetch 1000x https://httpbingo.org/get -aiohttp: 1.029s -httpx: 1.369s -httpx_pycurl: 0.637s -niquests: 0.715s - -# Second run: -Fetch 1000x https://httpbingo.org/get -aiohttp: 0.927s -httpx: 1.346s -httpx_pycurl: 0.677s -niquests: 0.655s -``` - -## Install - -```bash -pip install httpx-pycurl -``` - -Or with conda, - -```bash -conda install -n base conda-pypi -conda pypi install httpx-pycurl -``` +protocols built into `curl`. `AsyncPyCurlTransport` performs better than +`httpx`'s default `AsyncHttpTransport` with `http2=True`, taking about 78% of +the time to issue 128 requests in parallel. Under heavier usage `httpx-pycurl` +appears to pull further ahead of alternative libraries, without replacing all of +`httpx`; just `httpx`'s transport. ## Usage @@ -81,3 +50,82 @@ debug_transport = PyCurlTransport( debug_callback=lambda info_type, data: print(info_type, data), ) ``` + +## Installation + +```bash +pip install httpx-pycurl +``` + +Or with conda, + +```bash +conda install -n base conda-pypi +conda pypi install httpx-pycurl +``` + +## Performance + +`httpx-pycurl` is in early development but it passes most `httpx` tests and has +good performance. Our `tests/bench.py` uses `asyncio.gather()` to make many +`http/2` requests to `https://httpbingo.org/get` using `httpx`, `niquests`, and +`httpx` with `httpx-pycurl`'s transport. `httpx-pycurl` is the fastest library +tested. + +Running `tests/bench.py [N]` shows that the more efficient `http/2` libraries +shine when performing large numbers of parallel requests, and are closer +together when only groups of 128 parallel requests are made. + +``` +2 groups of 512 requests each... + +Time per group: +httpx: 0.679s ± 0.119s +niquests: 0.432s ± 0.140s +httpx_pycurl: 0.299s ± 0.056s + +Paired t-test: httpx_pycurl vs niquests +t-stat: -2.249 (approx p < 0.05 if |t| > 2.365) +Speedup: 1.44x +``` + +``` +4 groups of 256 requests each... + +Time per group: +httpx: 0.378s ± 0.077s +niquests: 0.259s ± 0.064s +httpx_pycurl: 0.190s ± 0.034s + +Paired t-test: httpx_pycurl vs niquests +t-stat: -4.208 (approx p < 0.05 if |t| > 2.365) +Speedup: 1.36x +``` + +``` +8 groups of 128 requests each... + +Time per group: +httpx: 0.215s ± 0.073s +niquests: 0.205s ± 0.038s +httpx_pycurl: 0.162s ± 0.033s + +Paired t-test: httpx_pycurl vs niquests +t-stat: -8.949 (approx p < 0.05 if |t| > 2.365) +Speedup: 1.27x +``` + +## Dependencies + +`httpx-pycurl` uses curl to support `http/2` instead of the `h2`, `hpack` and +`hyperframe` dependencies used by `httpx`. + +``` +$ pip install --dry-run httpx-pycurl +... +Would install anyio-4.13.0 certifi-2026.4.22 h11-0.16.0 httpcore-1.0.9 httpx-0.28.1 httpx-pycurl-0.0.4 idna-3.13 pycurl-7.45.7 + +$ pip install --dry-run httpx[http2] +... +Would install anyio-4.13.0 certifi-2026.4.22 h11-0.16.0 h2-4.3.0 hpack-4.1.0 httpcore-1.0.9 httpx-0.28.1 hyperframe-6.1.0 idna-3.13 +``` diff --git a/tests/bench.py b/tests/bench.py index eedd2ca..d2b9fc7 100644 --- a/tests/bench.py +++ b/tests/bench.py @@ -4,6 +4,7 @@ import asyncio import statistics +import sys import time import httpx @@ -16,14 +17,14 @@ USER_AGENT = "httpx-pycurl (bench)" DEFAULT_HEADERS = {"User-Agent": USER_AGENT} -# Make 1024 total requests for each library in groups. niquests seems to shine -# when batch sizes are large. When batch sizes are smaller, i.e. 128 requests -# each, vanilla httpx and niquests seem to be closer. -PARTITIONS = 4 -COUNT = 1024 // PARTITIONS +async def get_one(client, url): + response = await client.get(url) + assert len(response.content) + return response.content -async def bench(): + +async def bench(PARTITIONS, COUNT): """ Make COUNT requests using each of several clients. Print time taken by each. """ @@ -54,7 +55,7 @@ async def bench(): for client, name in clients: begin = time.perf_counter_ns() # async with client as client: - await asyncio.gather(*(client.get(URL) for _ in range(COUNT))) + await asyncio.gather(*(get_one(client, URL) for _ in range(COUNT))) end = time.perf_counter_ns() results_by_client[name].append((end - begin) / 1e9) @@ -79,4 +80,14 @@ async def bench(): if __name__ == "__main__": - asyncio.run(bench()) + # niquests seems to shine when batch sizes are large. When batch sizes are + # smaller, i.e. 128 requests each, vanilla httpx and niquests seem to be + # closer. + + PARTITIONS = 4 + if len(sys.argv) == 2: + PARTITIONS = int(sys.argv[1]) + + COUNT = 1024 // PARTITIONS + + asyncio.run(bench(PARTITIONS, COUNT))