diff --git a/README.md b/README.md index bdbc8c6..b8b7a86 100755 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![CodeFactor](https://www.codefactor.io/repository/github/realzza/xenopy/badge)](https://www.codefactor.io/repository/github/realzza/xenopy)  [![DOI](https://zenodo.org/badge/442515920.svg)](https://zenodo.org/badge/latestdoi/442515920) -**`XenoPy`** is a python library that builds upon [xeno-canto API 2.0](https://xeno-canto.org/article/153). +**`XenoPy`** is a python library that builds upon the [xeno-canto API v3](https://xeno-canto.org/explore/api). ## Install Install from `pip`. @@ -18,10 +18,20 @@ Checkout the [**birdData**](https://github.com/realzza/xenopy/tree/birdData) bra ## Usage Snippet You can directly search for bird data for a specific species. For instance, we retrieve data for [*African Silverbill*](https://xeno-canto.org/species/Euodice-cantans) whom's `quality` better than `C` since `2020-01-01`. + +The xeno-canto API v3 requires an API key. You can pass it directly to `Query`, or set it once in your environment as `XENO_CANTO_API_KEY`. + +```bash +export XENO_CANTO_API_KEY="your-api-key" +``` + ```python from xenopy import Query q = Query(name="African silverbill", q_gt="C", since="2020-01-01") + +# Or pass the key explicitly: +q = Query(name="African silverbill", q_gt="C", since="2020-01-01", api_key="your-api-key") ``` ### Retrieve Metafiles @@ -44,9 +54,13 @@ Two files will be generated while running `retrieve_recordings`, `kill_multiproc ## Define a `Query` As you can tell from the [Usage Snippet](#Usage-Snippet), defining a query is the most important step in communicating with the API. We determined the following interface to form a query based on the xeno-canto [search tips](https://xeno-canto.org/help/search). ```markdown -name: Species Name. Specify the name of bird you intend to retrieve data from. Both English names and Latin names are acceptable. +name: Convenience alias for English species name. API v3 sends this as en:"...". +en: English species name. +sp: Species. For a full scientific name, use a quoted binomial such as sp='"Larus fuscus"'. gen: Genus. Genus is part of a species' latin name, so it is searched by default when performing a basic search (as mentioned above). ssp: subspecies +fam: family +grp: animal group, e.g. birds, grasshoppers, bats, frogs, land mammals rec: recordist. Search for all recordings from a particular recordist. cnt: country. Search for all recordings from a particular country. loc: location. Search for all recordings from a specific location. @@ -59,8 +73,8 @@ type: Search for recordings of a particular sound type, e.g., type='song' nr: number. To search for a known recording number, use the nr tag: for example nr:76967. You can also search for a range of numbers as nr:88888-88890. lc: license. q: quality ratings. -q_lt: quality ratings less than -q_gt: quality ratings better than +q_lt: quality ratings less than; sent to API v3 as q:VALUE Usage Examples: Recordings are rated by quality. Quality ratings range from A (highest quality) to E (lowest quality). To search for recordings that match a certain quality rating, use the q, q_lt, and q_gt tags. For example: - q:A will return recordings with a quality rating of A. @@ -68,8 +82,8 @@ q_gt: quality ratings better than - q_lt:C will return recordings with a quality rating of D or E. - q_gt:C will return recordings with a quality rating of B or A. len: recording length control parameter. -len_lt: recording length less than -len_gt: recording length greater than +len_lt: recording length less than; sent to API v3 as len:VALUE Usage Examples: len:10 will return recordings with a duration of 10 seconds (with a margin of 1%, so actually between 9.9 and 10.1 seconds) len:10-15 will return recordings lasting between 10 and 15 seconds. @@ -82,6 +96,8 @@ since: - since=YYYY-MM-DD, since the particular date year: year month: month. year and month tags allow you to search for recordings that were recorded on a certain date. +api_key: xeno-canto API v3 key. If omitted, retrieve_* reads XENO_CANTO_API_KEY from the environment. +per_page: optional API v3 page size. ``` ## Citation @@ -113,5 +129,5 @@ If `XenoPy` is helpful in your project or research in any form, you can cite thi - [x] add multiprocessing downloading feature ## Open Source -The first generation of `xenocanto` [package](https://github.com/ntivirikin/xeno-canto-py) is hard to use also inefficient. Thus I wrapped the [2.0 API](https://xeno-canto.org/article/153) version in a more straightforward and efficient interface. -Feel free to file an issue had you encountered any bugs, or prompt a PR to `XenoPy` to join me in maintenance and optimization. \ No newline at end of file +The first generation of `xenocanto` [package](https://github.com/ntivirikin/xeno-canto-py) is hard to use also inefficient. Thus I wrapped the xeno-canto API in a more straightforward and efficient interface. +Feel free to file an issue had you encountered any bugs, or prompt a PR to `XenoPy` to join me in maintenance and optimization. diff --git a/query.py b/query.py index ae8ba95..bdf00c9 100644 --- a/query.py +++ b/query.py @@ -1,16 +1,27 @@ import argparse import json import os -from urllib import error, request +from urllib import error, parse, request -from multiprocess import Process from tqdm import tqdm from utils import chunks +try: + from multiprocess import Process +except ImportError: # pragma: no cover - exercised only when optional dep is absent + from multiprocessing import Process + +API_ENDPOINT = "https://xeno-canto.org/api/3/recordings" +API_KEY_ENV_VAR = "XENO_CANTO_API_KEY" + params = [ + "en", + "sp", "gen", "ssp", + "fam", + "grp", "rec", "cnt", "loc", @@ -35,12 +46,32 @@ ] +def _with_operator(operator, value): + value = str(value) + if value[:1] in ("<", ">", "="): + return value + return operator + value + + +def _format_query_value(value): + value = str(value) + if any(char.isspace() for char in value) and not ( + value.startswith('"') and value.endswith('"') + ): + value = '"' + value.replace('"', '\\"') + '"' + return value + + class Query: def __init__( self, name=None, + en=None, + sp=None, gen=None, ssp=None, + fam=None, + grp=None, rec=None, cnt=None, loc=None, @@ -62,12 +93,18 @@ def __init__( since=None, year=None, month=None, + api_key=None, + per_page=None, ): """ params: - name: Can search for bird name directly. str + name: Convenience alias for English species name. In xeno-canto API v3 this is sent as en:"...". str + en: English species name. + sp: Species. In API v3, quoted binomials such as sp='"Larus fuscus"' are supported. gen: Genus. Genus is part of a species' latin name, so it is searched by default when performing a basic search (as mentioned above). ssp: subspecies + fam: family. + grp: animal group, e.g. birds, grasshoppers, bats, frogs, land mammals. rec: recordist. Search for all recordings from a particular recordist. cnt: country. Search for all recordings from a particular country. loc: location. Search for all recordings from a specific location. @@ -103,13 +140,22 @@ def __init__( - since=YYYY-MM-DD, since the particular date year: year month: month. year and month tags allow you to search for recordings that were recorded on a certain date. + api_key: xeno-canto API v3 key. If omitted, retrieve_* reads XENO_CANTO_API_KEY from the environment. + per_page: optional API v3 page size. """ + self.api_key = api_key + self.per_page = per_page self.args = {} if name: - self.args["name"] = name.replace(" ", "%20") + self.args["en"] = name + if en: + self.args["en"] = en + self.args["sp"] = sp self.args["gen"] = gen self.args["ssp"] = ssp + self.args["fam"] = fam + self.args["grp"] = grp self.args["rec"] = rec self.args["cnt"] = cnt self.args["loc"] = loc @@ -122,25 +168,33 @@ def __init__( self.args["nr"] = nr self.args["lic"] = lic self.args["q"] = q - self.args["q_lt"] = q_lt - self.args["q_gt"] = q_gt self.args["len"] = length - self.args["len_lt"] = len_lt - self.args["len_gt"] = len_gt self.args["area"] = area self.args["since"] = since self.args["year"] = year self.args["month"] = month query_options = {k: v for k, v in self.args.items() if v} + if q_lt: + query_options["q_lt"] = _with_operator("<", q_lt) + if q_gt: + query_options["q_gt"] = _with_operator(">", q_gt) + if len_lt: + query_options["len_lt"] = _with_operator("<", len_lt) + if len_gt: + query_options["len_gt"] = _with_operator(">", len_gt) assert query_options, "empty query, please add query options" - if name: - del self.args["gen"], self.args["ssp"] - self.query = "%20".join( - ["%s:%s" % (k, v) for k, v in query_options.items()] - ).replace("name:", "") + query_terms = [] + for k, v in query_options.items(): + tag = ( + {"q_lt": "q", "q_gt": "q", "len_lt": "len", "len_gt": "len"}[k] + if k in {"q_lt", "q_gt", "len_lt", "len_gt"} + else k + ) + query_terms.append("%s:%s" % (tag, _format_query_value(v))) + self.query = " ".join(query_terms) self.args = query_options - print("query:", self.query.replace("%20", " ")) + print("query:", self.query) def _get_args(self): return self.args @@ -148,6 +202,75 @@ def _get_args(self): def _init_dir(self, d): os.makedirs(d, exist_ok=True) + def _get_api_key(self): + api_key = self.api_key or os.environ.get(API_KEY_ENV_VAR) + if not api_key: + raise ValueError( + "xeno-canto API v3 requires an API key. Pass api_key=... or set " + f"{API_KEY_ENV_VAR}." + ) + return api_key + + def _build_api_url(self, page): + query_params = { + "query": self.query, + "key": self._get_api_key(), + "page": page, + } + if self.per_page: + query_params["per_page"] = self.per_page + return API_ENDPOINT + "?" + parse.urlencode(query_params) + + def _redact_url(self, url): + parsed = parse.urlsplit(url) + query_params = parse.parse_qsl(parsed.query, keep_blank_values=True) + redacted = [ + (k, "***" if k == "key" else v) + for k, v in query_params + ] + return parse.urlunsplit( + parsed._replace(query=parse.urlencode(redacted)) + ) + + def _read_json_url(self, url, attempts): + n_attempts = 0 + while n_attempts < attempts: + try: + r = request.urlopen(url) # nosec + break + except error.HTTPError as e: + body = e.read().decode("UTF-8", errors="replace") + n_attempts += 1 + if n_attempts == attempts: + raise RuntimeError( + "xeno-canto API request failed: %s %s" + % (e, body.strip()) + ) from e + except error.URLError as e: + n_attempts += 1 + if n_attempts == attempts: + raise RuntimeError( + "xeno-canto API request failed: %s" % e + ) from e + data = json.loads(r.read().decode("UTF-8")) + if "error" in data: + raise RuntimeError("xeno-canto API returned an error: %s" % data) + return data + + def _download_url(self, url): + if url.startswith("//"): + url = "https:" + url + elif url.startswith("/"): + url = "https://xeno-canto.org" + url + + parsed = parse.urlsplit(url) + query_params = parse.parse_qsl(parsed.query, keep_blank_values=True) + if not any(k == "key" for k, _ in query_params): + query_params.append(("key", self._get_api_key())) + return parse.urlunsplit( + parsed._replace(query=parse.urlencode(query_params)) + ) + def retrieve_meta(self, recordings_only=False, verbose=False, attempts=10): """ params: @@ -159,41 +282,26 @@ def retrieve_meta(self, recordings_only=False, verbose=False, attempts=10): data_all: [dict] containing all metafiles from the query. """ - query_content = self.query print("... retrieving metadata ...") page, page_num = 1, 1 data_all = {} while page < page_num + 1: - url = ( - "https://www.xeno-canto.org/api/2/recordings?query={0}&page={1}".format( - query_content, page - ) - ) - n_attempts = 0 - while n_attempts < attempts: - try: - if verbose: - print(url) - r = request.urlopen(url) # nosec - break - except error.HTTPError as e: - n_attempts += 1 - if n_attempts == attempts: - print("An error has occurred: " + str(e)) - print("Bad query: %s" % query_content) + url = self._build_api_url(page) + if verbose: + print(self._redact_url(url)) - data = json.loads(r.read().decode("UTF-8")) + data = self._read_json_url(url, attempts) if not data_all: data_all = data else: data_all["recordings"] += data["recordings"] - page_num = data["numPages"] + page_num = int(data["numPages"]) page += 1 - del data_all["page"] - del data_all["numPages"] + data_all.pop("page", None) + data_all.pop("numPages", None) data_all["numRecordings"] = len(data_all["recordings"]) if recordings_only: return data_all["recordings"] @@ -219,7 +327,7 @@ def retrieve_recordings( problematic_urls = [] for curr_rec in tqdm(all_recordings, desc="process %d" % os.getpid()): - url = curr_rec["file"] + url = self._download_url(curr_rec["file"]) name = (curr_rec["en"]).replace(" ", "") track_id = curr_rec["id"] @@ -255,7 +363,7 @@ def __single_dl(self, pid, all_recordings, attempts=10, outdir="datasets/"): with open("kill_multiprocess.sh", "a") as f: f.write("kill -9 %d\n" % os.getpid()) isFirst = False - url = curr_rec["file"] + url = self._download_url(curr_rec["file"]) name = (curr_rec["en"]).replace(" ", "") track_id = curr_rec["id"] diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/test_query.py b/tests/test_query.py new file mode 100644 index 0000000..b939196 --- /dev/null +++ b/tests/test_query.py @@ -0,0 +1,158 @@ +import contextlib +import io +import json +import os +import tempfile +import unittest +from unittest import mock +from urllib import error, parse + +from query import API_ENDPOINT, API_KEY_ENV_VAR, Query + + +def quiet_call(func, *args, **kwargs): + with contextlib.redirect_stdout(io.StringIO()): + with contextlib.redirect_stderr(io.StringIO()): + return func(*args, **kwargs) + + +def quiet_query(*args, **kwargs): + return quiet_call(Query, *args, **kwargs) + + +class FakeResponse: + def __init__(self, payload): + self._payload = json.dumps(payload).encode("UTF-8") + + def read(self): + return self._payload + + +def parsed_query(url): + parsed = parse.urlsplit(url) + return parsed, dict(parse.parse_qsl(parsed.query)) + + +class QueryConstructionTests(unittest.TestCase): + def test_name_builds_tagged_v3_english_name_query(self): + q = quiet_query(name="African silverbill", q_gt="C", since="2020-01-01") + + self.assertEqual( + q.query, 'en:"African silverbill" since:2020-01-01 q:>C' + ) + + def test_v3_tags_quote_multi_word_values_and_convert_operators(self): + q = quiet_query( + gen="Larus", + sp="fuscus", + cnt="United States", + rec_type="song", + q_lt="C", + len_gt=120, + ) + + self.assertEqual( + q.query, + 'sp:fuscus gen:Larus cnt:"United States" type:song q:120', + ) + + def test_missing_api_key_fails_before_network_request(self): + q = quiet_query(sp='"larus fuscus"') + + with mock.patch.dict(os.environ, {}, clear=True): + with mock.patch("query.request.urlopen") as urlopen: + with self.assertRaisesRegex(ValueError, API_KEY_ENV_VAR): + quiet_call(q.retrieve_meta) + + urlopen.assert_not_called() + + +class MetadataRetrievalTests(unittest.TestCase): + def test_retrieve_meta_uses_v3_endpoint_key_per_page_and_paginates(self): + responses = [ + FakeResponse( + { + "numRecordings": "2", + "numSpecies": "1", + "page": 1, + "numPages": 2, + "recordings": [{"id": "1"}], + } + ), + FakeResponse( + { + "numRecordings": "2", + "numSpecies": "1", + "page": 2, + "numPages": 2, + "recordings": [{"id": "2"}], + } + ), + ] + q = quiet_query(sp='"larus fuscus"', api_key="secret", per_page=1) + + with mock.patch("query.request.urlopen", side_effect=responses) as urlopen: + data = quiet_call(q.retrieve_meta, verbose=True) + + self.assertEqual(data["numRecordings"], 2) + self.assertEqual(data["recordings"], [{"id": "1"}, {"id": "2"}]) + self.assertNotIn("page", data) + self.assertNotIn("numPages", data) + + requested_urls = [call.args[0] for call in urlopen.call_args_list] + first_url, second_url = requested_urls + parsed, params = parsed_query(first_url) + self.assertEqual(parse.urlunsplit(parsed._replace(query="")), API_ENDPOINT) + self.assertEqual(params["query"], 'sp:"larus fuscus"') + self.assertEqual(params["key"], "secret") + self.assertEqual(params["per_page"], "1") + self.assertEqual(params["page"], "1") + self.assertEqual(parsed_query(second_url)[1]["page"], "2") + + def test_http_api_errors_include_response_body(self): + q = quiet_query(sp='"larus fuscus"', api_key="bad-key") + body = io.BytesIO( + b'{"error":"client_error","message":"Missing or invalid key"}' + ) + http_error = error.HTTPError( + API_ENDPOINT, 401, "Unauthorized", hdrs={}, fp=body + ) + + with mock.patch("query.request.urlopen", side_effect=http_error): + with self.assertRaisesRegex(RuntimeError, "Missing or invalid key"): + quiet_call(q.retrieve_meta, attempts=1) + + +class RecordingDownloadTests(unittest.TestCase): + def test_retrieve_recordings_normalizes_file_url_and_appends_key(self): + q = quiet_query(sp='"larus fuscus"', api_key="secret") + recordings = [ + { + "id": "1065457", + "en": "Lesser Black-backed Gull", + "file": "//xeno-canto.org/1065457/download", + }, + { + "id": "restricted", + "en": "Hidden Bird", + "file": "", + }, + ] + + with tempfile.TemporaryDirectory() as tmpdir: + with mock.patch.object(q, "retrieve_meta", return_value=recordings): + with mock.patch("query.request.urlretrieve") as urlretrieve: + quiet_call(q.retrieve_recordings, outdir=tmpdir) + + urlretrieve.assert_called_once() + url, output_path = urlretrieve.call_args.args + parsed, params = parsed_query(url) + self.assertEqual(parsed.scheme, "https") + self.assertEqual(parsed.netloc, "xeno-canto.org") + self.assertEqual(parsed.path, "/1065457/download") + self.assertEqual(params["key"], "secret") + self.assertTrue(output_path.endswith("LesserBlack-backedGull/1065457.mp3")) + + +if __name__ == "__main__": + unittest.main()