Tutorial#
A quick runthrough demonstrating basic API queries.
import requests
Neuron metadata#
Let’s start in the same place as the neuromorpho.org API reference docs: grabbing metadata for a single neuron by neuron id.
endpoint = "https://neuromorpho.org/api/"
r = requests.get(endpoint + "neuron/id/1")
# Check that the status_code is 200 and raise if not
r.raise_for_status()
data = r.json()
data
Show code cell output
---------------------------------------------------------------------------
SSLError Traceback (most recent call last)
File ~/checkouts/readthedocs.org/user_builds/neuromorpho-api/envs/latest/lib/python3.11/site-packages/urllib3/connectionpool.py:466, in HTTPConnectionPool._make_request(self, conn, method, url, body, headers, retries, timeout, chunked, response_conn, preload_content, decode_content, enforce_content_length)
465 try:
--> 466 self._validate_conn(conn)
467 except (SocketTimeout, BaseSSLError) as e:
File ~/checkouts/readthedocs.org/user_builds/neuromorpho-api/envs/latest/lib/python3.11/site-packages/urllib3/connectionpool.py:1095, in HTTPSConnectionPool._validate_conn(self, conn)
1094 if conn.is_closed:
-> 1095 conn.connect()
1097 # TODO revise this, see https://github.com/urllib3/urllib3/issues/2791
File ~/checkouts/readthedocs.org/user_builds/neuromorpho-api/envs/latest/lib/python3.11/site-packages/urllib3/connection.py:652, in HTTPSConnection.connect(self)
650 server_hostname_rm_dot = server_hostname.rstrip(".")
--> 652 sock_and_verified = _ssl_wrap_socket_and_match_hostname(
653 sock=sock,
654 cert_reqs=self.cert_reqs,
655 ssl_version=self.ssl_version,
656 ssl_minimum_version=self.ssl_minimum_version,
657 ssl_maximum_version=self.ssl_maximum_version,
658 ca_certs=self.ca_certs,
659 ca_cert_dir=self.ca_cert_dir,
660 ca_cert_data=self.ca_cert_data,
661 cert_file=self.cert_file,
662 key_file=self.key_file,
663 key_password=self.key_password,
664 server_hostname=server_hostname_rm_dot,
665 ssl_context=self.ssl_context,
666 tls_in_tls=tls_in_tls,
667 assert_hostname=self.assert_hostname,
668 assert_fingerprint=self.assert_fingerprint,
669 )
670 self.sock = sock_and_verified.socket
File ~/checkouts/readthedocs.org/user_builds/neuromorpho-api/envs/latest/lib/python3.11/site-packages/urllib3/connection.py:805, in _ssl_wrap_socket_and_match_hostname(sock, cert_reqs, ssl_version, ssl_minimum_version, ssl_maximum_version, cert_file, key_file, key_password, ca_certs, ca_cert_dir, ca_cert_data, assert_hostname, assert_fingerprint, server_hostname, ssl_context, tls_in_tls)
803 server_hostname = normalized
--> 805 ssl_sock = ssl_wrap_socket(
806 sock=sock,
807 keyfile=key_file,
808 certfile=cert_file,
809 key_password=key_password,
810 ca_certs=ca_certs,
811 ca_cert_dir=ca_cert_dir,
812 ca_cert_data=ca_cert_data,
813 server_hostname=server_hostname,
814 ssl_context=context,
815 tls_in_tls=tls_in_tls,
816 )
818 try:
File ~/checkouts/readthedocs.org/user_builds/neuromorpho-api/envs/latest/lib/python3.11/site-packages/urllib3/util/ssl_.py:465, in ssl_wrap_socket(sock, keyfile, certfile, cert_reqs, ca_certs, server_hostname, ssl_version, ciphers, ssl_context, ca_cert_dir, key_password, ca_cert_data, tls_in_tls)
463 pass
--> 465 ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname)
466 return ssl_sock
File ~/checkouts/readthedocs.org/user_builds/neuromorpho-api/envs/latest/lib/python3.11/site-packages/urllib3/util/ssl_.py:509, in _ssl_wrap_socket_impl(sock, ssl_context, tls_in_tls, server_hostname)
507 return SSLTransport(sock, ssl_context, server_hostname)
--> 509 return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
File ~/.asdf/installs/python/3.11.9/lib/python3.11/ssl.py:517, in SSLContext.wrap_socket(self, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname, session)
511 def wrap_socket(self, sock, server_side=False,
512 do_handshake_on_connect=True,
513 suppress_ragged_eofs=True,
514 server_hostname=None, session=None):
515 # SSLSocket class handles server_hostname encoding before it calls
516 # ctx._wrap_socket()
--> 517 return self.sslsocket_class._create(
518 sock=sock,
519 server_side=server_side,
520 do_handshake_on_connect=do_handshake_on_connect,
521 suppress_ragged_eofs=suppress_ragged_eofs,
522 server_hostname=server_hostname,
523 context=self,
524 session=session
525 )
File ~/.asdf/installs/python/3.11.9/lib/python3.11/ssl.py:1104, in SSLSocket._create(cls, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname, context, session)
1103 raise ValueError("do_handshake_on_connect should not be specified for non-blocking sockets")
-> 1104 self.do_handshake()
1105 except:
File ~/.asdf/installs/python/3.11.9/lib/python3.11/ssl.py:1382, in SSLSocket.do_handshake(self, block)
1381 self.settimeout(None)
-> 1382 self._sslobj.do_handshake()
1383 finally:
SSLError: [SSL: DH_KEY_TOO_SMALL] dh key too small (_ssl.c:1006)
During handling of the above exception, another exception occurred:
SSLError Traceback (most recent call last)
File ~/checkouts/readthedocs.org/user_builds/neuromorpho-api/envs/latest/lib/python3.11/site-packages/urllib3/connectionpool.py:789, in HTTPConnectionPool.urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)
788 # Make the request on the HTTPConnection object
--> 789 response = self._make_request(
790 conn,
791 method,
792 url,
793 timeout=timeout_obj,
794 body=body,
795 headers=headers,
796 chunked=chunked,
797 retries=retries,
798 response_conn=response_conn,
799 preload_content=preload_content,
800 decode_content=decode_content,
801 **response_kw,
802 )
804 # Everything went great!
File ~/checkouts/readthedocs.org/user_builds/neuromorpho-api/envs/latest/lib/python3.11/site-packages/urllib3/connectionpool.py:490, in HTTPConnectionPool._make_request(self, conn, method, url, body, headers, retries, timeout, chunked, response_conn, preload_content, decode_content, enforce_content_length)
489 new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)
--> 490 raise new_e
492 # conn.request() calls http.client.*.request, not the method in
493 # urllib3.request. It also calls makefile (recv) on the socket.
SSLError: [SSL: DH_KEY_TOO_SMALL] dh key too small (_ssl.c:1006)
The above exception was the direct cause of the following exception:
MaxRetryError Traceback (most recent call last)
File ~/checkouts/readthedocs.org/user_builds/neuromorpho-api/envs/latest/lib/python3.11/site-packages/requests/adapters.py:667, in HTTPAdapter.send(self, request, stream, timeout, verify, cert, proxies)
666 try:
--> 667 resp = conn.urlopen(
668 method=request.method,
669 url=url,
670 body=request.body,
671 headers=request.headers,
672 redirect=False,
673 assert_same_host=False,
674 preload_content=False,
675 decode_content=False,
676 retries=self.max_retries,
677 timeout=timeout,
678 chunked=chunked,
679 )
681 except (ProtocolError, OSError) as err:
File ~/checkouts/readthedocs.org/user_builds/neuromorpho-api/envs/latest/lib/python3.11/site-packages/urllib3/connectionpool.py:843, in HTTPConnectionPool.urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)
841 new_e = ProtocolError("Connection aborted.", new_e)
--> 843 retries = retries.increment(
844 method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]
845 )
846 retries.sleep()
File ~/checkouts/readthedocs.org/user_builds/neuromorpho-api/envs/latest/lib/python3.11/site-packages/urllib3/util/retry.py:519, in Retry.increment(self, method, url, response, error, _pool, _stacktrace)
518 reason = error or ResponseError(cause)
--> 519 raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]
521 log.debug("Incremented Retry for (url='%s'): %r", url, new_retry)
MaxRetryError: HTTPSConnectionPool(host='neuromorpho.org', port=443): Max retries exceeded with url: /api/neuron/id/1 (Caused by SSLError(SSLError(1, '[SSL: DH_KEY_TOO_SMALL] dh key too small (_ssl.c:1006)')))
During handling of the above exception, another exception occurred:
SSLError Traceback (most recent call last)
Cell In[2], line 2
1 endpoint = "https://neuromorpho.org/api/"
----> 2 r = requests.get(endpoint + "neuron/id/1")
3 # Check that the status_code is 200 and raise if not
4 r.raise_for_status()
File ~/checkouts/readthedocs.org/user_builds/neuromorpho-api/envs/latest/lib/python3.11/site-packages/requests/api.py:73, in get(url, params, **kwargs)
62 def get(url, params=None, **kwargs):
63 r"""Sends a GET request.
64
65 :param url: URL for the new :class:`Request` object.
(...)
70 :rtype: requests.Response
71 """
---> 73 return request("get", url, params=params, **kwargs)
File ~/checkouts/readthedocs.org/user_builds/neuromorpho-api/envs/latest/lib/python3.11/site-packages/requests/api.py:59, in request(method, url, **kwargs)
55 # By using the 'with' statement we are sure the session is closed, thus we
56 # avoid leaving sockets open which can trigger a ResourceWarning in some
57 # cases, and look like a memory leak in others.
58 with sessions.Session() as session:
---> 59 return session.request(method=method, url=url, **kwargs)
File ~/checkouts/readthedocs.org/user_builds/neuromorpho-api/envs/latest/lib/python3.11/site-packages/requests/sessions.py:589, in Session.request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
584 send_kwargs = {
585 "timeout": timeout,
586 "allow_redirects": allow_redirects,
587 }
588 send_kwargs.update(settings)
--> 589 resp = self.send(prep, **send_kwargs)
591 return resp
File ~/checkouts/readthedocs.org/user_builds/neuromorpho-api/envs/latest/lib/python3.11/site-packages/requests/sessions.py:703, in Session.send(self, request, **kwargs)
700 start = preferred_clock()
702 # Send the request
--> 703 r = adapter.send(request, **kwargs)
705 # Total elapsed time of the request (approximately)
706 elapsed = preferred_clock() - start
File ~/checkouts/readthedocs.org/user_builds/neuromorpho-api/envs/latest/lib/python3.11/site-packages/requests/adapters.py:698, in HTTPAdapter.send(self, request, stream, timeout, verify, cert, proxies)
694 raise ProxyError(e, request=request)
696 if isinstance(e.reason, _SSLError):
697 # This branch is for urllib3 v1.22 and later.
--> 698 raise SSLError(e, request=request)
700 raise ConnectionError(e, request=request)
702 except ClosedPoolError as e:
SSLError: HTTPSConnectionPool(host='neuromorpho.org', port=443): Max retries exceeded with url: /api/neuron/id/1 (Caused by SSLError(SSLError(1, '[SSL: DH_KEY_TOO_SMALL] dh key too small (_ssl.c:1006)')))
Neuron query#
In practice, it is unlikely you will know the neuromorpho id(s) of the
neuron(s) you’re interested in.
This is where requests really comes in handy, allowing you to
specify the parameters supported by the neuromorpho api with
Python dictionaries.
For example, let’s construct a query to select all neurons by name from a single image in the Siegert archive:
# Neuron name - note the `*` wildcard to select all neurons whose name matches
# this pattern
siegert_dataset = "CB_CKp25_2w_F_Animal01_*"
# The GET parameters
query_params = {"q": f"neuron_name:{siegert_dataset}"}
r = requests.get(endpoint + "neuron/select/", params=query_params)
# We let requests handle the URL construction
r.url
The data for multiple neurons is returned in the form of a nested dict from which individual neuron metadata can be extracted.
r.raise_for_status() # Make sure the request was successful
data = r.json()["_embedded"]["neuronResources"]
f"Number of neurons matching query: {len(data)}"
# Individual neuron names
sorted(record["neuron_name"] for record in data)
Neuron traces#
Up to this point, we’ve only demonstrated how to extract the neuron metadata available via the neuromorpho API.
Individual SWC traces are also accessible via GET request, albeit from
different URLs.
Note
Accessing SWC files does not appear to be part of the public neuromorpho API, so the recipe below may not generalize to all datasets. It should however give you a rough idea of how the data appear to be organized.
The general form of URLs for the SWC data include three parameters:
The archive name
The data version (e.g.
Source-VersionorCNG versionfor original or standardized versions, respectively), andThe neuron name
Items 1 and 3 can be extracted from the neuron metadata, while item 2 specifies which version of SWC file you want.
Let’s return to our previous query of the Seigert data and download the SWC data for one of the neurons.
Let’s start by codifying the above with a string template:
import random
import string
# A general-ish URL template for accessing SWC data
swc_url_template = string.Template(
"https://neuromorpho.org/dableFiles/$archive/$version/$fname.swc"
)
Then we’ll arbitrarily select one of the neurons from the previous query for our example, and construct SWC url:
record = random.choice(data)
record["neuron_name"]
# Construct the SWC URL for the original (source) version of this specific neuron
swc_url = swc_url_template.substitute(
archive=record["archive"].lower(),
version="Source-Version",
fname=record["neuron_name"],
)
swc_url
Finally, we download the data. Note the raise_for_status is particularly
helpful here to catch issues with our URL template.
r = requests.get(swc_url)
r.raise_for_status()
If all went well, the SWC data should be available in the response’s content
as a byte-string:
import io
import pandas as pd
raw_data = io.BytesIO(r.content)
df = pd.read_csv(raw_data, delimiter=" ")
df