Jupyter q Magic Command
import pykx as kx
Create the external q process¶
To run this example, the Python code in the following cell will do the equivalent to executing the following in a terminal:
$ q -p 5001
import subprocess
import time
proc = subprocess.Popen(
('q', '-p', '5001')
)
time.sleep(5)
Executing against Embedded q¶
A cell begining with %%q
will execute q within PyKX
's EmbeddedQ
module.
%%q
til 10
0 1 2 3 4 5 6 7 8 9
Executing against an external q process over IPC¶
Connection information can also be included after the %%q
to connect to a remote q
process over
IPC.
Here is the list of currently supported connection parameters. If they specify a type a second value is expected to follow them to be used as the parameter. If no type follows them they can be used as a stand alone flag.
--host: A string object denoting the host to connect to
--port: An int object denoting the port to connect over
--user: A str object denoting the username to use when connecting
--password: A str object denoting the password to use when connecting
--timeout: A float object denoting the time in seconds before the query
times out, defaults to no timeout
--nolarge: Disable messages over 2GB being sent / received
--tls: Use a tls connection
--unix: Use a unix connection
--noctx: Disable the context interface
Connect to a q server running on localhost
at port 5001
as user
using password password
and disable the context interface.
%%q --host localhost --port 5001 --user user --pass password --noctx
til 10
--------------------------------------------------------------------------- QError Traceback (most recent call last) Cell In[4], line 1 ----> 1 get_ipython().run_cell_magic('q', '--host localhost --port 5001 --user user --pass password --noctx', 'til 10\n') File /usr/local/lib/python3.10/site-packages/IPython/core/interactiveshell.py:2422, in InteractiveShell.run_cell_magic(self, magic_name, line, cell) 2420 with self.builtin_trap: 2421 args = (magic_arg_s, cell) -> 2422 result = fn(*args, **kwargs) 2423 return result File /usr/local/lib/python3.10/site-packages/pykx/nbextension.py:70, in q(instructions, code) 67 if port is None: 68 raise kx.QError('--port must be set to create IPC connection in magic command.') ---> 70 _q = kx.SyncQConnection( 71 host, 72 port, 73 username=username, 74 password=password, 75 timeout=timeout, 76 large_messages=large_messages, 77 tls=tls, 78 unix=unix, 79 no_ctx=no_ctx 80 ) 81 try: 82 _q('.Q.ld') File /usr/local/lib/python3.10/site-packages/pykx/ipc.py:633, in SyncQConnection.__new__(cls, *args, **kwargs) 631 def __new__(cls, *args, **kwargs): 632 if 'tls' in kwargs.keys(): --> 633 return SecureQConnection(*args, **kwargs) 634 return object.__new__(cls) File /usr/local/lib/python3.10/site-packages/pykx/ipc.py:1705, in SecureQConnection.__init__(self, host, port, username, password, timeout, large_messages, tls, unix, sync, wait, lock, no_ctx, *args) 1703 warn(self._sync_deprecation_warning, DeprecationWarning) 1704 wait = sync -> 1705 self._init(host, 1706 port, 1707 *args, 1708 username=username, 1709 password=password, 1710 timeout=timeout, 1711 large_messages=large_messages, 1712 tls=tls, 1713 unix=unix, 1714 wait=wait, 1715 lock=lock, 1716 no_ctx=no_ctx, 1717 ) 1718 super().__init__() File /usr/local/lib/python3.10/site-packages/pykx/ipc.py:357, in QConnection._init(self, host, port, username, password, timeout, large_messages, tls, unix, wait, lock, no_ctx) 353 object.__setattr__(self, '_lock', lock) 354 object.__setattr__(self, 'closed', False) 355 object.__setattr__(self, 356 '_handle', --> 357 _ipc.init_handle(host, 358 port, 359 credentials, 360 unix, 361 tls, 362 timeout, 363 large_messages)) 364 if licensed: 365 object.__setattr__(self, '_finalizer', finalize(self, self._close, self._handle)) File /usr/local/lib/python3.10/site-packages/pykx/_ipc.pyx:36, in pykx._ipc.init_handle() File /usr/local/lib/python3.10/site-packages/pykx/wrappers.py:2829, in Composition.__call__(self, *args, **kwargs) 2827 if not licensed: 2828 raise LicenseException('call a q function in a Python process') -> 2829 if q('{.pykx.i.isw x}', self).py(): 2830 args = {i: K(x) for i, x in enumerate(args)} 2831 if args: # Avoid calling `max` on an empty sequence File /usr/local/lib/python3.10/site-packages/pykx/embedded_q.py:137, in EmbeddedQ.__call__(self, query, wait, sync, *args) 135 result = _keval(bytes(wrappers.CharVector(query)), *[wrappers.K(x) for x in args]) 136 if wait is None or wait: --> 137 return factory(result, False) 138 return self('::', wait=True) File /usr/local/lib/python3.10/site-packages/pykx/_wrappers.pyx:501, in pykx._wrappers._factory() File /usr/local/lib/python3.10/site-packages/pykx/_wrappers.pyx:494, in pykx._wrappers.factory() QError: .pykx.i.isw
All connection arguments are optional with the exception of the --port
argument. If --host
is not provided localhost
will be used as the default host.
%%q --port 5001
tab:([]a:1000?1000; b:1000?500.0; c:1000?`AAPL`MSFT`GOOG);
--------------------------------------------------------------------------- QError Traceback (most recent call last) Cell In[5], line 1 ----> 1 get_ipython().run_cell_magic('q', '--port 5001', 'tab:([]a:1000?1000; b:1000?500.0; c:1000?`AAPL`MSFT`GOOG);\n') File /usr/local/lib/python3.10/site-packages/IPython/core/interactiveshell.py:2422, in InteractiveShell.run_cell_magic(self, magic_name, line, cell) 2420 with self.builtin_trap: 2421 args = (magic_arg_s, cell) -> 2422 result = fn(*args, **kwargs) 2423 return result File /usr/local/lib/python3.10/site-packages/pykx/nbextension.py:70, in q(instructions, code) 67 if port is None: 68 raise kx.QError('--port must be set to create IPC connection in magic command.') ---> 70 _q = kx.SyncQConnection( 71 host, 72 port, 73 username=username, 74 password=password, 75 timeout=timeout, 76 large_messages=large_messages, 77 tls=tls, 78 unix=unix, 79 no_ctx=no_ctx 80 ) 81 try: 82 _q('.Q.ld') File /usr/local/lib/python3.10/site-packages/pykx/ipc.py:633, in SyncQConnection.__new__(cls, *args, **kwargs) 631 def __new__(cls, *args, **kwargs): 632 if 'tls' in kwargs.keys(): --> 633 return SecureQConnection(*args, **kwargs) 634 return object.__new__(cls) File /usr/local/lib/python3.10/site-packages/pykx/ipc.py:1705, in SecureQConnection.__init__(self, host, port, username, password, timeout, large_messages, tls, unix, sync, wait, lock, no_ctx, *args) 1703 warn(self._sync_deprecation_warning, DeprecationWarning) 1704 wait = sync -> 1705 self._init(host, 1706 port, 1707 *args, 1708 username=username, 1709 password=password, 1710 timeout=timeout, 1711 large_messages=large_messages, 1712 tls=tls, 1713 unix=unix, 1714 wait=wait, 1715 lock=lock, 1716 no_ctx=no_ctx, 1717 ) 1718 super().__init__() File /usr/local/lib/python3.10/site-packages/pykx/ipc.py:357, in QConnection._init(self, host, port, username, password, timeout, large_messages, tls, unix, wait, lock, no_ctx) 353 object.__setattr__(self, '_lock', lock) 354 object.__setattr__(self, 'closed', False) 355 object.__setattr__(self, 356 '_handle', --> 357 _ipc.init_handle(host, 358 port, 359 credentials, 360 unix, 361 tls, 362 timeout, 363 large_messages)) 364 if licensed: 365 object.__setattr__(self, '_finalizer', finalize(self, self._close, self._handle)) File /usr/local/lib/python3.10/site-packages/pykx/_ipc.pyx:36, in pykx._ipc.init_handle() File /usr/local/lib/python3.10/site-packages/pykx/wrappers.py:2829, in Composition.__call__(self, *args, **kwargs) 2827 if not licensed: 2828 raise LicenseException('call a q function in a Python process') -> 2829 if q('{.pykx.i.isw x}', self).py(): 2830 args = {i: K(x) for i, x in enumerate(args)} 2831 if args: # Avoid calling `max` on an empty sequence File /usr/local/lib/python3.10/site-packages/pykx/embedded_q.py:137, in EmbeddedQ.__call__(self, query, wait, sync, *args) 135 result = _keval(bytes(wrappers.CharVector(query)), *[wrappers.K(x) for x in args]) 136 if wait is None or wait: --> 137 return factory(result, False) 138 return self('::', wait=True) File /usr/local/lib/python3.10/site-packages/pykx/_wrappers.pyx:501, in pykx._wrappers._factory() File /usr/local/lib/python3.10/site-packages/pykx/_wrappers.pyx:494, in pykx._wrappers.factory() QError: .pykx.i.isw
It is possible to execute q
code spanning multiple lines.
%%q --port 5001
afunc: {[x; y]
x + y
};
afunc[0; 1]
afunc[2; 3]
--------------------------------------------------------------------------- QError Traceback (most recent call last) Cell In[6], line 1 ----> 1 get_ipython().run_cell_magic('q', '--port 5001', 'afunc: {[x; y]\n x + y \n };\nafunc[0; 1]\nafunc[2; 3]\n') File /usr/local/lib/python3.10/site-packages/IPython/core/interactiveshell.py:2422, in InteractiveShell.run_cell_magic(self, magic_name, line, cell) 2420 with self.builtin_trap: 2421 args = (magic_arg_s, cell) -> 2422 result = fn(*args, **kwargs) 2423 return result File /usr/local/lib/python3.10/site-packages/pykx/nbextension.py:70, in q(instructions, code) 67 if port is None: 68 raise kx.QError('--port must be set to create IPC connection in magic command.') ---> 70 _q = kx.SyncQConnection( 71 host, 72 port, 73 username=username, 74 password=password, 75 timeout=timeout, 76 large_messages=large_messages, 77 tls=tls, 78 unix=unix, 79 no_ctx=no_ctx 80 ) 81 try: 82 _q('.Q.ld') File /usr/local/lib/python3.10/site-packages/pykx/ipc.py:633, in SyncQConnection.__new__(cls, *args, **kwargs) 631 def __new__(cls, *args, **kwargs): 632 if 'tls' in kwargs.keys(): --> 633 return SecureQConnection(*args, **kwargs) 634 return object.__new__(cls) File /usr/local/lib/python3.10/site-packages/pykx/ipc.py:1705, in SecureQConnection.__init__(self, host, port, username, password, timeout, large_messages, tls, unix, sync, wait, lock, no_ctx, *args) 1703 warn(self._sync_deprecation_warning, DeprecationWarning) 1704 wait = sync -> 1705 self._init(host, 1706 port, 1707 *args, 1708 username=username, 1709 password=password, 1710 timeout=timeout, 1711 large_messages=large_messages, 1712 tls=tls, 1713 unix=unix, 1714 wait=wait, 1715 lock=lock, 1716 no_ctx=no_ctx, 1717 ) 1718 super().__init__() File /usr/local/lib/python3.10/site-packages/pykx/ipc.py:357, in QConnection._init(self, host, port, username, password, timeout, large_messages, tls, unix, wait, lock, no_ctx) 353 object.__setattr__(self, '_lock', lock) 354 object.__setattr__(self, 'closed', False) 355 object.__setattr__(self, 356 '_handle', --> 357 _ipc.init_handle(host, 358 port, 359 credentials, 360 unix, 361 tls, 362 timeout, 363 large_messages)) 364 if licensed: 365 object.__setattr__(self, '_finalizer', finalize(self, self._close, self._handle)) File /usr/local/lib/python3.10/site-packages/pykx/_ipc.pyx:36, in pykx._ipc.init_handle() File /usr/local/lib/python3.10/site-packages/pykx/wrappers.py:2829, in Composition.__call__(self, *args, **kwargs) 2827 if not licensed: 2828 raise LicenseException('call a q function in a Python process') -> 2829 if q('{.pykx.i.isw x}', self).py(): 2830 args = {i: K(x) for i, x in enumerate(args)} 2831 if args: # Avoid calling `max` on an empty sequence File /usr/local/lib/python3.10/site-packages/pykx/embedded_q.py:137, in EmbeddedQ.__call__(self, query, wait, sync, *args) 135 result = _keval(bytes(wrappers.CharVector(query)), *[wrappers.K(x) for x in args]) 136 if wait is None or wait: --> 137 return factory(result, False) 138 return self('::', wait=True) File /usr/local/lib/python3.10/site-packages/pykx/_wrappers.pyx:501, in pykx._wrappers._factory() File /usr/local/lib/python3.10/site-packages/pykx/_wrappers.pyx:494, in pykx._wrappers.factory() QError: .pykx.i.isw
Using the SQL interface¶
The s)
syntax to run SQL queries against local tables within the q
process.
Note: The s.k_
library must be loaded first to use the SQL interface
%%q --port 5001
\l s.k_
s) select * from tab where a>500 and b<250.0 limit 5
--------------------------------------------------------------------------- QError Traceback (most recent call last) Cell In[7], line 1 ----> 1 get_ipython().run_cell_magic('q', '--port 5001', '\\l s.k_\ns) select * from tab where a>500 and b<250.0 limit 5\n') File /usr/local/lib/python3.10/site-packages/IPython/core/interactiveshell.py:2422, in InteractiveShell.run_cell_magic(self, magic_name, line, cell) 2420 with self.builtin_trap: 2421 args = (magic_arg_s, cell) -> 2422 result = fn(*args, **kwargs) 2423 return result File /usr/local/lib/python3.10/site-packages/pykx/nbextension.py:70, in q(instructions, code) 67 if port is None: 68 raise kx.QError('--port must be set to create IPC connection in magic command.') ---> 70 _q = kx.SyncQConnection( 71 host, 72 port, 73 username=username, 74 password=password, 75 timeout=timeout, 76 large_messages=large_messages, 77 tls=tls, 78 unix=unix, 79 no_ctx=no_ctx 80 ) 81 try: 82 _q('.Q.ld') File /usr/local/lib/python3.10/site-packages/pykx/ipc.py:633, in SyncQConnection.__new__(cls, *args, **kwargs) 631 def __new__(cls, *args, **kwargs): 632 if 'tls' in kwargs.keys(): --> 633 return SecureQConnection(*args, **kwargs) 634 return object.__new__(cls) File /usr/local/lib/python3.10/site-packages/pykx/ipc.py:1705, in SecureQConnection.__init__(self, host, port, username, password, timeout, large_messages, tls, unix, sync, wait, lock, no_ctx, *args) 1703 warn(self._sync_deprecation_warning, DeprecationWarning) 1704 wait = sync -> 1705 self._init(host, 1706 port, 1707 *args, 1708 username=username, 1709 password=password, 1710 timeout=timeout, 1711 large_messages=large_messages, 1712 tls=tls, 1713 unix=unix, 1714 wait=wait, 1715 lock=lock, 1716 no_ctx=no_ctx, 1717 ) 1718 super().__init__() File /usr/local/lib/python3.10/site-packages/pykx/ipc.py:357, in QConnection._init(self, host, port, username, password, timeout, large_messages, tls, unix, wait, lock, no_ctx) 353 object.__setattr__(self, '_lock', lock) 354 object.__setattr__(self, 'closed', False) 355 object.__setattr__(self, 356 '_handle', --> 357 _ipc.init_handle(host, 358 port, 359 credentials, 360 unix, 361 tls, 362 timeout, 363 large_messages)) 364 if licensed: 365 object.__setattr__(self, '_finalizer', finalize(self, self._close, self._handle)) File /usr/local/lib/python3.10/site-packages/pykx/_ipc.pyx:36, in pykx._ipc.init_handle() File /usr/local/lib/python3.10/site-packages/pykx/wrappers.py:2829, in Composition.__call__(self, *args, **kwargs) 2827 if not licensed: 2828 raise LicenseException('call a q function in a Python process') -> 2829 if q('{.pykx.i.isw x}', self).py(): 2830 args = {i: K(x) for i, x in enumerate(args)} 2831 if args: # Avoid calling `max` on an empty sequence File /usr/local/lib/python3.10/site-packages/pykx/embedded_q.py:137, in EmbeddedQ.__call__(self, query, wait, sync, *args) 135 result = _keval(bytes(wrappers.CharVector(query)), *[wrappers.K(x) for x in args]) 136 if wait is None or wait: --> 137 return factory(result, False) 138 return self('::', wait=True) File /usr/local/lib/python3.10/site-packages/pykx/_wrappers.pyx:501, in pykx._wrappers._factory() File /usr/local/lib/python3.10/site-packages/pykx/_wrappers.pyx:494, in pykx._wrappers.factory() QError: .pykx.i.isw
Using namespaces¶
You can also use q
namespaces, and switch between them using \d
.
Note: The namespace is reset back to the base namespace .
between cells.
%%q --port 5001
\d .example
f: {[x] til x};
--------------------------------------------------------------------------- QError Traceback (most recent call last) Cell In[8], line 1 ----> 1 get_ipython().run_cell_magic('q', '--port 5001', '\\d .example\nf: {[x] til x};\n') File /usr/local/lib/python3.10/site-packages/IPython/core/interactiveshell.py:2422, in InteractiveShell.run_cell_magic(self, magic_name, line, cell) 2420 with self.builtin_trap: 2421 args = (magic_arg_s, cell) -> 2422 result = fn(*args, **kwargs) 2423 return result File /usr/local/lib/python3.10/site-packages/pykx/nbextension.py:70, in q(instructions, code) 67 if port is None: 68 raise kx.QError('--port must be set to create IPC connection in magic command.') ---> 70 _q = kx.SyncQConnection( 71 host, 72 port, 73 username=username, 74 password=password, 75 timeout=timeout, 76 large_messages=large_messages, 77 tls=tls, 78 unix=unix, 79 no_ctx=no_ctx 80 ) 81 try: 82 _q('.Q.ld') File /usr/local/lib/python3.10/site-packages/pykx/ipc.py:633, in SyncQConnection.__new__(cls, *args, **kwargs) 631 def __new__(cls, *args, **kwargs): 632 if 'tls' in kwargs.keys(): --> 633 return SecureQConnection(*args, **kwargs) 634 return object.__new__(cls) File /usr/local/lib/python3.10/site-packages/pykx/ipc.py:1705, in SecureQConnection.__init__(self, host, port, username, password, timeout, large_messages, tls, unix, sync, wait, lock, no_ctx, *args) 1703 warn(self._sync_deprecation_warning, DeprecationWarning) 1704 wait = sync -> 1705 self._init(host, 1706 port, 1707 *args, 1708 username=username, 1709 password=password, 1710 timeout=timeout, 1711 large_messages=large_messages, 1712 tls=tls, 1713 unix=unix, 1714 wait=wait, 1715 lock=lock, 1716 no_ctx=no_ctx, 1717 ) 1718 super().__init__() File /usr/local/lib/python3.10/site-packages/pykx/ipc.py:357, in QConnection._init(self, host, port, username, password, timeout, large_messages, tls, unix, wait, lock, no_ctx) 353 object.__setattr__(self, '_lock', lock) 354 object.__setattr__(self, 'closed', False) 355 object.__setattr__(self, 356 '_handle', --> 357 _ipc.init_handle(host, 358 port, 359 credentials, 360 unix, 361 tls, 362 timeout, 363 large_messages)) 364 if licensed: 365 object.__setattr__(self, '_finalizer', finalize(self, self._close, self._handle)) File /usr/local/lib/python3.10/site-packages/pykx/_ipc.pyx:36, in pykx._ipc.init_handle() File /usr/local/lib/python3.10/site-packages/pykx/wrappers.py:2829, in Composition.__call__(self, *args, **kwargs) 2827 if not licensed: 2828 raise LicenseException('call a q function in a Python process') -> 2829 if q('{.pykx.i.isw x}', self).py(): 2830 args = {i: K(x) for i, x in enumerate(args)} 2831 if args: # Avoid calling `max` on an empty sequence File /usr/local/lib/python3.10/site-packages/pykx/embedded_q.py:137, in EmbeddedQ.__call__(self, query, wait, sync, *args) 135 result = _keval(bytes(wrappers.CharVector(query)), *[wrappers.K(x) for x in args]) 136 if wait is None or wait: --> 137 return factory(result, False) 138 return self('::', wait=True) File /usr/local/lib/python3.10/site-packages/pykx/_wrappers.pyx:501, in pykx._wrappers._factory() File /usr/local/lib/python3.10/site-packages/pykx/_wrappers.pyx:494, in pykx._wrappers.factory() QError: .pykx.i.isw
%%q --port 5001
\d
.example.f[10]
--------------------------------------------------------------------------- QError Traceback (most recent call last) Cell In[9], line 1 ----> 1 get_ipython().run_cell_magic('q', '--port 5001', '\\d\n.example.f[10]\n') File /usr/local/lib/python3.10/site-packages/IPython/core/interactiveshell.py:2422, in InteractiveShell.run_cell_magic(self, magic_name, line, cell) 2420 with self.builtin_trap: 2421 args = (magic_arg_s, cell) -> 2422 result = fn(*args, **kwargs) 2423 return result File /usr/local/lib/python3.10/site-packages/pykx/nbextension.py:70, in q(instructions, code) 67 if port is None: 68 raise kx.QError('--port must be set to create IPC connection in magic command.') ---> 70 _q = kx.SyncQConnection( 71 host, 72 port, 73 username=username, 74 password=password, 75 timeout=timeout, 76 large_messages=large_messages, 77 tls=tls, 78 unix=unix, 79 no_ctx=no_ctx 80 ) 81 try: 82 _q('.Q.ld') File /usr/local/lib/python3.10/site-packages/pykx/ipc.py:633, in SyncQConnection.__new__(cls, *args, **kwargs) 631 def __new__(cls, *args, **kwargs): 632 if 'tls' in kwargs.keys(): --> 633 return SecureQConnection(*args, **kwargs) 634 return object.__new__(cls) File /usr/local/lib/python3.10/site-packages/pykx/ipc.py:1705, in SecureQConnection.__init__(self, host, port, username, password, timeout, large_messages, tls, unix, sync, wait, lock, no_ctx, *args) 1703 warn(self._sync_deprecation_warning, DeprecationWarning) 1704 wait = sync -> 1705 self._init(host, 1706 port, 1707 *args, 1708 username=username, 1709 password=password, 1710 timeout=timeout, 1711 large_messages=large_messages, 1712 tls=tls, 1713 unix=unix, 1714 wait=wait, 1715 lock=lock, 1716 no_ctx=no_ctx, 1717 ) 1718 super().__init__() File /usr/local/lib/python3.10/site-packages/pykx/ipc.py:357, in QConnection._init(self, host, port, username, password, timeout, large_messages, tls, unix, wait, lock, no_ctx) 353 object.__setattr__(self, '_lock', lock) 354 object.__setattr__(self, 'closed', False) 355 object.__setattr__(self, 356 '_handle', --> 357 _ipc.init_handle(host, 358 port, 359 credentials, 360 unix, 361 tls, 362 timeout, 363 large_messages)) 364 if licensed: 365 object.__setattr__(self, '_finalizer', finalize(self, self._close, self._handle)) File /usr/local/lib/python3.10/site-packages/pykx/_ipc.pyx:36, in pykx._ipc.init_handle() File /usr/local/lib/python3.10/site-packages/pykx/wrappers.py:2829, in Composition.__call__(self, *args, **kwargs) 2827 if not licensed: 2828 raise LicenseException('call a q function in a Python process') -> 2829 if q('{.pykx.i.isw x}', self).py(): 2830 args = {i: K(x) for i, x in enumerate(args)} 2831 if args: # Avoid calling `max` on an empty sequence File /usr/local/lib/python3.10/site-packages/pykx/embedded_q.py:137, in EmbeddedQ.__call__(self, query, wait, sync, *args) 135 result = _keval(bytes(wrappers.CharVector(query)), *[wrappers.K(x) for x in args]) 136 if wait is None or wait: --> 137 return factory(result, False) 138 return self('::', wait=True) File /usr/local/lib/python3.10/site-packages/pykx/_wrappers.pyx:501, in pykx._wrappers._factory() File /usr/local/lib/python3.10/site-packages/pykx/_wrappers.pyx:494, in pykx._wrappers.factory() QError: .pykx.i.isw
# Shutdown the q process we were connected to for the IPC demo
proc.kill()