Jupyter q Magic Command
import pykx as kx
Create the external q process¶
To run this example, the Python code in the following cell will do the equivalent to executing the following in a terminal:
$ q -p 5001
import subprocess
import time
proc = subprocess.Popen(
('q', '-p', '5001')
)
time.sleep(5)
Executing against Embedded q¶
A cell begining with %%q
will execute q within PyKX
's EmbeddedQ
module.
%%q
til 10
0 1 2 3 4 5 6 7 8 9
Executing against an external q process over IPC¶
Connection information can also be included after the %%q
to connect to a remote q
process over
IPC.
Here is the list of currently supported connection parameters. If they specify a type a second value is expected to follow them to be used as the parameter. If no type follows them they can be used as a stand alone flag.
--host: A string object denoting the host to connect to
--port: An int object denoting the port to connect over
--user: A str object denoting the username to use when connecting
--password: A str object denoting the password to use when connecting
--timeout: A float object denoting the time in seconds before the query
times out, defaults to no timeout
--nolarge: Disable messages over 2GB being sent / received
--tls: Use a tls connection
--unix: Use a unix connection
--noctx: Disable the context interface
Connect to a q server running on localhost
at port 5001
as user
using password password
and disable the context interface.
%%q --host localhost --port 5001 --user user --pass password --noctx
til 10
--------------------------------------------------------------------------- QError Traceback (most recent call last) Cell In[4], line 1 ----> 1 get_ipython().run_cell_magic('q', '--host localhost --port 5001 --user user --pass password --noctx', 'til 10\n') File /usr/local/lib/python3.10/site-packages/IPython/core/interactiveshell.py:2430, in InteractiveShell.run_cell_magic(self, magic_name, line, cell) 2428 with self.builtin_trap: 2429 args = (magic_arg_s, cell) -> 2430 result = fn(*args, **kwargs) 2432 # The code below prevents the output from being displayed 2433 # when using magics with decodator @output_can_be_silenced 2434 # when the last Python token in the expression is a ';'. 2435 if getattr(fn, magic.MAGIC_OUTPUT_CAN_BE_SILENCED, False): File /usr/local/lib/python3.10/site-packages/pykx/nbextension.py:70, in q(instructions, code) 67 if port is None: 68 raise kx.QError('--port must be set to create IPC connection in magic command.') ---> 70 _q = kx.SyncQConnection( 71 host, 72 port, 73 username=username, 74 password=password, 75 timeout=timeout, 76 large_messages=large_messages, 77 tls=tls, 78 unix=unix, 79 no_ctx=no_ctx 80 ) 81 try: 82 _q('.Q.ld') File /usr/local/lib/python3.10/site-packages/pykx/ipc.py:719, in SyncQConnection.__init__(self, host, port, username, password, timeout, large_messages, tls, unix, sync, wait, lock, no_ctx, *args) 717 warn(self._sync_deprecation_warning, DeprecationWarning) 718 wait = sync --> 719 self._init(host, 720 port, 721 *args, 722 username=username, 723 password=password, 724 timeout=timeout, 725 large_messages=large_messages, 726 tls=tls, 727 unix=unix, 728 wait=wait, 729 lock=lock, 730 no_ctx=no_ctx, 731 ) 732 super().__init__() File /usr/local/lib/python3.10/site-packages/pykx/ipc.py:357, in QConnection._init(self, host, port, username, password, timeout, large_messages, tls, unix, wait, lock, no_ctx) 353 object.__setattr__(self, '_lock', lock) 354 object.__setattr__(self, 'closed', False) 355 object.__setattr__(self, 356 '_handle', --> 357 _ipc.init_handle(host, 358 port, 359 credentials, 360 unix, 361 tls, 362 timeout, 363 large_messages)) 364 if licensed: 365 object.__setattr__(self, '_finalizer', finalize(self, self._close, self._handle)) File /usr/local/lib/python3.10/site-packages/pykx/_ipc.pyx:34, in pykx._ipc.init_handle() File /usr/local/lib/python3.10/site-packages/pykx/wrappers.py:2934, in Composition.__call__(self, *args, **kwargs) 2932 if not licensed: 2933 raise LicenseException('call a q function in a Python process') -> 2934 if q('{.pykx.i.isw x}', self).py(): 2935 args = {i: K(x) for i, x in enumerate(args)} 2936 if args: # Avoid calling `max` on an empty sequence File /usr/local/lib/python3.10/site-packages/pykx/embedded_q.py:138, in EmbeddedQ.__call__(self, query, wait, sync, *args) 136 result = _keval(bytes(wrappers.CharVector(query)), *[wrappers.K(x) for x in args]) 137 if wait is None or wait: --> 138 return factory(result, False) 139 return self('::', wait=True) File /usr/local/lib/python3.10/site-packages/pykx/_wrappers.pyx:501, in pykx._wrappers._factory() File /usr/local/lib/python3.10/site-packages/pykx/_wrappers.pyx:494, in pykx._wrappers.factory() QError: .pykx.i.isw
All connection arguments are optional with the exception of the --port
argument. If --host
is not provided localhost
will be used as the default host.
%%q --port 5001
tab:([]a:1000?1000; b:1000?500.0; c:1000?`AAPL`MSFT`GOOG);
--------------------------------------------------------------------------- QError Traceback (most recent call last) Cell In[5], line 1 ----> 1 get_ipython().run_cell_magic('q', '--port 5001', 'tab:([]a:1000?1000; b:1000?500.0; c:1000?`AAPL`MSFT`GOOG);\n') File /usr/local/lib/python3.10/site-packages/IPython/core/interactiveshell.py:2430, in InteractiveShell.run_cell_magic(self, magic_name, line, cell) 2428 with self.builtin_trap: 2429 args = (magic_arg_s, cell) -> 2430 result = fn(*args, **kwargs) 2432 # The code below prevents the output from being displayed 2433 # when using magics with decodator @output_can_be_silenced 2434 # when the last Python token in the expression is a ';'. 2435 if getattr(fn, magic.MAGIC_OUTPUT_CAN_BE_SILENCED, False): File /usr/local/lib/python3.10/site-packages/pykx/nbextension.py:70, in q(instructions, code) 67 if port is None: 68 raise kx.QError('--port must be set to create IPC connection in magic command.') ---> 70 _q = kx.SyncQConnection( 71 host, 72 port, 73 username=username, 74 password=password, 75 timeout=timeout, 76 large_messages=large_messages, 77 tls=tls, 78 unix=unix, 79 no_ctx=no_ctx 80 ) 81 try: 82 _q('.Q.ld') File /usr/local/lib/python3.10/site-packages/pykx/ipc.py:719, in SyncQConnection.__init__(self, host, port, username, password, timeout, large_messages, tls, unix, sync, wait, lock, no_ctx, *args) 717 warn(self._sync_deprecation_warning, DeprecationWarning) 718 wait = sync --> 719 self._init(host, 720 port, 721 *args, 722 username=username, 723 password=password, 724 timeout=timeout, 725 large_messages=large_messages, 726 tls=tls, 727 unix=unix, 728 wait=wait, 729 lock=lock, 730 no_ctx=no_ctx, 731 ) 732 super().__init__() File /usr/local/lib/python3.10/site-packages/pykx/ipc.py:357, in QConnection._init(self, host, port, username, password, timeout, large_messages, tls, unix, wait, lock, no_ctx) 353 object.__setattr__(self, '_lock', lock) 354 object.__setattr__(self, 'closed', False) 355 object.__setattr__(self, 356 '_handle', --> 357 _ipc.init_handle(host, 358 port, 359 credentials, 360 unix, 361 tls, 362 timeout, 363 large_messages)) 364 if licensed: 365 object.__setattr__(self, '_finalizer', finalize(self, self._close, self._handle)) File /usr/local/lib/python3.10/site-packages/pykx/_ipc.pyx:34, in pykx._ipc.init_handle() File /usr/local/lib/python3.10/site-packages/pykx/wrappers.py:2934, in Composition.__call__(self, *args, **kwargs) 2932 if not licensed: 2933 raise LicenseException('call a q function in a Python process') -> 2934 if q('{.pykx.i.isw x}', self).py(): 2935 args = {i: K(x) for i, x in enumerate(args)} 2936 if args: # Avoid calling `max` on an empty sequence File /usr/local/lib/python3.10/site-packages/pykx/embedded_q.py:138, in EmbeddedQ.__call__(self, query, wait, sync, *args) 136 result = _keval(bytes(wrappers.CharVector(query)), *[wrappers.K(x) for x in args]) 137 if wait is None or wait: --> 138 return factory(result, False) 139 return self('::', wait=True) File /usr/local/lib/python3.10/site-packages/pykx/_wrappers.pyx:501, in pykx._wrappers._factory() File /usr/local/lib/python3.10/site-packages/pykx/_wrappers.pyx:494, in pykx._wrappers.factory() QError: .pykx.i.isw
It is possible to execute q
code spanning multiple lines.
%%q --port 5001
afunc: {[x; y]
x + y
};
afunc[0; 1]
afunc[2; 3]
--------------------------------------------------------------------------- QError Traceback (most recent call last) Cell In[6], line 1 ----> 1 get_ipython().run_cell_magic('q', '--port 5001', 'afunc: {[x; y]\n x + y \n };\nafunc[0; 1]\nafunc[2; 3]\n') File /usr/local/lib/python3.10/site-packages/IPython/core/interactiveshell.py:2430, in InteractiveShell.run_cell_magic(self, magic_name, line, cell) 2428 with self.builtin_trap: 2429 args = (magic_arg_s, cell) -> 2430 result = fn(*args, **kwargs) 2432 # The code below prevents the output from being displayed 2433 # when using magics with decodator @output_can_be_silenced 2434 # when the last Python token in the expression is a ';'. 2435 if getattr(fn, magic.MAGIC_OUTPUT_CAN_BE_SILENCED, False): File /usr/local/lib/python3.10/site-packages/pykx/nbextension.py:70, in q(instructions, code) 67 if port is None: 68 raise kx.QError('--port must be set to create IPC connection in magic command.') ---> 70 _q = kx.SyncQConnection( 71 host, 72 port, 73 username=username, 74 password=password, 75 timeout=timeout, 76 large_messages=large_messages, 77 tls=tls, 78 unix=unix, 79 no_ctx=no_ctx 80 ) 81 try: 82 _q('.Q.ld') File /usr/local/lib/python3.10/site-packages/pykx/ipc.py:719, in SyncQConnection.__init__(self, host, port, username, password, timeout, large_messages, tls, unix, sync, wait, lock, no_ctx, *args) 717 warn(self._sync_deprecation_warning, DeprecationWarning) 718 wait = sync --> 719 self._init(host, 720 port, 721 *args, 722 username=username, 723 password=password, 724 timeout=timeout, 725 large_messages=large_messages, 726 tls=tls, 727 unix=unix, 728 wait=wait, 729 lock=lock, 730 no_ctx=no_ctx, 731 ) 732 super().__init__() File /usr/local/lib/python3.10/site-packages/pykx/ipc.py:357, in QConnection._init(self, host, port, username, password, timeout, large_messages, tls, unix, wait, lock, no_ctx) 353 object.__setattr__(self, '_lock', lock) 354 object.__setattr__(self, 'closed', False) 355 object.__setattr__(self, 356 '_handle', --> 357 _ipc.init_handle(host, 358 port, 359 credentials, 360 unix, 361 tls, 362 timeout, 363 large_messages)) 364 if licensed: 365 object.__setattr__(self, '_finalizer', finalize(self, self._close, self._handle)) File /usr/local/lib/python3.10/site-packages/pykx/_ipc.pyx:34, in pykx._ipc.init_handle() File /usr/local/lib/python3.10/site-packages/pykx/wrappers.py:2934, in Composition.__call__(self, *args, **kwargs) 2932 if not licensed: 2933 raise LicenseException('call a q function in a Python process') -> 2934 if q('{.pykx.i.isw x}', self).py(): 2935 args = {i: K(x) for i, x in enumerate(args)} 2936 if args: # Avoid calling `max` on an empty sequence File /usr/local/lib/python3.10/site-packages/pykx/embedded_q.py:138, in EmbeddedQ.__call__(self, query, wait, sync, *args) 136 result = _keval(bytes(wrappers.CharVector(query)), *[wrappers.K(x) for x in args]) 137 if wait is None or wait: --> 138 return factory(result, False) 139 return self('::', wait=True) File /usr/local/lib/python3.10/site-packages/pykx/_wrappers.pyx:501, in pykx._wrappers._factory() File /usr/local/lib/python3.10/site-packages/pykx/_wrappers.pyx:494, in pykx._wrappers.factory() QError: .pykx.i.isw
Using the SQL interface¶
The s)
syntax to run SQL queries against local tables within the q
process.
Note: The s.k_
library must be loaded first to use the SQL interface
%%q --port 5001
\l s.k_
s) select * from tab where a>500 and b<250.0 limit 5
--------------------------------------------------------------------------- QError Traceback (most recent call last) Cell In[7], line 1 ----> 1 get_ipython().run_cell_magic('q', '--port 5001', '\\l s.k_\ns) select * from tab where a>500 and b<250.0 limit 5\n') File /usr/local/lib/python3.10/site-packages/IPython/core/interactiveshell.py:2430, in InteractiveShell.run_cell_magic(self, magic_name, line, cell) 2428 with self.builtin_trap: 2429 args = (magic_arg_s, cell) -> 2430 result = fn(*args, **kwargs) 2432 # The code below prevents the output from being displayed 2433 # when using magics with decodator @output_can_be_silenced 2434 # when the last Python token in the expression is a ';'. 2435 if getattr(fn, magic.MAGIC_OUTPUT_CAN_BE_SILENCED, False): File /usr/local/lib/python3.10/site-packages/pykx/nbextension.py:70, in q(instructions, code) 67 if port is None: 68 raise kx.QError('--port must be set to create IPC connection in magic command.') ---> 70 _q = kx.SyncQConnection( 71 host, 72 port, 73 username=username, 74 password=password, 75 timeout=timeout, 76 large_messages=large_messages, 77 tls=tls, 78 unix=unix, 79 no_ctx=no_ctx 80 ) 81 try: 82 _q('.Q.ld') File /usr/local/lib/python3.10/site-packages/pykx/ipc.py:719, in SyncQConnection.__init__(self, host, port, username, password, timeout, large_messages, tls, unix, sync, wait, lock, no_ctx, *args) 717 warn(self._sync_deprecation_warning, DeprecationWarning) 718 wait = sync --> 719 self._init(host, 720 port, 721 *args, 722 username=username, 723 password=password, 724 timeout=timeout, 725 large_messages=large_messages, 726 tls=tls, 727 unix=unix, 728 wait=wait, 729 lock=lock, 730 no_ctx=no_ctx, 731 ) 732 super().__init__() File /usr/local/lib/python3.10/site-packages/pykx/ipc.py:357, in QConnection._init(self, host, port, username, password, timeout, large_messages, tls, unix, wait, lock, no_ctx) 353 object.__setattr__(self, '_lock', lock) 354 object.__setattr__(self, 'closed', False) 355 object.__setattr__(self, 356 '_handle', --> 357 _ipc.init_handle(host, 358 port, 359 credentials, 360 unix, 361 tls, 362 timeout, 363 large_messages)) 364 if licensed: 365 object.__setattr__(self, '_finalizer', finalize(self, self._close, self._handle)) File /usr/local/lib/python3.10/site-packages/pykx/_ipc.pyx:34, in pykx._ipc.init_handle() File /usr/local/lib/python3.10/site-packages/pykx/wrappers.py:2934, in Composition.__call__(self, *args, **kwargs) 2932 if not licensed: 2933 raise LicenseException('call a q function in a Python process') -> 2934 if q('{.pykx.i.isw x}', self).py(): 2935 args = {i: K(x) for i, x in enumerate(args)} 2936 if args: # Avoid calling `max` on an empty sequence File /usr/local/lib/python3.10/site-packages/pykx/embedded_q.py:138, in EmbeddedQ.__call__(self, query, wait, sync, *args) 136 result = _keval(bytes(wrappers.CharVector(query)), *[wrappers.K(x) for x in args]) 137 if wait is None or wait: --> 138 return factory(result, False) 139 return self('::', wait=True) File /usr/local/lib/python3.10/site-packages/pykx/_wrappers.pyx:501, in pykx._wrappers._factory() File /usr/local/lib/python3.10/site-packages/pykx/_wrappers.pyx:494, in pykx._wrappers.factory() QError: .pykx.i.isw
Using namespaces¶
You can also use q
namespaces, and switch between them using \d
.
Note: The namespace is reset back to the base namespace .
between cells.
%%q --port 5001
\d .example
f: {[x] til x};
--------------------------------------------------------------------------- QError Traceback (most recent call last) Cell In[8], line 1 ----> 1 get_ipython().run_cell_magic('q', '--port 5001', '\\d .example\nf: {[x] til x};\n') File /usr/local/lib/python3.10/site-packages/IPython/core/interactiveshell.py:2430, in InteractiveShell.run_cell_magic(self, magic_name, line, cell) 2428 with self.builtin_trap: 2429 args = (magic_arg_s, cell) -> 2430 result = fn(*args, **kwargs) 2432 # The code below prevents the output from being displayed 2433 # when using magics with decodator @output_can_be_silenced 2434 # when the last Python token in the expression is a ';'. 2435 if getattr(fn, magic.MAGIC_OUTPUT_CAN_BE_SILENCED, False): File /usr/local/lib/python3.10/site-packages/pykx/nbextension.py:70, in q(instructions, code) 67 if port is None: 68 raise kx.QError('--port must be set to create IPC connection in magic command.') ---> 70 _q = kx.SyncQConnection( 71 host, 72 port, 73 username=username, 74 password=password, 75 timeout=timeout, 76 large_messages=large_messages, 77 tls=tls, 78 unix=unix, 79 no_ctx=no_ctx 80 ) 81 try: 82 _q('.Q.ld') File /usr/local/lib/python3.10/site-packages/pykx/ipc.py:719, in SyncQConnection.__init__(self, host, port, username, password, timeout, large_messages, tls, unix, sync, wait, lock, no_ctx, *args) 717 warn(self._sync_deprecation_warning, DeprecationWarning) 718 wait = sync --> 719 self._init(host, 720 port, 721 *args, 722 username=username, 723 password=password, 724 timeout=timeout, 725 large_messages=large_messages, 726 tls=tls, 727 unix=unix, 728 wait=wait, 729 lock=lock, 730 no_ctx=no_ctx, 731 ) 732 super().__init__() File /usr/local/lib/python3.10/site-packages/pykx/ipc.py:357, in QConnection._init(self, host, port, username, password, timeout, large_messages, tls, unix, wait, lock, no_ctx) 353 object.__setattr__(self, '_lock', lock) 354 object.__setattr__(self, 'closed', False) 355 object.__setattr__(self, 356 '_handle', --> 357 _ipc.init_handle(host, 358 port, 359 credentials, 360 unix, 361 tls, 362 timeout, 363 large_messages)) 364 if licensed: 365 object.__setattr__(self, '_finalizer', finalize(self, self._close, self._handle)) File /usr/local/lib/python3.10/site-packages/pykx/_ipc.pyx:34, in pykx._ipc.init_handle() File /usr/local/lib/python3.10/site-packages/pykx/wrappers.py:2934, in Composition.__call__(self, *args, **kwargs) 2932 if not licensed: 2933 raise LicenseException('call a q function in a Python process') -> 2934 if q('{.pykx.i.isw x}', self).py(): 2935 args = {i: K(x) for i, x in enumerate(args)} 2936 if args: # Avoid calling `max` on an empty sequence File /usr/local/lib/python3.10/site-packages/pykx/embedded_q.py:138, in EmbeddedQ.__call__(self, query, wait, sync, *args) 136 result = _keval(bytes(wrappers.CharVector(query)), *[wrappers.K(x) for x in args]) 137 if wait is None or wait: --> 138 return factory(result, False) 139 return self('::', wait=True) File /usr/local/lib/python3.10/site-packages/pykx/_wrappers.pyx:501, in pykx._wrappers._factory() File /usr/local/lib/python3.10/site-packages/pykx/_wrappers.pyx:494, in pykx._wrappers.factory() QError: .pykx.i.isw
%%q --port 5001
\d
.example.f[10]
--------------------------------------------------------------------------- QError Traceback (most recent call last) Cell In[9], line 1 ----> 1 get_ipython().run_cell_magic('q', '--port 5001', '\\d\n.example.f[10]\n') File /usr/local/lib/python3.10/site-packages/IPython/core/interactiveshell.py:2430, in InteractiveShell.run_cell_magic(self, magic_name, line, cell) 2428 with self.builtin_trap: 2429 args = (magic_arg_s, cell) -> 2430 result = fn(*args, **kwargs) 2432 # The code below prevents the output from being displayed 2433 # when using magics with decodator @output_can_be_silenced 2434 # when the last Python token in the expression is a ';'. 2435 if getattr(fn, magic.MAGIC_OUTPUT_CAN_BE_SILENCED, False): File /usr/local/lib/python3.10/site-packages/pykx/nbextension.py:70, in q(instructions, code) 67 if port is None: 68 raise kx.QError('--port must be set to create IPC connection in magic command.') ---> 70 _q = kx.SyncQConnection( 71 host, 72 port, 73 username=username, 74 password=password, 75 timeout=timeout, 76 large_messages=large_messages, 77 tls=tls, 78 unix=unix, 79 no_ctx=no_ctx 80 ) 81 try: 82 _q('.Q.ld') File /usr/local/lib/python3.10/site-packages/pykx/ipc.py:719, in SyncQConnection.__init__(self, host, port, username, password, timeout, large_messages, tls, unix, sync, wait, lock, no_ctx, *args) 717 warn(self._sync_deprecation_warning, DeprecationWarning) 718 wait = sync --> 719 self._init(host, 720 port, 721 *args, 722 username=username, 723 password=password, 724 timeout=timeout, 725 large_messages=large_messages, 726 tls=tls, 727 unix=unix, 728 wait=wait, 729 lock=lock, 730 no_ctx=no_ctx, 731 ) 732 super().__init__() File /usr/local/lib/python3.10/site-packages/pykx/ipc.py:357, in QConnection._init(self, host, port, username, password, timeout, large_messages, tls, unix, wait, lock, no_ctx) 353 object.__setattr__(self, '_lock', lock) 354 object.__setattr__(self, 'closed', False) 355 object.__setattr__(self, 356 '_handle', --> 357 _ipc.init_handle(host, 358 port, 359 credentials, 360 unix, 361 tls, 362 timeout, 363 large_messages)) 364 if licensed: 365 object.__setattr__(self, '_finalizer', finalize(self, self._close, self._handle)) File /usr/local/lib/python3.10/site-packages/pykx/_ipc.pyx:34, in pykx._ipc.init_handle() File /usr/local/lib/python3.10/site-packages/pykx/wrappers.py:2934, in Composition.__call__(self, *args, **kwargs) 2932 if not licensed: 2933 raise LicenseException('call a q function in a Python process') -> 2934 if q('{.pykx.i.isw x}', self).py(): 2935 args = {i: K(x) for i, x in enumerate(args)} 2936 if args: # Avoid calling `max` on an empty sequence File /usr/local/lib/python3.10/site-packages/pykx/embedded_q.py:138, in EmbeddedQ.__call__(self, query, wait, sync, *args) 136 result = _keval(bytes(wrappers.CharVector(query)), *[wrappers.K(x) for x in args]) 137 if wait is None or wait: --> 138 return factory(result, False) 139 return self('::', wait=True) File /usr/local/lib/python3.10/site-packages/pykx/_wrappers.pyx:501, in pykx._wrappers._factory() File /usr/local/lib/python3.10/site-packages/pykx/_wrappers.pyx:494, in pykx._wrappers.factory() QError: .pykx.i.isw
# Shutdown the q process we were connected to for the IPC demo
proc.kill()