Initial commit.

.gitignore

@@ -1,3 +1,4 @@
+*.ipynb
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

README.md

@@ -1,2 +1,62 @@
-# jupyter_micropython_kernel
+# Jupyter MicroPython Kernel
-Jupyter kernel to interact with a MicroPython or CircuitPython board over its serial REPL.
+
+Jupyter kernel to interact with a MicroPython or CircuitPython board over its serial REPL.  Note this is _highly_ experimental and still alpha/beta quality.  Try it out but don't be surprised if it behaves in odd or unexpected ways!
+
+## Installation
+
+First install Jupyter: http://jupyter.org/install.html
+
+Then clone this repository and install the setup.py (assuming python 3.0, be
+sure to use the same version of python as Jupyter is installed with):
+
+    python3 setup.py install
+
+On Mac OSX and some Linux flavors you might need to run as root with sudo flavor
+the above command.  Make sure the installation completes successfully and that
+you do not see any error messages.
+
+Finally create a Jupyter kernel specification for the serial port and baud rate
+of your MicroPython board.  Unfortunately there is no UI or ability to pick the
+serial port/baud from the notebook so you'll have to bake this in to a kernel
+configuration.
+
+From the Jupyter kernel docs find your user specific Jupyter kernel spec location: http://jupyter-client.readthedocs.io/en/latest/kernels.html#kernel-specs  You want the **user** location:
+
+*   Windows: %APPDATA%\jupyter\kernels (note if you aren't sure where this is located see: http://www.pcworld.com/article/2690709/windows/whats-in-the-hidden-windows-appdata-folder-and-how-to-find-it-if-you-need-it.html)
+*   macOS: ~/Library/Jupyter/kernels
+*   Linux: ~/.local/share/jupyter/kernels
+
+Create the above kernels folder if it doesn't already exist. Then inside the
+kernels folder create a new folder called 'micropython' and copy the included
+kernel.json file inside it.
+
+Open the copied kernel.json file and edit it so the 4th line:
+
+    "/dev/tty.SLAB_USBtoUART", "115200",
+
+Is the serial name and baud rate of your MicroPython board.  For example if using COM4 and 115200 baud you would change it to:
+
+    "COM4", "115200",
+
+Also change the display name of the kernel on line 6:
+
+    "display_name": "MicroPython - /dev/tty.SLAB_USBtoUART",
+
+Set a value that describes your board, like:
+
+    "display_name": "MicroPython - COM4",
+
+This is the name you will see in Jupyter's notebook UI when picking the kernel
+to start.  You don't need to change any other config in the kernel.json.  Be
+very careful to make sure all the commands, double quotes, etc. are present
+(this needs to be a valid JSON formatted file).
+
+At this point you should have the following file: <Jupyter kernel spec location>/micropython/kernel.json
+
+Now run Jupyter notebooks:
+
+    jupyter notebook
+
+In the notebook click the New button in the upper right, you should see your
+MicroPython kernel display name listed.  Click it to create a notebook using
+that board connection (make sure the board is connected first!).

jupyter_micropython_kernel/__main__.py

@@ -0,0 +1,22 @@
+import logging
+import sys
+
+from ipykernel.kernelapp import IPKernelApp
+
+from jupyter_micropython_kernel.kernel import make_micropython_kernel
+
+
+logging.basicConfig(level=logging.DEBUG)
+
+# Parse out required port name and baud rate parameters.  Remove them from argv
+# because the IPKernelApp will go on to parse the arguments and get confused
+# if it finds extra args like them.  Not super elegant but I see no other way
+# to pass custom arguments/parameters to kernels.
+if len(sys.argv) < 3:
+    raise RuntimeError('Expected at least PORT and BAUD parameters!')
+port = sys.argv[1]
+baud = sys.argv[2]
+del sys.argv[1:3]
+
+# Create and launch the kernel.
+IPKernelApp.launch_instance(kernel_class=make_micropython_kernel(port, baud))

jupyter_micropython_kernel/kernel.py

@@ -0,0 +1,78 @@
+import logging
+
+from ipykernel.kernelbase import Kernel
+import pkg_resources
+
+from jupyter_micropython_kernel.pyboard import Pyboard
+
+
+# Create global logger for debug messages.
+logger = logging.getLogger(__name__)
+# Get version from setuptools.  This is used to tell Jupyter the version of
+# this kernel.
+version = pkg_resources.require('jupyter_micropython_kernel')[0].version
+
+
+def make_micropython_kernel(port, baud):
+    # Create a MicroPython kernel class and return it.  This is done so instance
+    # specific config like port and baud rate can be set.  Unfortunately the
+    # IPython kernel wrapper design doesn't appear to allow for
+    # instance-specific configuration (i.e. you don't create the instance
+    # and call its constructor to control how it's built).  As a workaround
+    # we'll just build a separate kernel class with a class-specific port and
+    # baud rate baked in.
+    class MicroPythonKernel(Kernel):
+        implementation = 'micropython'
+        implementation_version = version
+        language = 'micropython'
+        language_version = version
+        language_info = {
+            'name': 'python',
+            'mimetype': 'text/x-python',
+            'file_extension': '.py',
+        }
+        banner = 'MicroPython Kernel - port: {} - baud: {}'.format(port, baud)
+
+        def __init__(self, **kwargs):
+            super().__init__(**kwargs)
+            # Open MicroPython board and enter raw REPL which resets the board
+            # and makes it ready to accept commands.
+            logger.debug('Opening MicroPython board connection on port: {} baud: {}'.format(port, baud))
+            self._board = Pyboard(port, baudrate=baud)
+            self._board.enter_raw_repl()
+
+        def do_execute(self, code, silent, store_history=True,
+                       user_expressions=None, allow_stdin=False):
+            # Run the specified code on the connected MicroPython board.
+            result, error = self._board.exec_raw(code)
+            logger.debug('Result: {} Error: {}'.format(result, error))
+            # If there was an error send it back, otherwise send the result.
+            # Make sure to convert this to a JSON serializable string from the
+            # raw bytes (assumes UTF-8 encoding).  This doesn't really feel
+            # like the right way to send back errors but the docs are really
+            # hard to figure out what's expected (do you send a stream_content
+            # with name stderr? is there more to return?).
+            failed = error is not None and len(error) > 0
+            response = result.decode('utf-8') if not failed else error.decode('utf-8')
+            # Send the result when not in silent mode.
+            if not silent:
+                stream_content = {'name': 'stdout', 'text': response }
+                self.send_response(self.iopub_socket, 'stream', stream_content)
+            return {'status': 'ok' if not failed else 'error',
+                    # The base class increments the execution count
+                    'execution_count': self.execution_count,
+                    'payload': [],
+                    'user_expressions': {},
+                   }
+
+        def do_shutdown(self, restart):
+            # Be nice and try to exit the raw REPL, but ignore any failure
+            # in case the connection is already dead.
+            logger.debug('Shutting down MicroPython board connection.')
+            try:
+                self._board.exit_raw_repl()
+            except:
+                pass
+            self._board.close()
+
+    return MicroPythonKernel

jupyter_micropython_kernel/pyboard.py

@@ -0,0 +1,335 @@
+# MicroPython board communication class from MicroPython source:
+#   https://github.com/micropython/micropython/blob/master/tools/pyboard.py
+#!/usr/bin/env python
+
+"""
+pyboard interface
+
+This module provides the Pyboard class, used to communicate with and
+control the pyboard over a serial USB connection.
+
+Example usage:
+
+    import pyboard
+    pyb = pyboard.Pyboard('/dev/ttyACM0')
+
+Or:
+
+    pyb = pyboard.Pyboard('192.168.1.1')
+
+Then:
+
+    pyb.enter_raw_repl()
+    pyb.exec('pyb.LED(1).on()')
+    pyb.exit_raw_repl()
+
+Note: if using Python2 then pyb.exec must be written as pyb.exec_.
+To run a script from the local machine on the board and print out the results:
+
+    import pyboard
+    pyboard.execfile('test.py', device='/dev/ttyACM0')
+
+This script can also be run directly.  To execute a local script, use:
+
+    ./pyboard.py test.py
+
+Or:
+
+    python pyboard.py test.py
+
+"""
+
+import sys
+import time
+
+try:
+    stdout = sys.stdout.buffer
+except AttributeError:
+    # Python2 doesn't have buffer attr
+    stdout = sys.stdout
+
+def stdout_write_bytes(b):
+    b = b.replace(b"\x04", b"")
+    stdout.write(b)
+    stdout.flush()
+
+class PyboardError(BaseException):
+    pass
+
+class TelnetToSerial:
+    def __init__(self, ip, user, password, read_timeout=None):
+        import telnetlib
+        self.tn = telnetlib.Telnet(ip, timeout=15)
+        self.read_timeout = read_timeout
+        if b'Login as:' in self.tn.read_until(b'Login as:', timeout=read_timeout):
+            self.tn.write(bytes(user, 'ascii') + b"\r\n")
+
+            if b'Password:' in self.tn.read_until(b'Password:', timeout=read_timeout):
+                # needed because of internal implementation details of the telnet server
+                time.sleep(0.2)
+                self.tn.write(bytes(password, 'ascii') + b"\r\n")
+
+                if b'for more information.' in self.tn.read_until(b'Type "help()" for more information.', timeout=read_timeout):
+                    # login succesful
+                    from collections import deque
+                    self.fifo = deque()
+                    return
+
+        raise PyboardError('Failed to establish a telnet connection with the board')
+
+    def __del__(self):
+        self.close()
+
+    def close(self):
+        try:
+            self.tn.close()
+        except:
+            # the telnet object might not exist yet, so ignore this one
+            pass
+
+    def read(self, size=1):
+        while len(self.fifo) < size:
+            timeout_count = 0
+            data = self.tn.read_eager()
+            if len(data):
+                self.fifo.extend(data)
+                timeout_count = 0
+            else:
+                time.sleep(0.25)
+                if self.read_timeout is not None and timeout_count > 4 * self.read_timeout:
+                    break
+                timeout_count += 1
+
+        data = b''
+        while len(data) < size and len(self.fifo) > 0:
+            data += bytes([self.fifo.popleft()])
+        return data
+
+    def write(self, data):
+        self.tn.write(data)
+        return len(data)
+
+    def inWaiting(self):
+        n_waiting = len(self.fifo)
+        if not n_waiting:
+            data = self.tn.read_eager()
+            self.fifo.extend(data)
+            return len(data)
+        else:
+            return n_waiting
+
+class Pyboard:
+    def __init__(self, device, baudrate=115200, user='micro', password='python', wait=0):
+        if device and device[0].isdigit() and device[-1].isdigit() and device.count('.') == 3:
+            # device looks like an IP address
+            self.serial = TelnetToSerial(device, user, password, read_timeout=10)
+        else:
+            import serial
+            delayed = False
+            for attempt in range(wait + 1):
+                try:
+                    self.serial = serial.Serial(device, baudrate=baudrate, interCharTimeout=1)
+                    break
+                except (OSError, IOError): # Py2 and Py3 have different errors
+                    if wait == 0:
+                        continue
+                    if attempt == 0:
+                        sys.stdout.write('Waiting {} seconds for pyboard '.format(wait))
+                        delayed = True
+                time.sleep(1)
+                sys.stdout.write('.')
+                sys.stdout.flush()
+            else:
+                if delayed:
+                    print('')
+                raise PyboardError('failed to access ' + device)
+            if delayed:
+                print('')
+
+    def close(self):
+        self.serial.close()
+
+    def read_until(self, min_num_bytes, ending, timeout=10, data_consumer=None):
+        data = self.serial.read(min_num_bytes)
+        if data_consumer:
+            data_consumer(data)
+        timeout_count = 0
+        while True:
+            if data.endswith(ending):
+                break
+            elif self.serial.inWaiting() > 0:
+                new_data = self.serial.read(1)
+                data = data + new_data
+                if data_consumer:
+                    data_consumer(new_data)
+                timeout_count = 0
+            else:
+                timeout_count += 1
+                if timeout is not None and timeout_count >= 100 * timeout:
+                    break
+                time.sleep(0.01)
+        return data
+
+    def enter_raw_repl(self):
+        self.serial.write(b'\r\x03\x03') # ctrl-C twice: interrupt any running program
+
+        # flush input (without relying on serial.flushInput())
+        n = self.serial.inWaiting()
+        while n > 0:
+            self.serial.read(n)
+            n = self.serial.inWaiting()
+
+        self.serial.write(b'\r\x01') # ctrl-A: enter raw REPL
+        data = self.read_until(1, b'raw REPL; CTRL-B to exit\r\n>')
+        if not data.endswith(b'raw REPL; CTRL-B to exit\r\n>'):
+            print(data)
+            raise PyboardError('could not enter raw repl')
+
+        self.serial.write(b'\x04') # ctrl-D: soft reset
+        data = self.read_until(1, b'soft reboot\r\n')
+        if not data.endswith(b'soft reboot\r\n'):
+            print(data)
+            raise PyboardError('could not enter raw repl')
+        # By splitting this into 2 reads, it allows boot.py to print stuff,
+        # which will show up after the soft reboot and before the raw REPL.
+        # Modification from original pyboard.py below:
+        #   Add a small delay and send Ctrl-C twice after soft reboot to ensure
+        #   any main program loop in main.py is interrupted.
+        time.sleep(0.5)
+        self.serial.write(b'\x03\x03')
+        # End modification above.
+        data = self.read_until(1, b'raw REPL; CTRL-B to exit\r\n')
+        if not data.endswith(b'raw REPL; CTRL-B to exit\r\n'):
+            print(data)
+            raise PyboardError('could not enter raw repl')
+
+    def exit_raw_repl(self):
+        self.serial.write(b'\r\x02') # ctrl-B: enter friendly REPL
+
+    def follow(self, timeout, data_consumer=None):
+        # wait for normal output
+        data = self.read_until(1, b'\x04', timeout=timeout, data_consumer=data_consumer)
+        if not data.endswith(b'\x04'):
+            raise PyboardError('timeout waiting for first EOF reception')
+        data = data[:-1]
+
+        # wait for error output
+        data_err = self.read_until(1, b'\x04', timeout=timeout)
+        if not data_err.endswith(b'\x04'):
+            raise PyboardError('timeout waiting for second EOF reception')
+        data_err = data_err[:-1]
+
+        # return normal and error output
+        return data, data_err
+
+    def exec_raw_no_follow(self, command):
+        if isinstance(command, bytes):
+            command_bytes = command
+        else:
+            command_bytes = bytes(command, encoding='utf8')
+
+        # check we have a prompt
+        data = self.read_until(1, b'>')
+        if not data.endswith(b'>'):
+            raise PyboardError('could not enter raw repl')
+
+        # write command
+        for i in range(0, len(command_bytes), 256):
+            self.serial.write(command_bytes[i:min(i + 256, len(command_bytes))])
+            time.sleep(0.01)
+        self.serial.write(b'\x04')
+
+        # check if we could exec command
+        data = self.serial.read(2)
+        if data != b'OK':
+            raise PyboardError('could not exec command')
+
+    def exec_raw(self, command, timeout=10, data_consumer=None):
+        self.exec_raw_no_follow(command);
+        return self.follow(timeout, data_consumer)
+
+    def eval(self, expression):
+        ret = self.exec_('print({})'.format(expression))
+        ret = ret.strip()
+        return ret
+
+    def exec_(self, command):
+        ret, ret_err = self.exec_raw(command)
+        if ret_err:
+            raise PyboardError('exception', ret, ret_err)
+        return ret
+
+    def execfile(self, filename):
+        with open(filename, 'rb') as f:
+            pyfile = f.read()
+        return self.exec_(pyfile)
+
+    def get_time(self):
+        t = str(self.eval('pyb.RTC().datetime()'), encoding='utf8')[1:-1].split(', ')
+        return int(t[4]) * 3600 + int(t[5]) * 60 + int(t[6])
+
+# in Python2 exec is a keyword so one must use "exec_"
+# but for Python3 we want to provide the nicer version "exec"
+setattr(Pyboard, "exec", Pyboard.exec_)
+
+def execfile(filename, device='/dev/ttyACM0', baudrate=115200, user='micro', password='python'):
+    pyb = Pyboard(device, baudrate, user, password)
+    pyb.enter_raw_repl()
+    output = pyb.execfile(filename)
+    stdout_write_bytes(output)
+    pyb.exit_raw_repl()
+    pyb.close()
+
+def main():
+    import argparse
+    cmd_parser = argparse.ArgumentParser(description='Run scripts on the pyboard.')
+    cmd_parser.add_argument('--device', default='/dev/ttyACM0', help='the serial device or the IP address of the pyboard')
+    cmd_parser.add_argument('-b', '--baudrate', default=115200, help='the baud rate of the serial device')
+    cmd_parser.add_argument('-u', '--user', default='micro', help='the telnet login username')
+    cmd_parser.add_argument('-p', '--password', default='python', help='the telnet login password')
+    cmd_parser.add_argument('-c', '--command', help='program passed in as string')
+    cmd_parser.add_argument('-w', '--wait', default=0, type=int, help='seconds to wait for USB connected board to become available')
+    cmd_parser.add_argument('--follow', action='store_true', help='follow the output after running the scripts [default if no scripts given]')
+    cmd_parser.add_argument('files', nargs='*', help='input files')
+    args = cmd_parser.parse_args()
+
+    def execbuffer(buf):
+        try:
+            pyb = Pyboard(args.device, args.baudrate, args.user, args.password, args.wait)
+            pyb.enter_raw_repl()
+            ret, ret_err = pyb.exec_raw(buf, timeout=None, data_consumer=stdout_write_bytes)
+            pyb.exit_raw_repl()
+            pyb.close()
+        except PyboardError as er:
+            print(er)
+            sys.exit(1)
+        except KeyboardInterrupt:
+            sys.exit(1)
+        if ret_err:
+            stdout_write_bytes(ret_err)
+            sys.exit(1)
+
+    if args.command is not None:
+        execbuffer(args.command.encode('utf-8'))
+
+    for filename in args.files:
+        with open(filename, 'rb') as f:
+            pyfile = f.read()
+            execbuffer(pyfile)
+
+    if args.follow or (args.command is None and len(args.files) == 0):
+        try:
+            pyb = Pyboard(args.device, args.baudrate, args.user, args.password, args.wait)
+            ret, ret_err = pyb.follow(timeout=None, data_consumer=stdout_write_bytes)
+            pyb.close()
+        except PyboardError as er:
+            print(er)
+            sys.exit(1)
+        except KeyboardInterrupt:
+            sys.exit(1)
+        if ret_err:
+            stdout_write_bytes(ret_err)
+            sys.exit(1)
+
+if __name__ == "__main__":
+    main()

kernel.json

@@ -0,0 +1,8 @@
+{
+ "argv": ["python3",
+          "-m", "jupyter_micropython_kernel",
+          "/dev/tty.SLAB_USBtoUART", "115200",
+          "-f", "{connection_file}"],
+ "display_name": "MicroPython - /dev/tty.SLAB_USBtoUART",
+ "language": "micropython"
+}

setup.py

@@ -0,0 +1,11 @@
+from setuptools import setup
+
+setup(name='jupyter_micropython_kernel',
+      version='0.1.0',
+      description='External MicroPython kernel for Jupyter notebooks.',
+      author='Tony DiCola',
+      author_email='tdicola@adafruit.com',
+      url='https://github.com/adafruit/jupyter_micropython_kernel',
+      packages=['jupyter_micropython_kernel'],
+      install_requires=['pyserial']
+     )