From 41af049604e3cdc9104988b29a5e2081b7c19e0d Mon Sep 17 00:00:00 2001 From: Tony DiCola Date: Fri, 14 Apr 2017 20:38:53 -0700 Subject: [PATCH] Initial commit. --- .gitignore | 1 + README.md | 64 ++++- jupyter_micropython_kernel/__init__.py | 0 jupyter_micropython_kernel/__main__.py | 22 ++ jupyter_micropython_kernel/kernel.py | 78 ++++++ jupyter_micropython_kernel/pyboard.py | 335 +++++++++++++++++++++++++ kernel.json | 8 + setup.py | 11 + 8 files changed, 517 insertions(+), 2 deletions(-) create mode 100644 jupyter_micropython_kernel/__init__.py create mode 100644 jupyter_micropython_kernel/__main__.py create mode 100644 jupyter_micropython_kernel/kernel.py create mode 100644 jupyter_micropython_kernel/pyboard.py create mode 100644 kernel.json create mode 100644 setup.py diff --git a/.gitignore b/.gitignore index 72364f9..3180bc0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +*.ipynb # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/README.md b/README.md index 83fb00a..e4d310a 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,62 @@ -# jupyter_micropython_kernel -Jupyter kernel to interact with a MicroPython or CircuitPython board over its serial REPL. +# Jupyter MicroPython Kernel + +Jupyter kernel to interact with a MicroPython or CircuitPython board over its serial REPL. Note this is _highly_ experimental and still alpha/beta quality. Try it out but don't be surprised if it behaves in odd or unexpected ways! + +## Installation + +First install Jupyter: http://jupyter.org/install.html + +Then clone this repository and install the setup.py (assuming python 3.0, be +sure to use the same version of python as Jupyter is installed with): + + python3 setup.py install + +On Mac OSX and some Linux flavors you might need to run as root with sudo flavor +the above command. Make sure the installation completes successfully and that +you do not see any error messages. + +Finally create a Jupyter kernel specification for the serial port and baud rate +of your MicroPython board. Unfortunately there is no UI or ability to pick the +serial port/baud from the notebook so you'll have to bake this in to a kernel +configuration. + +From the Jupyter kernel docs find your user specific Jupyter kernel spec location: http://jupyter-client.readthedocs.io/en/latest/kernels.html#kernel-specs You want the **user** location: + +* Windows: %APPDATA%\jupyter\kernels (note if you aren't sure where this is located see: http://www.pcworld.com/article/2690709/windows/whats-in-the-hidden-windows-appdata-folder-and-how-to-find-it-if-you-need-it.html) +* macOS: ~/Library/Jupyter/kernels +* Linux: ~/.local/share/jupyter/kernels + +Create the above kernels folder if it doesn't already exist. Then inside the +kernels folder create a new folder called 'micropython' and copy the included +kernel.json file inside it. + +Open the copied kernel.json file and edit it so the 4th line: + + "/dev/tty.SLAB_USBtoUART", "115200", + +Is the serial name and baud rate of your MicroPython board. For example if using COM4 and 115200 baud you would change it to: + + "COM4", "115200", + +Also change the display name of the kernel on line 6: + + "display_name": "MicroPython - /dev/tty.SLAB_USBtoUART", + +Set a value that describes your board, like: + + "display_name": "MicroPython - COM4", + +This is the name you will see in Jupyter's notebook UI when picking the kernel +to start. You don't need to change any other config in the kernel.json. Be +very careful to make sure all the commands, double quotes, etc. are present +(this needs to be a valid JSON formatted file). + +At this point you should have the following file: /micropython/kernel.json + +Now run Jupyter notebooks: + + jupyter notebook + +In the notebook click the New button in the upper right, you should see your +MicroPython kernel display name listed. Click it to create a notebook using +that board connection (make sure the board is connected first!). diff --git a/jupyter_micropython_kernel/__init__.py b/jupyter_micropython_kernel/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/jupyter_micropython_kernel/__main__.py b/jupyter_micropython_kernel/__main__.py new file mode 100644 index 0000000..12d1a68 --- /dev/null +++ b/jupyter_micropython_kernel/__main__.py @@ -0,0 +1,22 @@ +import logging +import sys + +from ipykernel.kernelapp import IPKernelApp + +from jupyter_micropython_kernel.kernel import make_micropython_kernel + + +logging.basicConfig(level=logging.DEBUG) + +# Parse out required port name and baud rate parameters. Remove them from argv +# because the IPKernelApp will go on to parse the arguments and get confused +# if it finds extra args like them. Not super elegant but I see no other way +# to pass custom arguments/parameters to kernels. +if len(sys.argv) < 3: + raise RuntimeError('Expected at least PORT and BAUD parameters!') +port = sys.argv[1] +baud = sys.argv[2] +del sys.argv[1:3] + +# Create and launch the kernel. +IPKernelApp.launch_instance(kernel_class=make_micropython_kernel(port, baud)) diff --git a/jupyter_micropython_kernel/kernel.py b/jupyter_micropython_kernel/kernel.py new file mode 100644 index 0000000..459cb7f --- /dev/null +++ b/jupyter_micropython_kernel/kernel.py @@ -0,0 +1,78 @@ +import logging + +from ipykernel.kernelbase import Kernel +import pkg_resources + +from jupyter_micropython_kernel.pyboard import Pyboard + + +# Create global logger for debug messages. +logger = logging.getLogger(__name__) +# Get version from setuptools. This is used to tell Jupyter the version of +# this kernel. +version = pkg_resources.require('jupyter_micropython_kernel')[0].version + + +def make_micropython_kernel(port, baud): + # Create a MicroPython kernel class and return it. This is done so instance + # specific config like port and baud rate can be set. Unfortunately the + # IPython kernel wrapper design doesn't appear to allow for + # instance-specific configuration (i.e. you don't create the instance + # and call its constructor to control how it's built). As a workaround + # we'll just build a separate kernel class with a class-specific port and + # baud rate baked in. + class MicroPythonKernel(Kernel): + implementation = 'micropython' + implementation_version = version + language = 'micropython' + language_version = version + language_info = { + 'name': 'python', + 'mimetype': 'text/x-python', + 'file_extension': '.py', + } + banner = 'MicroPython Kernel - port: {} - baud: {}'.format(port, baud) + + def __init__(self, **kwargs): + super().__init__(**kwargs) + # Open MicroPython board and enter raw REPL which resets the board + # and makes it ready to accept commands. + logger.debug('Opening MicroPython board connection on port: {} baud: {}'.format(port, baud)) + self._board = Pyboard(port, baudrate=baud) + self._board.enter_raw_repl() + + def do_execute(self, code, silent, store_history=True, + user_expressions=None, allow_stdin=False): + # Run the specified code on the connected MicroPython board. + result, error = self._board.exec_raw(code) + logger.debug('Result: {} Error: {}'.format(result, error)) + # If there was an error send it back, otherwise send the result. + # Make sure to convert this to a JSON serializable string from the + # raw bytes (assumes UTF-8 encoding). This doesn't really feel + # like the right way to send back errors but the docs are really + # hard to figure out what's expected (do you send a stream_content + # with name stderr? is there more to return?). + failed = error is not None and len(error) > 0 + response = result.decode('utf-8') if not failed else error.decode('utf-8') + # Send the result when not in silent mode. + if not silent: + stream_content = {'name': 'stdout', 'text': response } + self.send_response(self.iopub_socket, 'stream', stream_content) + return {'status': 'ok' if not failed else 'error', + # The base class increments the execution count + 'execution_count': self.execution_count, + 'payload': [], + 'user_expressions': {}, + } + + def do_shutdown(self, restart): + # Be nice and try to exit the raw REPL, but ignore any failure + # in case the connection is already dead. + logger.debug('Shutting down MicroPython board connection.') + try: + self._board.exit_raw_repl() + except: + pass + self._board.close() + + return MicroPythonKernel diff --git a/jupyter_micropython_kernel/pyboard.py b/jupyter_micropython_kernel/pyboard.py new file mode 100644 index 0000000..4b27b69 --- /dev/null +++ b/jupyter_micropython_kernel/pyboard.py @@ -0,0 +1,335 @@ +# MicroPython board communication class from MicroPython source: +# https://github.com/micropython/micropython/blob/master/tools/pyboard.py +#!/usr/bin/env python + +""" +pyboard interface + +This module provides the Pyboard class, used to communicate with and +control the pyboard over a serial USB connection. + +Example usage: + + import pyboard + pyb = pyboard.Pyboard('/dev/ttyACM0') + +Or: + + pyb = pyboard.Pyboard('192.168.1.1') + +Then: + + pyb.enter_raw_repl() + pyb.exec('pyb.LED(1).on()') + pyb.exit_raw_repl() + +Note: if using Python2 then pyb.exec must be written as pyb.exec_. +To run a script from the local machine on the board and print out the results: + + import pyboard + pyboard.execfile('test.py', device='/dev/ttyACM0') + +This script can also be run directly. To execute a local script, use: + + ./pyboard.py test.py + +Or: + + python pyboard.py test.py + +""" + +import sys +import time + +try: + stdout = sys.stdout.buffer +except AttributeError: + # Python2 doesn't have buffer attr + stdout = sys.stdout + +def stdout_write_bytes(b): + b = b.replace(b"\x04", b"") + stdout.write(b) + stdout.flush() + +class PyboardError(BaseException): + pass + +class TelnetToSerial: + def __init__(self, ip, user, password, read_timeout=None): + import telnetlib + self.tn = telnetlib.Telnet(ip, timeout=15) + self.read_timeout = read_timeout + if b'Login as:' in self.tn.read_until(b'Login as:', timeout=read_timeout): + self.tn.write(bytes(user, 'ascii') + b"\r\n") + + if b'Password:' in self.tn.read_until(b'Password:', timeout=read_timeout): + # needed because of internal implementation details of the telnet server + time.sleep(0.2) + self.tn.write(bytes(password, 'ascii') + b"\r\n") + + if b'for more information.' in self.tn.read_until(b'Type "help()" for more information.', timeout=read_timeout): + # login succesful + from collections import deque + self.fifo = deque() + return + + raise PyboardError('Failed to establish a telnet connection with the board') + + def __del__(self): + self.close() + + def close(self): + try: + self.tn.close() + except: + # the telnet object might not exist yet, so ignore this one + pass + + def read(self, size=1): + while len(self.fifo) < size: + timeout_count = 0 + data = self.tn.read_eager() + if len(data): + self.fifo.extend(data) + timeout_count = 0 + else: + time.sleep(0.25) + if self.read_timeout is not None and timeout_count > 4 * self.read_timeout: + break + timeout_count += 1 + + data = b'' + while len(data) < size and len(self.fifo) > 0: + data += bytes([self.fifo.popleft()]) + return data + + def write(self, data): + self.tn.write(data) + return len(data) + + def inWaiting(self): + n_waiting = len(self.fifo) + if not n_waiting: + data = self.tn.read_eager() + self.fifo.extend(data) + return len(data) + else: + return n_waiting + +class Pyboard: + def __init__(self, device, baudrate=115200, user='micro', password='python', wait=0): + if device and device[0].isdigit() and device[-1].isdigit() and device.count('.') == 3: + # device looks like an IP address + self.serial = TelnetToSerial(device, user, password, read_timeout=10) + else: + import serial + delayed = False + for attempt in range(wait + 1): + try: + self.serial = serial.Serial(device, baudrate=baudrate, interCharTimeout=1) + break + except (OSError, IOError): # Py2 and Py3 have different errors + if wait == 0: + continue + if attempt == 0: + sys.stdout.write('Waiting {} seconds for pyboard '.format(wait)) + delayed = True + time.sleep(1) + sys.stdout.write('.') + sys.stdout.flush() + else: + if delayed: + print('') + raise PyboardError('failed to access ' + device) + if delayed: + print('') + + def close(self): + self.serial.close() + + def read_until(self, min_num_bytes, ending, timeout=10, data_consumer=None): + data = self.serial.read(min_num_bytes) + if data_consumer: + data_consumer(data) + timeout_count = 0 + while True: + if data.endswith(ending): + break + elif self.serial.inWaiting() > 0: + new_data = self.serial.read(1) + data = data + new_data + if data_consumer: + data_consumer(new_data) + timeout_count = 0 + else: + timeout_count += 1 + if timeout is not None and timeout_count >= 100 * timeout: + break + time.sleep(0.01) + return data + + def enter_raw_repl(self): + self.serial.write(b'\r\x03\x03') # ctrl-C twice: interrupt any running program + + # flush input (without relying on serial.flushInput()) + n = self.serial.inWaiting() + while n > 0: + self.serial.read(n) + n = self.serial.inWaiting() + + self.serial.write(b'\r\x01') # ctrl-A: enter raw REPL + data = self.read_until(1, b'raw REPL; CTRL-B to exit\r\n>') + if not data.endswith(b'raw REPL; CTRL-B to exit\r\n>'): + print(data) + raise PyboardError('could not enter raw repl') + + self.serial.write(b'\x04') # ctrl-D: soft reset + data = self.read_until(1, b'soft reboot\r\n') + if not data.endswith(b'soft reboot\r\n'): + print(data) + raise PyboardError('could not enter raw repl') + # By splitting this into 2 reads, it allows boot.py to print stuff, + # which will show up after the soft reboot and before the raw REPL. + # Modification from original pyboard.py below: + # Add a small delay and send Ctrl-C twice after soft reboot to ensure + # any main program loop in main.py is interrupted. + time.sleep(0.5) + self.serial.write(b'\x03\x03') + # End modification above. + data = self.read_until(1, b'raw REPL; CTRL-B to exit\r\n') + if not data.endswith(b'raw REPL; CTRL-B to exit\r\n'): + print(data) + raise PyboardError('could not enter raw repl') + + def exit_raw_repl(self): + self.serial.write(b'\r\x02') # ctrl-B: enter friendly REPL + + def follow(self, timeout, data_consumer=None): + # wait for normal output + data = self.read_until(1, b'\x04', timeout=timeout, data_consumer=data_consumer) + if not data.endswith(b'\x04'): + raise PyboardError('timeout waiting for first EOF reception') + data = data[:-1] + + # wait for error output + data_err = self.read_until(1, b'\x04', timeout=timeout) + if not data_err.endswith(b'\x04'): + raise PyboardError('timeout waiting for second EOF reception') + data_err = data_err[:-1] + + # return normal and error output + return data, data_err + + def exec_raw_no_follow(self, command): + if isinstance(command, bytes): + command_bytes = command + else: + command_bytes = bytes(command, encoding='utf8') + + # check we have a prompt + data = self.read_until(1, b'>') + if not data.endswith(b'>'): + raise PyboardError('could not enter raw repl') + + # write command + for i in range(0, len(command_bytes), 256): + self.serial.write(command_bytes[i:min(i + 256, len(command_bytes))]) + time.sleep(0.01) + self.serial.write(b'\x04') + + # check if we could exec command + data = self.serial.read(2) + if data != b'OK': + raise PyboardError('could not exec command') + + def exec_raw(self, command, timeout=10, data_consumer=None): + self.exec_raw_no_follow(command); + return self.follow(timeout, data_consumer) + + def eval(self, expression): + ret = self.exec_('print({})'.format(expression)) + ret = ret.strip() + return ret + + def exec_(self, command): + ret, ret_err = self.exec_raw(command) + if ret_err: + raise PyboardError('exception', ret, ret_err) + return ret + + def execfile(self, filename): + with open(filename, 'rb') as f: + pyfile = f.read() + return self.exec_(pyfile) + + def get_time(self): + t = str(self.eval('pyb.RTC().datetime()'), encoding='utf8')[1:-1].split(', ') + return int(t[4]) * 3600 + int(t[5]) * 60 + int(t[6]) + +# in Python2 exec is a keyword so one must use "exec_" +# but for Python3 we want to provide the nicer version "exec" +setattr(Pyboard, "exec", Pyboard.exec_) + +def execfile(filename, device='/dev/ttyACM0', baudrate=115200, user='micro', password='python'): + pyb = Pyboard(device, baudrate, user, password) + pyb.enter_raw_repl() + output = pyb.execfile(filename) + stdout_write_bytes(output) + pyb.exit_raw_repl() + pyb.close() + +def main(): + import argparse + cmd_parser = argparse.ArgumentParser(description='Run scripts on the pyboard.') + cmd_parser.add_argument('--device', default='/dev/ttyACM0', help='the serial device or the IP address of the pyboard') + cmd_parser.add_argument('-b', '--baudrate', default=115200, help='the baud rate of the serial device') + cmd_parser.add_argument('-u', '--user', default='micro', help='the telnet login username') + cmd_parser.add_argument('-p', '--password', default='python', help='the telnet login password') + cmd_parser.add_argument('-c', '--command', help='program passed in as string') + cmd_parser.add_argument('-w', '--wait', default=0, type=int, help='seconds to wait for USB connected board to become available') + cmd_parser.add_argument('--follow', action='store_true', help='follow the output after running the scripts [default if no scripts given]') + cmd_parser.add_argument('files', nargs='*', help='input files') + args = cmd_parser.parse_args() + + def execbuffer(buf): + try: + pyb = Pyboard(args.device, args.baudrate, args.user, args.password, args.wait) + pyb.enter_raw_repl() + ret, ret_err = pyb.exec_raw(buf, timeout=None, data_consumer=stdout_write_bytes) + pyb.exit_raw_repl() + pyb.close() + except PyboardError as er: + print(er) + sys.exit(1) + except KeyboardInterrupt: + sys.exit(1) + if ret_err: + stdout_write_bytes(ret_err) + sys.exit(1) + + if args.command is not None: + execbuffer(args.command.encode('utf-8')) + + for filename in args.files: + with open(filename, 'rb') as f: + pyfile = f.read() + execbuffer(pyfile) + + if args.follow or (args.command is None and len(args.files) == 0): + try: + pyb = Pyboard(args.device, args.baudrate, args.user, args.password, args.wait) + ret, ret_err = pyb.follow(timeout=None, data_consumer=stdout_write_bytes) + pyb.close() + except PyboardError as er: + print(er) + sys.exit(1) + except KeyboardInterrupt: + sys.exit(1) + if ret_err: + stdout_write_bytes(ret_err) + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/kernel.json b/kernel.json new file mode 100644 index 0000000..97b16f2 --- /dev/null +++ b/kernel.json @@ -0,0 +1,8 @@ +{ + "argv": ["python3", + "-m", "jupyter_micropython_kernel", + "/dev/tty.SLAB_USBtoUART", "115200", + "-f", "{connection_file}"], + "display_name": "MicroPython - /dev/tty.SLAB_USBtoUART", + "language": "micropython" +} diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..c875f21 --- /dev/null +++ b/setup.py @@ -0,0 +1,11 @@ +from setuptools import setup + +setup(name='jupyter_micropython_kernel', + version='0.1.0', + description='External MicroPython kernel for Jupyter notebooks.', + author='Tony DiCola', + author_email='tdicola@adafruit.com', + url='https://github.com/adafruit/jupyter_micropython_kernel', + packages=['jupyter_micropython_kernel'], + install_requires=['pyserial'] + )