Docker add utilities

.github/workflows/test.yml

@@ -10,7 +10,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        debian: [buster, bullseye]
+        debian: [bullseye]
     steps:
       - name: Checkout code
         uses: actions/checkout@v2.3.4

Dockerfile

@@ -26,6 +26,25 @@ RUN apt-get update && \
     apt-get install -y pt-web-vnc && \
     apt-get clean
 
+# Install pitop SDK prerequisites
+RUN apt-get update && \
+    apt-get install -y pkg-config libsystemd0 libsystemd-dev && \
+    pip3 install cmake && \
+    apt-get clean
+
+# Install pitop SDK
+# using pip for onnxruntime as there is only armhf debian build
+RUN pip3 install pitop==0.30.0.post1
+
+# Install useful extras from pt-os
+RUN apt-get update && \
+    # not installable apt-get install -y pt-os-ui-mods && \
+    apt-get install -y chromium && \
+    apt-get install -y vim && \
+    apt-get install -y python3-matplotlib && \
+    # no audio DEBIAN_FRONTEND='noninteractive' apt-get install -y sonic-pi python3-sonic && \
+    apt-get clean
+
 WORKDIR /further-link
 
 COPY pyproject.toml setup.py setup.cfg MANIFEST.in ./
@@ -36,9 +55,11 @@ COPY further_link further_link
 RUN pip3 install .
 
 ENV FURTHER_LINK_NOSSL=true
+ENV FURTHER_LINK_MAX_PROCESSES=4
+ENV PITOP_VIRTUAL_HARDWARE=1
 ENV PYTHONUNBUFFERED=1
 ENV PYTHON_PACKAGE_VERSION=$PYTHON_PACKAGE_VERSION
 
 EXPOSE 8028
-EXPOSE 60100-60999
+EXPOSE 61100-61103
 CMD [ "further-link" ]

docs/README.md

@@ -46,7 +46,7 @@ directory with the names `cert.pem` and `key.pem`.
 ### Working directory
 The default working directory where files are uploaded and executed from is
 `~/further`. This can be overridden by setting env var FURTHER_LINK_WORK_DIR.
-For more information related to the `run-py` API see the options of the
+For more information related to the `run` API see the options of the
 `upload` and `start` messages in the documentation below.
 
 ## API
@@ -57,122 +57,6 @@ For more information related to the `run-py` API see the options of the
   ([example](../tests/test_data/upload_data.py)). Responds after creating files is
   complete with a simple 200 OK.
 
-### Websocket Endpoint /run-py
-Each websocket client connected on `/run-py` can manage a single python process
-at a time.
-
-#### Example usage
-- Connect websocket on `/run-py` (using [websocat](https://github.com/vi/websocat)):
-```
-websocat ws://localhost:8028/run-py
-```
-- Send `start` command with `sourceScript`:
-```
-{ "type": "start", "data": { "sourceScript": "print('hi')" } }
-```
-- Receive `stdout` response:
-```
-{ "type": "stdout", "data": { "output": "hi\n" } }
-```
-- Receive `stopped` response:
-```
-{ "type": "stopped", "data": { "exitCode": 0 } }
-```
-
-#### Spec
-##### Options
-This connection has some options which can be selected with query
-parameters:
-
-```
-/run-py?user=root
-```
-The `user` parameter is used to select the Linux user which the code is
-executed as. By default the `pi` user is selected if it exists, otherwise
-the user executing the server is used.
-
-```
-/run-py?pty=1
-```
-The pty parameter, if set to 1 or true, will create a pseudoterminal interface
-for the python process IO streams, in order to provide terminal behaviours such
-as 'cooked mode'. This is useful to produce identical behaviour of programs to
-that on the command line and to easily interface with terminal emulators such
-as [xterm.js](https://github.com/xtermjs/xterm.js/).
-
-##### Message Types
-Websocket messages sent between client and server must be in JSON with two top
-level properties: required string `type` and optional object `data`.
-
-Message types accepted by the server are:
-```
-{
- "type":"[ping|start|stop|stdin|upload|keyevent]",
- "data": {...}
-}
-```
-
-Message types sent from the server are:
-```
-{
- "type":"[pong|error|started|stopped|stdout|stderr|uploaded|keylisten]",
- "data": {...}
-}
-```
-
-Message and response details:
-- `ping` command from the client will be met with a `pong` response from
-    the server. This can be used to keep the socket active to prevent automatic
-    closures.
-- `pong` response is sent by the server immediately after a `ping` is received
-<br>
-
-- `error` response is sent for bad commands or server errors e.g. `data: { message: "something went wrong and it's not your python code" }`
-<br>
-
-- `start` command will start a new python process. The code to run can be
-    specified in data as either a `souceScript` or `sourcePath`. For
-    `sourceScript` an additional `directoryName` can be passed to specify an
-    (uploaded) directory to run the script in, within the work dir, otherwise
-    `/tmp` is used. If `sourcePath` is not absolute, it is assumed to be
-    relative to the work dir.
-    e.g. `data: {sourceScript:"print('hi')", directoryName: "myproject"}`
-    or `data: {sourcePath: "myproject/run.py"}`
-- `started` response is sent after a successful process `start`, has no data.
-<br>
-
-- `stdin` command is used to send data to process stdin e.g. `data: { input: "this can be read by python\n" }`.
-- `stdout` response is sent when process prints to stdout. e.g. `data: { output: "this was printed by python" }`
-- `stderr` response is sent when process prints to stderr e.g. `data: { output: "Traceback bleh bleh" }`
-<br>
-
-- `stop` command is used to stop a running process early, has no data.
-- `stopped` response is sent when a process finished and has the exit code in e.g. `data: { exitCode: 0 }`
-<br>
-
-- `upload` command is used to create a directory of files in the work dir.
-    Files are provided in the data as `text` type, with the text content, or
-    `url` type, with a url for the file to be downloaded from.
-    [Example](..tests/test_data/upload_data.py)
-- `uploaded` response is sent after a successful upload , has no data.
-<br>
-
-- `video` response is sent by the server with data.output containing a base64
-    encoded mjpeg frame for the client to render as a video feed.
-<br>
-
-- `keylisten` message is sent by the server to indicate it would like to
-    receive keyboard events from the client for a specific key. This would be
-    initiated by user code using the further_link.KeyboardButton python module.
-    Keys are specified as web [KeyboardEvent.key](https://developer.mozilla.org/en-US/docs/Web/API/KeyboardEvent/key)
-    strings e.g. `data: { key: "ArrowUp" }`
-- `keyevent` message is sent by the client to provide keyboard events to the
-    server so that they can be forwarded to user code using
-    further_link.KeyboardButton. The data includes a key string matching those
-    used in `keylisten` and an event string which is either "keydown" or
-    "keyup" e.g. `data: { key: "ArrowUp", event: "keydown" }`
-<br>
-
 ### Websocket Endpoint /run
 Each websocket client connected on `/run` can manage multuiple processes of
 different types, addressing them by a unique id.

further_link/__main__.py

@@ -1,19 +1,17 @@
 import logging
 import os
 import sys
-from json import dumps
 
 import aiohttp_cors
 import click
 from aiohttp import web
 
 from further_link.endpoint.apt_version import apt_version
 from further_link.endpoint.run import run as run_handler
-from further_link.endpoint.run_py import run_py
+from further_link.endpoint.status import live, status, version
 from further_link.endpoint.upload import upload
 from further_link.util import vnc
 from further_link.util.ssl_context import ssl_context
-from further_link.version import __version__
 
 logging.basicConfig(
     stream=sys.stdout,
@@ -38,27 +36,21 @@ def create_app():
         },
     )
 
-    async def status(_):
-        return web.Response(text="OK")
-
-    async def version(_):
-        return web.Response(text=dumps({"version": __version__}))
-
     status_resource = cors.add(app.router.add_resource("/status"))
     cors.add(status_resource.add_route("GET", status))
 
-    status_resource = cors.add(app.router.add_resource("/version/apt/{pkg}"))
-    cors.add(status_resource.add_route("GET", apt_version))
-
     status_resource = cors.add(app.router.add_resource("/version"))
     cors.add(status_resource.add_route("GET", version))
 
+    status_resource = cors.add(app.router.add_resource("/live"))
+    cors.add(status_resource.add_route("GET", live))
+
+    status_resource = cors.add(app.router.add_resource("/version/apt/{pkg}"))
+    cors.add(status_resource.add_route("GET", apt_version))
+
     status_resource = cors.add(app.router.add_resource("/upload"))
     cors.add(status_resource.add_route("POST", upload))
 
-    exec_resource = cors.add(app.router.add_resource("/run-py"))
-    cors.add(exec_resource.add_route("GET", run_py))
-
     exec_resource = cors.add(app.router.add_resource("/run"))
     cors.add(exec_resource.add_route("GET", run_handler))
 

further_link/endpoint/run.py

@@ -27,25 +27,30 @@ def __init__(self, socket, user=None, pty=False):
         }
 
     async def stop(self):
+        logging.debug(f"{self.id} Stopping processes: {self.process_handlers}")
         # the dictionary will be mutated so use list to make a copy
         for p in list(self.process_handlers.values()):
             try:
                 await p.stop()
             except InvalidOperation:
-                pass
+                logging.debug(f"{self.id} Invalid operation stopping process: {p}")
 
     async def send(self, type, data=None, process_id=None):
         try:
             await self.socket.send_str(create_message(type, data, process_id))
         except ConnectionResetError:
-            pass  # already disconnected
+            logging.debug(f"{self.id} Send failed - client disconnected")
 
     async def handle_message(self, message):
         try:
             m_type, m_data, m_process = parse_message(message)
 
             process_handler = self.process_handlers.get(m_process)
 
+            logging.debug(
+                f"{self.id} Handling {m_type} message for handler {process_handler}"
+            )
+
             if m_type == "ping":
                 await self.send("pong")
 
@@ -99,7 +104,7 @@ async def handle_message(self, message):
                 await process_handler.send_key_event(m_data["key"], m_data["event"])
 
             else:
-                raise BadMessage()
+                raise BadMessage(f"Bad {m_type} message for handler {process_handler}")
 
         except (BadMessage, InvalidOperation):
             logging.exception(f"{self.id} Bad Message")
@@ -117,7 +122,7 @@ async def add_handler(self, process_id, runner, path, code, novncOptions):
 
         async def on_start():
             await self.send("started", None, process_id)
-            logging.info(f"{self.id} Started {process_id}")
+            logging.info(f"{self.id} Sent started {process_id}")
 
         async def on_stop(exit_code):
             # process_id may be reused with other runners so clean up handler
@@ -141,11 +146,18 @@ async def on_display_activity(connection_details: VncConnectionDetails):
             )
 
         handler = handler_class(self.user, self.pty)
+        logging.debug(
+            f"{self.id} Created {runner} handler for {process_id}, ID {handler.id}"
+        )
+
         handler.on_start = on_start
         handler.on_stop = on_stop
         handler.on_display_activity = on_display_activity
         handler.on_output = on_output
+
+        logging.debug(f"{self.id} Starting handler {handler.id}")
         await handler.start(path, code, novncOptions=novncOptions)
+        logging.debug(f"{self.id} Started handler {handler.id}")
 
         self.process_handlers[process_id] = handler
 
@@ -155,7 +167,7 @@ async def run(request):
     user = query_params.get("user", None)
     pty = query_params.get("pty", "").lower() in ["1", "true"]
 
-    socket = web.WebSocketResponse()
+    socket = web.WebSocketResponse(autoclose=True)
     await socket.prepare(request)
 
     run_manager = RunManager(socket, user=user, pty=pty)
@@ -164,12 +176,20 @@ async def run(request):
     try:
         async for message in socket:
             logging.debug(f"{run_manager.id} Received Message {message.data}")
+            logging.debug(f"{run_manager.id} message: {message.type} {message}")
+            logging.debug(f"{run_manager.id} socket closing: {socket._closing}")
+            logging.debug(f"{run_manager.id} socket closed: {socket._closed}")
             await run_manager.handle_message(message.data)
+            logging.debug(f"{run_manager.id} Handled Message {message.data}")
 
     except asyncio.CancelledError:
-        pass
+        logging.debug(f"{run_manager.id} Run cancelled error")
+
+    except Exception as e:
+        logging.exception(f"{run_manager.id} Run error: {e}")
 
     finally:
+        logging.debug(f"{run_manager.id} Closing connection")
         await socket.close()
         logging.info(f"{run_manager.id} Closed connection")
         await run_manager.stop()