bpo-40280: Add limited Emscripten REPL (GH-32284)

Co-authored-by: Katie Bell <katie@katharos.id.au>

Makefile.pre.in

@@ -807,15 +807,22 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS)
 	else true; \
 	fi
 
-# wasm32-emscripten build
+# wasm32-emscripten browser build
 # wasm assets directory is relative to current build dir, e.g. "./usr/local".
 # --preload-file turns a relative asset path into an absolute path.
 
 $(WASM_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \
-                pybuilddir.txt $(srcdir)/Tools/wasm/wasm_assets.py
+                pybuilddir.txt $(srcdir)/Tools/wasm/wasm_assets.py \
+				python.html python.worker.js
 	$(PYTHON_FOR_BUILD) $(srcdir)/Tools/wasm/wasm_assets.py \
 	    --builddir . --prefix $(prefix)
 
+python.html: $(srcdir)/Tools/wasm/python.html python.worker.js
+	@cp $(srcdir)/Tools/wasm/python.html $@
+
+python.worker.js: $(srcdir)/Tools/wasm/python.worker.js
+	@cp $(srcdir)/Tools/wasm/python.worker.js $@
+
 ##########################################################################
 # Build static libmpdec.a
 LIBMPDEC_CFLAGS=$(PY_STDMODULE_CFLAGS) $(CCSHARED) @LIBMPDEC_CFLAGS@

Misc/NEWS.d/next/Tools-Demos/2022-04-03-11-47-45.bpo-40280.Q_IJik.rst

@@ -0,0 +1,2 @@
+Replace Emscripten's limited shell with Katie Bell's browser-ui REPL from
+python-wasm project.

Tools/wasm/README.md

@@ -55,9 +55,13 @@ emrun builddir/emscripten-browser/python.html
 or
 
 ```shell
-python3 -m http.server
+./Tools/wasm/wasm_webserver.py
 ```
 
+and open http://localhost:8000/builddir/emscripten-browser/python.html . This
+directory structure enables the *C/C++ DevTools Support (DWARF)* to load C
+and header files with debug builds.
+
 ### Cross compile to wasm32-emscripten for node
 
 ```
@@ -79,17 +83,17 @@ popd
 node --experimental-wasm-threads --experimental-wasm-bulk-memory builddir/emscripten-node/python.js
 ```
 
-## wasm32-emscripten limitations and issues
+# wasm32-emscripten limitations and issues
 
-- Heap and stack are limited.
+Emscripten before 3.1.8 has known bugs that can cause memory corruption and
-- Most stdlib modules with a dependency on external libraries are missing:
+resource leaks. 3.1.8 contains several fixes for bugs in date and time
-  ``ctypes``, ``readline``, ``sqlite3``, ``ssl``, and more.
+functions.
-- Shared extension modules are not implemented yet. All extension modules
+
-  are statically linked into the main binary.
+## Network stack
-  The experimental configure option ``--enable-wasm-dynamic-linking`` enables
+
-  dynamic extensions.
+- Python's socket module does not work with Emscripten's emulated POSIX
-- Processes are not supported. System calls like fork, popen, and subprocess
+  sockets yet. Network modules like ``asyncio``, ``urllib``, ``selectors``,
-  fail with ``ENOSYS`` or ``ENOSUP``.
+  etc. are not available.
 - Only ``AF_INET`` and ``AF_INET6`` with ``SOCK_STREAM`` (TCP) or
   ``SOCK_DGRAM`` (UDP) are available. ``AF_UNIX`` is not supported.
 - ``socketpair`` does not work.
@@ -98,8 +102,21 @@ node --experimental-wasm-threads --experimental-wasm-bulk-memory builddir/emscri
   does not resolve to a real IP address. IPv6 is not available.
 - The ``select`` module is limited. ``select.select()`` crashes the runtime
   due to lack of exectfd support.
+
+## processes, threads, signals
+
+- Processes are not supported. System calls like fork, popen, and subprocess
+  fail with ``ENOSYS`` or ``ENOSUP``.
 - Signal support is limited. ``signal.alarm``, ``itimer``, ``sigaction``
   are not available or do not work correctly. ``SIGTERM`` exits the runtime.
+- Keyboard interrupt (CTRL+C) handling is not implemented yet.
+- Browser builds cannot start new threads. Node's web workers consume
+  extra file descriptors.
+- Resource-related functions like ``os.nice`` and most functions of the
+  ``resource`` module are not available.
+
+## file system
+
 - Most user, group, and permission related function and modules are not
   supported or don't work as expected, e.g.``pwd`` module, ``grp`` module,
   ``os.setgroups``, ``os.chown``, and so on. ``lchown`` and `lchmod`` are
@@ -113,23 +130,35 @@ node --experimental-wasm-threads --experimental-wasm-bulk-memory builddir/emscri
   and are disabled.
 - Large file support crashes the runtime and is disabled.
 - ``mmap`` module is unstable. flush (``msync``) can crash the runtime.
-- Resource-related functions like ``os.nice`` and most functions of the
+
-  ``resource`` module are not available.
+## Misc
+
+- Heap memory and stack size are limited. Recursion or extensive memory
+  consumption can crash Python.
+- Most stdlib modules with a dependency on external libraries are missing,
+  e.g. ``ctypes``, ``readline``, ``sqlite3``, ``ssl``, and more.
+- Shared extension modules are not implemented yet. All extension modules
+  are statically linked into the main binary.
+  The experimental configure option ``--enable-wasm-dynamic-linking`` enables
+  dynamic extensions.
 - glibc extensions for date and time formatting are not available.
 - ``locales`` module is affected by musl libc issues,
   [bpo-46390](https://bugs.python.org/issue46390).
 - Python's object allocator ``obmalloc`` is disabled by default.
 - ``ensurepip`` is not available.
 
-### wasm32-emscripten in browsers
+## wasm32-emscripten in browsers
 
+- The interactive shell does not handle copy 'n paste and unicode support
+  well.
 - The bundled stdlib is limited. Network-related modules,
   distutils, multiprocessing, dbm, tests and similar modules
   are not shipped. All other modules are bundled as pre-compiled
   ``pyc`` files.
 - Threading is not supported.
+- In-memory file system (MEMFS) is not persistent and limited.
 
-### wasm32-emscripten in node
+## wasm32-emscripten in node
 
 Node builds use ``NODERAWFS``, ``USE_PTHREADS`` and ``PROXY_TO_PTHREAD``
 linker options.

Tools/wasm/python.html

@@ -0,0 +1,245 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <meta name="author" content="Katie Bell">
+    <meta name="description" content="Simple REPL for Python WASM">
+    <title>wasm-python terminal</title>
+    <link rel="stylesheet" href="https://unpkg.com/xterm@4.18.0/css/xterm.css" crossorigin/>
+    <style>
+        body {
+            font-family: arial;
+            max-width: 800px;
+            margin: 0 auto
+        }
+        #code {
+            width: 100%;
+            height: 180px;
+        }
+        #info {
+            padding-top: 20px;
+        }
+        .button-container {
+            display: flex;
+            justify-content: end;
+            height: 50px;
+            align-items: center;
+            gap: 10px;
+        }
+        button {
+            padding: 6px 18px;
+        }
+    </style>
+    <script src="https://unpkg.com/xterm@4.18.0/lib/xterm.js" crossorigin></script>
+    <script type="module">
+class WorkerManager {
+    constructor(workerURL, standardIO, readyCallBack) {
+        this.workerURL = workerURL
+        this.worker = null
+        this.standardIO = standardIO
+        this.readyCallBack = readyCallBack
+
+        this.initialiseWorker()
+    }
+
+    async initialiseWorker() {
+        if (!this.worker) {
+            this.worker = new Worker(this.workerURL)
+            this.worker.addEventListener('message', this.handleMessageFromWorker)
+        }
+    }
+
+    async run(options) {
+        this.worker.postMessage({
+            type: 'run',
+            args: options.args || [],
+            files: options.files || {}
+        })
+    }
+
+    handleStdinData(inputValue) {
+        if (this.stdinbuffer && this.stdinbufferInt) {
+            let startingIndex = 1
+            if (this.stdinbufferInt[0] > 0) {
+                startingIndex = this.stdinbufferInt[0]
+            }
+            const data = new TextEncoder().encode(inputValue)
+            data.forEach((value, index) => {
+                this.stdinbufferInt[startingIndex + index] = value
+            })
+
+            this.stdinbufferInt[0] = startingIndex + data.length - 1
+            Atomics.notify(this.stdinbufferInt, 0, 1)
+        }
+    }
+
+    handleMessageFromWorker = (event) => {
+        const type = event.data.type
+        if (type === 'ready') {
+            this.readyCallBack()
+        } else if (type === 'stdout') {
+            this.standardIO.stdout(event.data.stdout)
+        } else if (type === 'stderr') {
+            this.standardIO.stderr(event.data.stderr)
+        } else if (type === 'stdin') {
+            // Leave it to the terminal to decide whether to chunk it into lines
+            // or send characters depending on the use case.
+            this.stdinbuffer = event.data.buffer
+            this.stdinbufferInt = new Int32Array(this.stdinbuffer)
+            this.standardIO.stdin().then((inputValue) => {
+                this.handleStdinData(inputValue)
+            })
+        } else if (type === 'finished') {
+            this.standardIO.stderr(`Exited with status: ${event.data.returnCode}\r\n`)
+        }
+    }
+}
+
+class WasmTerminal {
+
+    constructor() {
+        this.input = ''
+        this.resolveInput = null
+        this.activeInput = false
+        this.inputStartCursor = null
+
+        this.xterm = new Terminal(
+            { scrollback: 10000, fontSize: 14, theme: { background: '#1a1c1f' }, cols: 100}
+        );
+
+        this.xterm.onKey((keyEvent) => {
+            // Fix for iOS Keyboard Jumping on space
+            if (keyEvent.key === " ") {
+                keyEvent.domEvent.preventDefault();
+            }
+        });
+
+        this.xterm.onData(this.handleTermData)
+    }
+
+    open(container) {
+        this.xterm.open(container);
+    }
+
+    handleReadComplete(lastChar) {
+        this.resolveInput(this.input + lastChar)
+        this.activeInput = false
+    }
+
+    handleTermData = (data) => {
+        if (!this.activeInput) {
+            return
+        }
+        const ord = data.charCodeAt(0);
+        let ofs;
+
+        // TODO: Handle ANSI escape sequences
+        if (ord === 0x1b) {
+        // Handle special characters
+        } else if (ord < 32 || ord === 0x7f) {
+            switch (data) {
+                case "\r": // ENTER
+                case "\x0a": // CTRL+J
+                case "\x0d": // CTRL+M
+                    this.xterm.write('\r\n');
+                    this.handleReadComplete('\n');
+                    break;
+                case "\x7F": // BACKSPACE
+                case "\x08": // CTRL+H
+                case "\x04": // CTRL+D
+                    this.handleCursorErase(true);
+                    break;
+            }
+        } else {
+            this.handleCursorInsert(data);
+        }
+    }
+
+    handleCursorInsert(data) {
+        this.input += data;
+        this.xterm.write(data)
+    }
+
+    handleCursorErase() {
+        // Don't delete past the start of input
+        if (this.xterm.buffer.active.cursorX <= this.inputStartCursor) {
+            return
+        }
+        this.input = this.input.slice(0, -1)
+        this.xterm.write('\x1B[D')
+        this.xterm.write('\x1B[P')
+    }
+
+    prompt = async () => {
+        this.activeInput = true
+        // Hack to allow stdout/stderr to finish before we figure out where input starts
+        setTimeout(() => {this.inputStartCursor = this.xterm.buffer.active.cursorX}, 1)
+        return new Promise((resolve, reject) => {
+            this.resolveInput = (value) => {
+                this.input = ''
+                resolve(value)
+            }
+        })
+    }
+
+    clear() {
+        this.xterm.clear();
+    }
+
+    print(message) {
+        const normInput = message.replace(/[\r\n]+/g, "\n").replace(/\n/g, "\r\n");
+        this.xterm.write(normInput);
+    }
+}
+
+const replButton = document.getElementById('repl')
+const clearButton = document.getElementById('clear')
+
+window.onload = () => {
+    const terminal = new WasmTerminal()
+    terminal.open(document.getElementById('terminal'))
+
+    const stdio = {
+        stdout: (s) => { terminal.print(s) },
+        stderr: (s) => { terminal.print(s) },
+        stdin: async () => {
+            return await terminal.prompt()
+        }
+    }
+
+    replButton.addEventListener('click', (e) => {
+        // Need to use "-i -" to force interactive mode.
+        // Looks like isatty always returns false in emscripten
+        pythonWorkerManager.run({args: ['-i', '-'], files: {}})
+    })
+
+    clearButton.addEventListener('click', (e) => {
+        terminal.clear()
+    })
+
+    const readyCallback = () => {
+        replButton.removeAttribute('disabled')
+        clearButton.removeAttribute('disabled')
+    }
+
+    const pythonWorkerManager = new WorkerManager('./python.worker.js', stdio, readyCallback)
+}
+    </script>
+</head>
+<body>
+    <h1>Simple REPL for Python WASM</h1>
+    <div id="terminal"></div>
+    <div class="button-container">
+      <button id="repl" disabled>Start REPL</button>
+      <button id="clear" disabled>Clear</button>
+    </div>
+    <div id="info">
+        The simple REPL provides a limited Python experience in the browser.
+        <a href="https://github.com/python/cpython/blob/main/Tools/wasm/README.md">
+        Tools/wasm/README.md</a> contains a list of known limitations and
+        issues. Networking, subprocesses, and threading are not available.
+    </div>
+</body>
+</html>

Tools/wasm/python.worker.js

@@ -0,0 +1,87 @@
+class StdinBuffer {
+    constructor() {
+        this.sab = new SharedArrayBuffer(128 * Int32Array.BYTES_PER_ELEMENT)
+        this.buffer = new Int32Array(this.sab)
+        this.readIndex = 1;
+        this.numberOfCharacters = 0;
+        this.sentNull = true
+    }
+
+    prompt() {
+        this.readIndex = 1
+        Atomics.store(this.buffer, 0, -1)
+        postMessage({
+            type: 'stdin',
+            buffer: this.sab
+        })
+        Atomics.wait(this.buffer, 0, -1)
+        this.numberOfCharacters = this.buffer[0]
+    }
+
+    stdin = () => {
+        if (this.numberOfCharacters + 1 === this.readIndex) {
+            if (!this.sentNull) {
+                // Must return null once to indicate we're done for now.
+                this.sentNull = true
+                return null
+            }
+            this.sentNull = false
+            this.prompt()
+        }
+        const char = this.buffer[this.readIndex]
+        this.readIndex += 1
+        // How do I send an EOF??
+        return char
+    }
+}
+
+const stdoutBufSize = 128;
+const stdoutBuf = new Int32Array()
+let index = 0;
+
+const stdout = (charCode) => {
+    if (charCode) {
+        postMessage({
+            type: 'stdout',
+            stdout: String.fromCharCode(charCode),
+        })
+    } else {
+        console.log(typeof charCode, charCode)
+    }
+}
+
+const stderr = (charCode) => {
+    if (charCode) {
+        postMessage({
+            type: 'stderr',
+            stderr: String.fromCharCode(charCode),
+        })
+    } else {
+        console.log(typeof charCode, charCode)
+    }
+}
+
+const stdinBuffer = new StdinBuffer()
+
+var Module = {
+    noInitialRun: true,
+    stdin: stdinBuffer.stdin,
+    stdout: stdout,
+    stderr: stderr,
+    onRuntimeInitialized: () => {
+        postMessage({type: 'ready', stdinBuffer: stdinBuffer.sab})
+    }
+}
+
+onmessage = (event) => {
+    if (event.data.type === 'run') {
+        // TODO: Set up files from event.data.files
+        const ret = callMain(event.data.args)
+        postMessage({
+            type: 'finished',
+            returnCode: ret
+        })
+    }
+}
+
+importScripts('python.js')

Tools/wasm/wasm_webserver.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+import argparse
+from http import server
+
+parser = argparse.ArgumentParser(
+    description="Start a local webserver with a Python terminal."
+)
+parser.add_argument(
+    "--port", type=int, default=8000, help="port for the http server to listen on"
+)
+parser.add_argument(
+    "--bind", type=str, default="127.0.0.1", help="Bind address (empty for all)"
+)
+
+
+class MyHTTPRequestHandler(server.SimpleHTTPRequestHandler):
+    def end_headers(self):
+        self.send_my_headers()
+        super().end_headers()
+
+    def send_my_headers(self):
+        self.send_header("Cross-Origin-Opener-Policy", "same-origin")
+        self.send_header("Cross-Origin-Embedder-Policy", "require-corp")
+
+
+def main():
+    args = parser.parse_args()
+    if not args.bind:
+        args.bind = None
+
+    server.test(
+        HandlerClass=MyHTTPRequestHandler,
+        protocol="HTTP/1.1",
+        port=args.port,
+        bind=args.bind,
+    )
+
+if __name__ == "__main__":
+    main()

configure

@@ -6338,7 +6338,7 @@ else
 
   case $ac_sys_system/$ac_sys_emscripten_target in #(
   Emscripten/browser*) :
-    EXEEXT=.html ;; #(
+    EXEEXT=.js ;; #(
   Emscripten/node*) :
     EXEEXT=.js ;; #(
   WASI/*) :

configure.ac

@@ -1137,7 +1137,7 @@ AC_ARG_WITH([suffix],
   )
 ], [
   AS_CASE([$ac_sys_system/$ac_sys_emscripten_target],
-    [Emscripten/browser*], [EXEEXT=.html],
+    [Emscripten/browser*], [EXEEXT=.js],
     [Emscripten/node*], [EXEEXT=.js],
     [WASI/*], [EXEEXT=.wasm],
     [EXEEXT=]