Add startup testing for Wasm Pinecone build (#1910)

* Only include go-sqlite3 on the relevant binaries

* The driver name is always sqlite3 now

* Update to matrix-org/go-sqlite3-js@e537baa

* Add initial Wasm test harness

* Upgrade go-sqlite3-js

This fixes an error about semicolons in single statements.

* Add browser-like WebSocket API for testing

* Upgrade go-sqlite3-js

This upgrade includes printing panic messages next to stacks.

* Run for all PRs targeting any branch

* Use manual Node caching

* Temporarily run for all pushes

* Use npm ci instead of install

* Use HTTPS auth for repo packages

* Match path style from build.sh

* update utp

Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>

.github/workflows/wasm.yml

@@ -0,0 +1,49 @@
+name: WebAssembly
+
+on:
+    push:
+    pull_request:
+
+jobs:
+    test:
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v2
+
+            - name: Install Go
+              uses: actions/setup-go@v2
+              with:
+                  go-version: 1.16.5
+
+            - uses: actions/cache@v2
+              with:
+                  path: |
+                      ~/.cache/go-build
+                      ~/go/pkg/mod
+                  key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+                  restore-keys: |
+                      ${{ runner.os }}-go-
+
+            - name: Install Node
+              uses: actions/setup-node@v2
+              with:
+                  node-version: 14
+
+            - uses: actions/cache@v2
+              with:
+                  path: ~/.npm
+                  key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+                  restore-keys: |
+                      ${{ runner.os }}-node-
+
+            - name: Reconfigure Git to use HTTPS auth for repo packages
+              run: >
+                  git config --global url."https://github.com/".insteadOf
+                  ssh://git@github.com/
+
+            - name: Install test dependencies
+              working-directory: ./test/wasm
+              run: npm ci
+
+            - name: Test
+              run: ./test-dendritejs.sh

.gitignore

@@ -3,6 +3,9 @@
 # Hidden files
 .*
 
+# Allow GitHub config
+!.github
+
 # Downloads
 /.downloads
 
@@ -36,6 +39,7 @@ _testmain.go
 *.exe
 *.test
 *.prof
+*.wasm
 
 # Generated keys
 *.pem
@@ -53,3 +57,6 @@ dendrite.yaml
 
 # Generated code
 cmd/dendrite-demo-yggdrasil/embed/fs*.go
+
+# Test dependencies
+test/wasm/node_modules

build-dendritejs.sh

@@ -1,4 +1,4 @@
 #!/bin/sh -eu
 
 export GIT_COMMIT=$(git rev-list -1 HEAD) && \
-GOOS=js GOARCH=wasm go build -ldflags "-X main.GitCommit=$GIT_COMMIT" -o main.wasm ./cmd/dendritejs
+GOOS=js GOARCH=wasm go build -ldflags "-X main.GitCommit=$GIT_COMMIT" -o bin/main.wasm ./cmd/dendritejs-pinecone

build.sh

@@ -21,4 +21,4 @@ mkdir -p bin
 
 CGO_ENABLED=1 go build -trimpath -ldflags "$FLAGS" -v -o "bin/" ./cmd/...
 
-CGO_ENABLED=0 GOOS=js GOARCH=wasm go build -trimpath -ldflags "$FLAGS" -o bin/main.wasm ./cmd/dendritejs
+CGO_ENABLED=0 GOOS=js GOARCH=wasm go build -trimpath -ldflags "$FLAGS" -o bin/main.wasm ./cmd/dendritejs-pinecone

cmd/dendritejs-pinecone/main.go

@@ -144,6 +144,13 @@ func generateKey() ed25519.PrivateKey {
 }
 
 func main() {
+	startup()
+
+	// We want to block forever to let the fetch and libp2p handler serve the APIs
+	select {}
+}
+
+func startup() {
 	sk := generateKey()
 	pk := sk.Public().(ed25519.PublicKey)
 
@@ -251,7 +258,4 @@ func main() {
 			}
 		}
 	}()
-
-	// We want to block forever to let the fetch and libp2p handler serve the APIs
-	select {}
 }

cmd/dendritejs-pinecone/main_test.go

@@ -0,0 +1,25 @@
+// Copyright 2021 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build wasm
+
+package main
+
+import (
+	"testing"
+)
+
+func TestStartup(t *testing.T) {
+	startup()
+}

docs/p2p.md

@@ -2,14 +2,23 @@
 
 These are the instructions for setting up P2P Dendrite, current as of May 2020. There's both Go stuff and JS stuff to do to set this up.
 
-
 ### Dendrite
 
+#### Build
+
 - The `master` branch has a WASM-only binary for dendrite: `./cmd/dendritejs`.
 - Build it and copy assets to riot-web.
 ```
-$ GOOS=js GOARCH=wasm go build -o main.wasm ./cmd/dendritejs
+$ ./build-dendritejs.sh
-$ cp main.wasm ../riot-web/src/vector/dendrite.wasm
+$ cp bin/main.wasm ../riot-web/src/vector/dendrite.wasm
+```
+
+#### Test
+
+To check that the Dendrite side is working well as Wasm, you can run the
+Wasm-specific tests:
+```
+$ ./test-dendritejs.sh
 ```
 
 ### Rendezvous

go.mod

@@ -37,7 +37,7 @@ require (
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.7-0.20210414154423-1157a4212dcb
 	github.com/morikuni/aec v1.0.0 // indirect
-	github.com/neilalexander/utp v0.1.1-0.20210705212447-691f29ad692b
+	github.com/neilalexander/utp v0.1.1-0.20210720104546-52626cdf31b2
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
 	github.com/ngrok/sqlmw v0.0.0-20200129213757-d5c93a81bec6
 	github.com/opentracing/opentracing-go v1.2.0

go.sum

@@ -118,9 +118,7 @@ github.com/anacrolix/log v0.3.0/go.mod h1:lWvLTqzAnCWPJA08T2HCstZi0L1y2Wyvm3FJgw
 github.com/anacrolix/missinggo v1.1.2-0.20190815015349-b888af804467/go.mod h1:MBJu3Sk/k3ZfGYcS7z18gwfu72Ey/xopPFJJbTi5yIo=
 github.com/anacrolix/missinggo v1.2.1 h1:0IE3TqX5y5D0IxeMwTyIgqdDew4QrzcXaaEnJQyjHvw=
 github.com/anacrolix/missinggo v1.2.1/go.mod h1:J5cMhif8jPmFoC3+Uvob3OXXNIhOUikzMt+uUjeM21Y=
-github.com/anacrolix/missinggo/perf v1.0.0 h1:7ZOGYziGEBytW49+KmYGTaNfnwUqP1HBsy6BqESAJVw=
 github.com/anacrolix/missinggo/perf v1.0.0/go.mod h1:ljAFWkBuzkO12MQclXzZrosP5urunoLS0Cbvb4V0uMQ=
-github.com/anacrolix/sync v0.2.0 h1:oRe22/ZB+v7v/5Mbc4d2zE0AXEZy0trKyKLjqYOt6tY=
 github.com/anacrolix/sync v0.2.0/go.mod h1:BbecHL6jDSExojhNtgTFSBcdGerzNc64tz3DCOj/I0g=
 github.com/anacrolix/tagflag v0.0.0-20180109131632-2146c8d41bf0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw=
 github.com/anacrolix/utp v0.1.0/go.mod h1:MDwc+vsGEq7RMw6lr2GKOEqjWny5hO5OZXRVNaBJ2Dk=
@@ -1186,8 +1184,8 @@ github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/
 github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
 github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
 github.com/neilalexander/utp v0.1.1-0.20210622132614-ee9a34a30488/go.mod h1:NPHGhPc0/wudcaCqL/H5AOddkRf8GPRhzOujuUKGQu8=
-github.com/neilalexander/utp v0.1.1-0.20210705212447-691f29ad692b h1:XNm+Ks3bVziRJxcMaIbzumWEw7l52z9Rek6cMHgln1g=
+github.com/neilalexander/utp v0.1.1-0.20210720104546-52626cdf31b2 h1:txJOiDxsypF8RbzbcyOD3ovip+uUWNZE/Zo7qLdARZQ=
-github.com/neilalexander/utp v0.1.1-0.20210705212447-691f29ad692b/go.mod h1:ylsx0342RjGHjOoVKhR/wz/7Lhiusonihfj4QLxEMcU=
+github.com/neilalexander/utp v0.1.1-0.20210720104546-52626cdf31b2/go.mod h1:NPHGhPc0/wudcaCqL/H5AOddkRf8GPRhzOujuUKGQu8=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
 github.com/ngrok/sqlmw v0.0.0-20200129213757-d5c93a81bec6 h1:evlcQnJY+v8XRRchV3hXzpHDl6GcEZeLXAhlH9Csdww=

test-dendritejs.sh

@@ -0,0 +1,3 @@
+#!/bin/sh -eu
+
+GOOS=js GOARCH=wasm go test -v -exec "$(pwd)/test/wasm/index.js" ./cmd/dendritejs-pinecone

test/wasm/index.js

@@ -0,0 +1,52 @@
+#!/usr/bin/env node
+
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+const fs = require('fs');
+const path = require('path');
+const childProcess = require('child_process');
+
+(async function() {
+    // sql.js
+    const initSqlJs = require('sql.js');
+    await initSqlJs().then(SQL => {
+        global._go_sqlite = SQL;
+        console.log("Loaded sqlite")
+    });
+    // dendritejs expects to write to `/idb` so we create that here
+    // Since this is testing only, we use the default in-memory FS
+    global._go_sqlite.FS.mkdir("/idb");
+
+    // WebSocket
+    const WebSocket = require('isomorphic-ws');
+    global.WebSocket = WebSocket;
+
+    // Load the generic Go Wasm exec helper inline to trigger built-in run call
+    // This approach avoids copying `wasm_exec.js` into the repo, which is nice
+    // to aim for since it can differ between Go versions.
+    const goRoot = await new Promise((resolve, reject) => {
+        childProcess.execFile('go', ['env', 'GOROOT'], (err, out) => {
+            if (err) {
+                reject("Can't find go");
+            }
+            resolve(out.trim());
+        });
+    });
+    const execPath = path.join(goRoot, 'misc/wasm/wasm_exec.js');
+    const execCode = fs.readFileSync(execPath, 'utf8');
+    eval(execCode);
+})();

test/wasm/package-lock.json

@@ -0,0 +1,67 @@
+{
+  "name": "wasm",
+  "lockfileVersion": 2,
+  "requires": true,
+  "packages": {
+    "": {
+      "dependencies": {
+        "isomorphic-ws": "^4.0.1",
+        "sql.js": "github:neilalexander/sql.js#252a72bf57b0538cbd49bbd6f70af71e516966ae",
+        "ws": "^7.5.2"
+      }
+    },
+    "node_modules/isomorphic-ws": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/isomorphic-ws/-/isomorphic-ws-4.0.1.tgz",
+      "integrity": "sha512-BhBvN2MBpWTaSHdWRb/bwdZJ1WaehQ2L1KngkCkfLUGF0mAWAT1sQUQacEmQ0jXkFw/czDXPNQSL5u2/Krsz1w==",
+      "peerDependencies": {
+        "ws": "*"
+      }
+    },
+    "node_modules/sql.js": {
+      "version": "1.5.0",
+      "resolved": "git+ssh://git@github.com/neilalexander/sql.js.git#252a72bf57b0538cbd49bbd6f70af71e516966ae",
+      "integrity": "sha512-EFYI/yMoQ1U08nZxQOZ7+4S0nOpKF45EVoWGef8L1kvSCMP3B3xSzwZeOmoF2tBVpbMssAgHEz43cf0ZulRDSQ==",
+      "license": "MIT"
+    },
+    "node_modules/ws": {
+      "version": "7.5.2",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.2.tgz",
+      "integrity": "sha512-lkF7AWRicoB9mAgjeKbGqVUekLnSNO4VjKVnuPHpQeOxZOErX6BPXwJk70nFslRCEEA8EVW7ZjKwXaP9N+1sKQ==",
+      "engines": {
+        "node": ">=8.3.0"
+      },
+      "peerDependencies": {
+        "bufferutil": "^4.0.1",
+        "utf-8-validate": "^5.0.2"
+      },
+      "peerDependenciesMeta": {
+        "bufferutil": {
+          "optional": true
+        },
+        "utf-8-validate": {
+          "optional": true
+        }
+      }
+    }
+  },
+  "dependencies": {
+    "isomorphic-ws": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/isomorphic-ws/-/isomorphic-ws-4.0.1.tgz",
+      "integrity": "sha512-BhBvN2MBpWTaSHdWRb/bwdZJ1WaehQ2L1KngkCkfLUGF0mAWAT1sQUQacEmQ0jXkFw/czDXPNQSL5u2/Krsz1w==",
+      "requires": {}
+    },
+    "sql.js": {
+      "version": "git+ssh://git@github.com/neilalexander/sql.js.git#252a72bf57b0538cbd49bbd6f70af71e516966ae",
+      "integrity": "sha512-EFYI/yMoQ1U08nZxQOZ7+4S0nOpKF45EVoWGef8L1kvSCMP3B3xSzwZeOmoF2tBVpbMssAgHEz43cf0ZulRDSQ==",
+      "from": "sql.js@github:neilalexander/sql.js#252a72bf57b0538cbd49bbd6f70af71e516966ae"
+    },
+    "ws": {
+      "version": "7.5.2",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.2.tgz",
+      "integrity": "sha512-lkF7AWRicoB9mAgjeKbGqVUekLnSNO4VjKVnuPHpQeOxZOErX6BPXwJk70nFslRCEEA8EVW7ZjKwXaP9N+1sKQ==",
+      "requires": {}
+    }
+  }
+}

test/wasm/package.json

@@ -0,0 +1,7 @@
+{
+  "dependencies": {
+    "isomorphic-ws": "^4.0.1",
+    "sql.js": "github:neilalexander/sql.js#252a72bf57b0538cbd49bbd6f70af71e516966ae",
+    "ws": "^7.5.2"
+  }
+}