scripts: pytest-based Android device testing

Part-of: <https://gitlab.freedesktop.org/monado/monado/-/merge_requests/2311>

doc/changes/misc_features/mr.2311.md

@@ -0,0 +1 @@
+Add: Lifecycle tests that launch and switch OpenXR applications on Android.

tests/android/.gitignore

@@ -0,0 +1,4 @@
+# SPDX-FileCopyrightText: 2024, Collabora, Ltd.
+# SPDX-License-Identifier: CC0-1.0
+
+*.xml

tests/android/README.md

@@ -0,0 +1,63 @@
+# Android device tests
+
+<!--
+Copyright 2024, Collabora, Ltd.
+
+SPDX-License-Identifier: CC-BY-4.0
+-->
+
+This directory contains some tests to run on an Android device over ADB,
+primarily to verify some lifecycle handling right now.
+
+It uses [pytest][] as the test runner, which is more readable, maintainable, and
+usable than the earlier bash scripts, and gives the option for e.g. junit-style
+output for CI usage.
+
+The actual tests are in the `test_*.py` files, while `conftest.py` configures
+the pytest framework, as well as provides constants and fixtures for use in the
+tests themselves.
+
+[pytest]: https://pytest.org
+
+## Preconditions
+
+- Have `adb` in the path, and only the device under test connected. (or, have
+  environment variables set appropriately so that
+  `adb shell getprop ro.product.vendor.device` shows you the expected device.)
+- Have Python and pytest installed, either system-wide or in a `venv`. The
+  version of pytest in Debian Bookworm is suitable.
+- Have built and installed the "outOfProcessDebug" build of Monado.
+  (`./gradlew installOOPD` in root dir)
+- Have set the out-of-process runtime as active in the OpenXR Runtime Broker (if
+  applicable).
+- Have installed both Vulkan and OpenGL ES versions of the `hello_xr` sample app
+  from Khronos. (Binaries from a release are fine.)
+- If you want the tests to pass, turn on "Draw over other apps" -- but this is
+  actually a bug to require this.
+
+## Instructions
+
+Change your current working directory to this one. (Otherwise the extra
+argument you need to enable the ADB tests will not be recognized by pytest.)
+
+```sh
+cd tests/android/
+```
+
+Run the following command, or one based on it:
+
+```sh
+pytest -v --adb
+```
+
+Or, if you want junit-style output, add a variation on these two args:
+`--junit-xml=android-tests.xml -o junit_family=xunit1` for an overall command
+like:
+
+```sh
+pytest -v --adb --junit-xml=android-tests.xml -o junit_family=xunit1
+```
+
+Another useful options is `-k` which can be followed by a pattern used to
+selecet a subset of tests to run. Handy if you only want to run one or two
+tests.

tests/android/conftest.py

@@ -0,0 +1,273 @@
+# Copyright 2024, Collabora, Ltd.
+#
+# SPDX-License-Identifier: BSL-1.0
+#
+# Author: Rylie Pavlik <rylie.pavlik@collabora.com>
+"""
+pytest configuration and helpers for adb/Android.
+
+Helper functions to run tests using adb on an Android device.
+
+pytest serves as the test runner and collects the results.
+
+See README.md in this directory for requirements and running instructions.
+"""
+
+import subprocess
+import time
+from pathlib import Path
+from typing import Optional
+
+import pytest
+
+_NATIVE_ACTIVITY = "android.app.NativeActivity"
+helloxr_vulkan_pkg = "org.khronos.openxr.hello_xr.vulkan"
+helloxr_vulkan_activity = f"{helloxr_vulkan_pkg}/{_NATIVE_ACTIVITY}"
+helloxr_gles_pkg = "org.khronos.openxr.hello_xr.opengles"
+helloxr_gles_activity = f"{helloxr_gles_pkg}/{_NATIVE_ACTIVITY}"
+
+RUNTIME = "org.freedesktop.monado.openxr_runtime.out_of_process"
+
+_PKGS_TO_STOP = [
+    helloxr_gles_pkg,
+    helloxr_vulkan_pkg,
+    "org.khronos.openxr.cts",
+    "org.freedesktop.monado.openxr_runtime.in_process",
+    RUNTIME,
+]
+
+_REPO_ROOT = Path(__file__).resolve().parent.parent.parent
+
+
+class AdbTesting:
+    """pytest fixture for testing on an Android device with adb."""
+
+    def __init__(self, tmp_path: Optional[Path]):
+        """Initialize members."""
+        self.tmp_path = tmp_path
+        self.logcat_grabbed = False
+        self.logcat_echoed = False
+        self.log: Optional[str] = None
+
+        # TODO allow most of the following to be configured
+        self._ndk_ver = "26.3.11579264"
+        self._build_variant = "outOfProcessDebug"
+        self._sdk_root = Path.home() / "Android" / "Sdk"
+        self._arch = "arm64-v8a"
+        self._ndk_stack = self._sdk_root / "ndk" / self._ndk_ver / "ndk-stack"
+        self._caps_build_variant = (
+            self._build_variant[0].upper() + self._build_variant[1:]
+        )
+        self._sym_dir = (
+            _REPO_ROOT
+            / "src"
+            / "xrt"
+            / "targets"
+            / "openxr_android"
+            / "build"
+            / "intermediates"
+            / "merged_native_libs"
+            / self._build_variant
+            / f"merge{self._caps_build_variant}NativeLibs"
+            / "out"
+            / "lib"
+            / self._arch
+        )
+
+    def adb_call(self, cmd: list[str], **kwargs):
+        """Call ADB in a subprocess."""
+        return subprocess.check_call(cmd, **kwargs)
+
+    def adb_check_output(self, cmd: list[str], **kwargs):
+        """Call ADB in a subprocess and get the output."""
+        return subprocess.check_output(cmd, encoding="utf-8", **kwargs)
+
+    def get_device_state(self) -> Optional[str]:
+        try:
+            state = self.adb_check_output(["adb", "get-state"]).strip()
+            return state.strip()
+
+        except subprocess.CalledProcessError:
+            return None
+
+    def send_key(self, keyevent):
+        """Send a key event with `adb shell input keyevent`."""
+        time.sleep(1)
+        print(f"*** {keyevent}")
+        self.adb_call(["adb", "shell", "input", "keyevent", keyevent])
+
+    def send_tap(self, x, y):
+        """Send touchscreen tap at x, y."""
+        #     sleep 1
+        print(f"*** tap {x} {y}")
+        self.adb_call(
+            [
+                "adb",
+                "shell",
+                "input",
+                "touchscreen",
+                "tap",
+                str(x),
+                str(y),
+            ]
+        )
+
+    def start_activity(self, activity):
+        """Start an activity and wait with `adb shell am start-activity -W`."""
+        time.sleep(1)
+        print(f"*** starting {activity} and waiting")
+        self.adb_call(["adb", "shell", "am", "start-activity", "-W", activity])
+
+    def stop_and_clear_all(self):
+        """
+        Stop relevant packages and clear their data.
+
+        This second step removes them from recents.
+        """
+        print("*** stopping all relevant packages and clearing their data")
+        for pkg in _PKGS_TO_STOP:
+            self.adb_call(["adb", "shell", "am", "force-stop", pkg])
+            self.adb_call(["adb", "shell", "pm", "clear", pkg])
+
+    def start_test(
+        self,
+    ):
+        """Ensure a clean starting setup."""
+        self.stop_and_clear_all()
+        self.adb_call(["adb", "logcat", "-c"])
+
+    @property
+    def logcat_fn(self):
+        """Get computed path of logcat file."""
+        assert self.tmp_path
+        return self.tmp_path / "logcat.txt"
+
+    def grab_logcat(self):
+        """Save logcat to the named file and return it as a string."""
+        log = self.adb_check_output(["adb", "logcat", "-d"])
+        log_fn = self.logcat_fn
+        with open(log_fn, "w", encoding="utf-8") as fp:
+            fp.write(log)
+        print(f"Logcat saved to {log_fn}")
+        self.logcat_grabbed = True
+        self.log = log
+        return log
+
+    def dump_logcat(self):
+        """Print logcat to stdout."""
+        if self.logcat_echoed:
+            return
+        if not self.logcat_grabbed:
+            self.grab_logcat()
+        print("*** Logcat:")
+        print(self.log)
+        self.logcat_echoed = True
+
+    def check_for_crash(self):
+        """Dump logcat, and look for a crash."""
+        __tracebackhide__ = True
+        log = self.grab_logcat()
+        log_fn = self.logcat_fn
+        crash_name = log_fn.with_name("crashes.txt")
+        if "backtrace:" in log:
+            subprocess.check_call(
+                " ".join(
+                    (
+                        str(x)
+                        for x in (
+                            self._ndk_stack,
+                            "-sym",
+                            self._sym_dir,
+                            "-i",
+                            log_fn,
+                            ">",
+                            crash_name,
+                        )
+                    )
+                ),
+                shell=True,
+            )
+            self.dump_logcat()
+
+            with open(crash_name, "r", encoding="utf-8") as fp:
+                print("*** Crash Backtrace(s):")
+                print(fp.read())
+            print(f"*** Crash backtrace in {crash_name}")
+            pytest.fail("Native crash backtrace detected in test")
+
+        if "FATAL EXCEPTION" in log:
+            self.dump_logcat()
+            pytest.fail("Java exception detected in test")
+
+        if "WindowManager: ANR" in log:
+            self.dump_logcat()
+            pytest.fail("ANR detected in test")
+
+        self.stop_and_clear_all()
+
+
+@pytest.fixture(scope="function")
+def adb(tmp_path):
+    """Provide AdbTesting instance and cleanup as pytest fixture."""
+    adb_testing = AdbTesting(tmp_path)
+
+    adb_testing.start_test()
+
+    # Body of test happens here
+    yield adb_testing
+
+    # Grab logcat in case it wasn't grabbed, and dump to stdout
+    adb_testing.dump_logcat()
+
+    # Cleanup
+    adb_testing.stop_and_clear_all()
+
+
+@pytest.fixture(scope="session", autouse=True)
+def log_device_and_build(record_testsuite_property):
+    """Session fixture, autoused, to record device data."""
+    adb = AdbTesting(None)
+
+    try:
+        device = adb.adb_check_output(
+            ["adb", "shell", "getprop", "ro.product.vendor.device"]
+        )
+        build = adb.adb_check_output(
+            ["adb", "shell", "getprop", "ro.product.build.fingerprint"]
+        )
+    except subprocess.CalledProcessError:
+        pytest.skip("adb shell getprop failed - no device connected?")
+
+    record_testsuite_property("device", device.strip())
+    record_testsuite_property("build", build.strip())
+
+
+def device_is_available():
+    """Return True if an ADB device is available."""
+    adb = AdbTesting(None)
+
+    try:
+        state = adb.adb_check_output(["adb", "get-state"]).strip()
+        return state == "device"
+
+    except subprocess.CalledProcessError:
+        return False
+
+
+def pytest_addoption(parser: pytest.Parser):
+    """Pytest addoption function."""
+    parser.addoption(
+        "--adb",
+        action="store_true",
+        default=False,
+        help="run ADB/Android device tests",
+    )
+
+
+def pytest_configure(config):
+    """Configure function for pytest."""
+    config.addinivalue_line(
+        "markers",
+        "adb: mark test as needing adb and thus only run when an "
+        "Android device is available",
+    )

tests/android/test_app_switch.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+# Copyright 2024, Collabora, Ltd.
+#
+# SPDX-License-Identifier: BSL-1.0
+#
+# Author: Rylie Pavlik <rylie.pavlik@collabora.com>
+"""
+Tests of app lifecycle behavior (app pause/resume and switch).
+
+See README.md in this directory for requirements and running instructions.
+"""
+
+import time
+
+import pytest
+from conftest import helloxr_gles_activity, helloxr_vulkan_activity, helloxr_gles_pkg
+
+# Ignore missing docstrings:
+# flake8: noqa: D103
+
+skipif_not_adb = pytest.mark.skipif(
+    "not config.getoption('adb')", reason="--adb not passed to pytest"
+)
+
+# All the tests in this module require a device and ADB.
+pytestmark = [pytest.mark.adb, skipif_not_adb]
+
+
+def test_launch_and_back(adb):
+    # Launch activity
+    adb.start_activity(helloxr_gles_activity)
+    time.sleep(2)
+
+    # Press "Back"
+    adb.send_key("KEYCODE_BACK")
+
+    time.sleep(5)
+
+    adb.check_for_crash()
+
+
+def test_home_and_resume(adb):
+
+    # Launch activity
+    adb.start_activity(helloxr_gles_activity)
+    time.sleep(2)
+
+    # Press "Home"
+    adb.send_key("KEYCODE_HOME")
+    time.sleep(1)
+
+    # Press "App Switch"
+    adb.send_key("KEYCODE_APP_SWITCH")
+    time.sleep(1)
+
+    # Tap "somewhat middle" to re-select recent app
+    adb.send_tap(400, 400)
+
+    time.sleep(5)
+
+    adb.check_for_crash()
+
+
+def test_home_and_start(adb):
+
+    # Launch activity
+    adb.start_activity(helloxr_gles_activity)
+    time.sleep(2)
+
+    # Press "Home"
+    adb.send_key("KEYCODE_HOME")
+    time.sleep(2)
+
+    # Launch activity again
+    adb.start_activity(helloxr_gles_activity)
+
+    time.sleep(5)
+
+    adb.check_for_crash()
+
+
+def test_launch_second(adb):
+
+    # Launch A
+    adb.start_activity(helloxr_gles_activity)
+    time.sleep(2)
+
+    # Launch B
+    adb.start_activity(helloxr_vulkan_activity)
+
+    time.sleep(5)
+
+    adb.check_for_crash()
+
+
+def test_home_and_launch_second(adb):
+
+    # Launch A
+    adb.start_activity(helloxr_gles_activity)
+    time.sleep(2)
+
+    # Press "Home"
+    adb.send_key("KEYCODE_HOME")
+    time.sleep(2)
+
+    # Launch B
+    adb.start_activity(helloxr_vulkan_activity)
+
+    time.sleep(5)
+
+    adb.check_for_crash()
+
+
+def test_launch_a_b_a(adb):
+
+    # Launch A
+    adb.start_activity(helloxr_gles_activity)
+    time.sleep(2)
+
+    # Launch B
+    adb.start_activity(helloxr_vulkan_activity)
+    time.sleep(2)
+
+    # Launch A (again)
+    adb.start_activity(helloxr_gles_activity)
+
+    time.sleep(5)
+
+    adb.check_for_crash()

tests/android/test_single_app_reliability.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+# Copyright 2024, Collabora, Ltd.
+#
+# SPDX-License-Identifier: BSL-1.0
+#
+# Author: Rylie Pavlik <rylie.pavlik@collabora.com>
+"""
+Tests of a single app without inducing extra session state changes.
+
+See README.md in this directory for requirements and running instructions.
+"""
+
+import time
+
+import pytest
+from conftest import helloxr_gles_activity, helloxr_gles_pkg
+
+# Ignore missing docstrings:
+# flake8: noqa: D103
+
+skipif_not_adb = pytest.mark.skipif(
+    "not config.getoption('adb')", reason="--adb not passed to pytest"
+)
+
+# All the tests in this module require a device and ADB.
+pytestmark = [pytest.mark.adb, skipif_not_adb]
+
+
+def test_just_launch(adb):
+    # Just launch activity and make sure it starts OK.
+    adb.start_activity(helloxr_gles_activity)
+
+    time.sleep(5)
+
+    adb.check_for_crash()
+
+
+def test_launch_and_monkey(adb):
+    # Launch activity
+    adb.start_activity(helloxr_gles_activity)
+    time.sleep(2)
+
+    # Release the monkey! 1k events.
+    adb.adb_call(
+        [
+            "adb",
+            "shell",
+            "monkey",
+            "-p",
+            helloxr_gles_pkg,
+            "-v",
+            "1000",
+            # seed
+            "-s",
+            "100",
+            "--pct-syskeys",
+            "0",
+        ]
+    )
+
+    adb.check_for_crash()