Backport end-to-end tests to TinyPilot Community (#1451)

This is code that we already reviewed and merged into Pro, but it would be useful in Community as well. The only change here from what's already in Pro is fd616979e1 as we're adjusting the test for Community. <a data-ca-tag href="https://codeapprove.com/pr/tiny-pilot/tinypilot/1451"><img src="https://codeapprove.com/external/github-tag-allbg.png" alt="Review on CodeApprove" /></a> --------- Co-authored-by: Andrew Farrell <123831+amfarrell@users.noreply.github.com> Co-authored-by: Andrew M. Farrell <armorsmith42@gmail.com>
2023-10-01 22:58:29 +03:00 · 2023-06-16 07:45:22 -04:00
parent bf1b4f611a
commit 7c25815df6
11 changed files with 365 additions and 3 deletions
--- a/.circleci/continue_config.yml
+++ b/.circleci/continue_config.yml
@@ -94,6 +94,45 @@ jobs:
      - run:
          name: Run build script
          command: ./dev-scripts/build-javascript
+  e2e:
+    docker:
+      # To run tests against the dev server, Playwright needs a CircleCI image
+      # with Python, Node, and a browser. While the build_python and
+      # build_javascript steps use python-3.7.4 and node-14.17.5 respectively,
+      # the CircleCI image with *both* python and node in the closest versions
+      # is python:3.7.11-browsers. It has python-3.7.11 and node-14.17.1.
+      #
+      # See: https://circleci.com/developer/images/image/cimg/python#tagging-scheme
+      - image: cimg/python:3.7.11-browsers
+    environment:
+      NODE_ENV: development
+    steps:
+      - checkout
+      - run:
+          name: Create virtual environment
+          command: python3 -m venv venv
+      - run:
+          name: Install python requirements
+          command: |
+            . venv/bin/activate
+            pip install --requirement requirements.txt
+            pip install --requirement dev_requirements.txt
+      - run:
+          name: Install node dependencies
+          command: npm install
+      - run:
+          name: Enable mock scripts
+          command: sudo ./dev-scripts/enable-mock-scripts
+      - run:
+          name: Enable passwordless sudo
+          command: sudo ./dev-scripts/enable-passwordless-sudo
+      - run:
+          name: Run playwright tests
+          command: npx playwright test
+      - store_artifacts:
+          path: playwright-report
+      - store_artifacts:
+          path: e2e-results
  build_debian_package:
    docker:
      - image: cimg/base:stable
@@ -300,6 +339,7 @@ workflows:
      - build_python
      - build_javascript
      - build_debian_package
+      - e2e
      - lint_debian_package:
          requires:
            - build_debian_package
--- a/.gitignore
+++ b/.gitignore
@@ -74,3 +74,8 @@ node_modules

 # Debian package builds
 debian-pkg/releases/
+
+# End-to-End test data
+e2e-results/
+/playwright-report/
+/playwright/.cache/
--- a/.prettierignore
+++ b/.prettierignore
@@ -1,2 +1,4 @@
 app/static/third-party
 venv
+e2e-results
+playwright-report
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -23,10 +23,11 @@ python3 -m venv venv && \
  pip install --requirement requirements.txt && \
  pip install --requirement dev_requirements.txt && \
  npm install && \
+  sudo npx playwright install-deps && \
  ./dev-scripts/enable-multiarch-docker
 ```

-### Run automated tests
+### Run dev tests

 To run TinyPilot's build scripts, run:

@@ -34,6 +35,20 @@ To run TinyPilot's build scripts, run:
 ./dev-scripts/build
 ```

+### Run end-to-end tests
+
+To spawn a TinyPilot local dev server and run TinyPilot's end-to-end tests against that dev server, run:
+
+```bash
+./dev-scripts/run-e2e-tests
+```
+
+To run TinyPilot's end-to-end tests against a running TinyPilot device, first turn off HTTPS redirection. Open the device's page in your browser and click through the privacy error. Then, navigate the menu options `System > Security`. Turn off "Require encrypted connection (HTTPS)". Finally, run the tests by passing an http URL as the first argument like so:
+
+```bash
+./dev-scripts/run-e2e-tests http://tinypilot.local
+```
+
 ### Enable Git hooks

 If you're planning to contribute code to TinyPilot, it's a good idea to enable the standard Git hooks so that build scripts run before you commit. That way, you can see if basic tests pass in a few seconds rather than waiting a few minutes to watch them run in CircleCI.
@@ -42,7 +57,7 @@ If you're planning to contribute code to TinyPilot, it's a good idea to enable t
 ./hooks/enable_hooks
 ```

-### Enable mock scripts
+### Enable mock scripts and passwordless sudo access

 The TinyPilot server backend uses several privileged scripts (provisioned to [`/opt/tinypilot-privileged/scripts/`](debian-pkg/opt/tinypilot-privileged/scripts)). Those scripts exist on a provisioned TinyPilot device, but they don't exist on a dev machine.

@@ -52,6 +67,12 @@ To set up symlinks that mock out those scripts and facilitate development, run t
 sudo ./dev-scripts/enable-mock-scripts
 ```

+If you do not already have passwordless sudo enabled in general, you also need to allow the server backend to execute these privileged scripts and other services without interactively prompting you for a password. To do that, run:
+
+```bash
+sudo ./dev-scripts/enable-passwordless-sudo
+```
+
 ### Run in dev mode

 To run TinyPilot on a non-Pi machine, run:
--- a/dev-scripts/enable-passwordless-sudo
+++ b/dev-scripts/enable-passwordless-sudo
@@ -0,0 +1,52 @@
+#!/bin/bash
+#
+# Enable passwordless sudo for paths that TinyPilot requires.
+
+# Note: If you run with passwordless sudo enabled globally for your user
+# account, you don't need to run this script.
+
+# In production, TinyPilot has permission to run an allowlist of paths with sudo
+# permissions. For developers who don't have passwordless sudo enabled globally
+# for their user account, certain TinyPilot functionality will fail waiting on a
+# sudo escalation prompt. This script prevents those failures by granting your
+# dev user account passwordless sudo for the specific paths the TinyPilot dev
+# server needs.
+#
+# To undo this script's changes, run:
+#   sudo rm /etc/sudoers.d/tinypilot
+
+if [[ "${EUID}" -ne 0 ]]; then
+  echo "This script requires root privileges." >&2
+  echo "Please re-run with sudo:" >&2
+  echo "  sudo ${0}" >&2
+  exit 1
+fi
+
+# Exit on first failure.
+set -e
+
+# Echo commands before executing them, by default to stderr.
+set -x
+
+# Exit on unset variable.
+set -u
+
+# This script must be run as sudo but needs the name of the user who invoked it.
+readonly USERNAME="${SUDO_USER}"
+
+# Change directory to repository root.
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+readonly SCRIPT_DIR
+cd "${SCRIPT_DIR}/.."
+
+readonly SUDOERS_SOURCE="./debian-pkg/etc/sudoers.d/tinypilot"
+readonly SUDOERS_DEST='/etc/sudoers.d/tinypilot'
+
+# Create parent folder in case it does not exist.
+sudo mkdir -p "$( dirname ${SUDOERS_DEST} )"
+
+# Copy the sudoers file from the debian package while modifying it to apply to
+# the invoking user.
+sed "s/^tinypilot/${USERNAME}/" < "${SUDOERS_SOURCE}" > "${SUDOERS_DEST}"
+
+echo "Wrote TinyPilot dev sudoers.d file to \"${SUDOERS_DEST}\""
--- a/dev-scripts/run-e2e-tests
+++ b/dev-scripts/run-e2e-tests
@@ -0,0 +1,52 @@
+#!/bin/bash
+#
+# Run end-to-end tests of UI navigation.
+
+# Exit on unset variable.
+set -u
+
+# Exit on first failure.
+set -e
+
+print_help() {
+  cat << EOF
+Usage: ${0##*/} [target URL]
+Run end-to-end tests of UI navigation.
+  target URL: URL of running TinyPilot server to test against.
+    Must start with 'http'.
+    If not specified, the script first starts a local dev server.
+  -h Display this help and exit.
+EOF
+}
+
+# Parse command-line arguments.
+while getopts 'h' opt; do
+  case "${opt}" in
+    h)
+      print_help
+      exit
+      ;;
+    *)
+      print_help >&2
+      exit 1
+  esac
+done
+
+# Echo commands before executing them, by default to stderr.
+set -x
+
+# Change directory to repository root.
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+readonly SCRIPT_DIR
+cd "${SCRIPT_DIR}/.."
+
+# If the first argument starts with http, strip it to export it as E2E_BASE_URL
+# That way, playwright.config.js can use it to override the default base URL and
+# will not start a local webserver.
+if [[ $# -gt 0 && "$1" =~ ^http ]]; then
+  export E2E_BASE_URL="$1"
+  readonly E2E_BASE_URL
+  shift
+fi
+
+npx playwright test
--- a/dev-scripts/serve-dev
+++ b/dev-scripts/serve-dev
@@ -11,7 +11,7 @@ set -u

 # Serve TinyPilot in dev mode.
 HOST=0.0.0.0 \
-  PORT=8000 \
+  PORT=${PORT:=8000} \
  DEBUG=1 \
  USE_RELOADER=1 \
  APP_SETTINGS_FILE=../dev_app_settings.cfg \
--- a/e2e/about.spec.js
+++ b/e2e/about.spec.js
@@ -0,0 +1,94 @@
+import { test, expect } from "@playwright/test";
+
+test("shows about page, license, privacy policy, and dependency pages and licenses", async ({
+  page,
+}) => {
+  await page.goto("/");
+  await page.getByText("Help", { exact: true }).hover();
+  await page.getByText("About", { exact: true }).click();
+  await expect(
+    page.getByRole("heading", { name: "About TinyPilot" })
+  ).toBeVisible();
+
+  const licensePagePromise = page.waitForEvent("popup");
+  await page.getByRole("link", { name: "MIT license" }).click();
+  const licensePage = await licensePagePromise;
+  await expect(licensePage.locator("body")).toContainText(
+    "Copyright 2022 TinyPilot, LLC"
+  );
+  await licensePage.close();
+
+  {
+    const privacyPolicyPagePromise = page.waitForEvent("popup");
+    await page.getByRole("link", { name: "Privacy Policy" }).click();
+    const privacyPolicyPage = await privacyPolicyPagePromise;
+    await expect(privacyPolicyPage.locator("body")).toContainText(
+      "PRIVACY POLICY"
+    );
+    await privacyPolicyPage.close();
+  }
+
+  {
+    const flaskProjectPagePromise = page.waitForEvent("popup");
+    await page
+      .getByRole("link", { name: "Flask", exact: true })
+      .first()
+      .click();
+    const flaskProjectPage = await flaskProjectPagePromise;
+    await expect(flaskProjectPage).toHaveURL(
+      new RegExp("https://flask.palletsprojects.com.*")
+    );
+    // We assert the presense of some text so the trace report shows the page render.
+    await expect(flaskProjectPage.locator("body")).not.toBeEmpty();
+    await flaskProjectPage.close();
+  }
+
+  {
+    const flaskLicensePagePromise = page.waitForEvent("popup");
+    await page
+      .getByRole("listitem")
+      .filter({ hasText: "Flask (License)" })
+      .getByRole("link", { name: "License" })
+      .click();
+    const flaskLicensePage = await flaskLicensePagePromise;
+    await expect(flaskLicensePage).toHaveURL(
+      new RegExp(".*licensing/Flask/license.*")
+    );
+    await expect(flaskLicensePage.locator("body")).not.toBeEmpty();
+    await flaskLicensePage.close();
+  }
+
+  {
+    const cryptographyProjectPagePromise = page.waitForEvent("popup");
+    await page
+      .getByRole("link", { name: "cryptography", exact: true })
+      .first()
+      .click();
+    const cryptographyProjectPage = await cryptographyProjectPagePromise;
+    await expect(cryptographyProjectPage).toHaveURL(
+      new RegExp("https://cryptography.io.*")
+    );
+    await expect(cryptographyProjectPage.locator("body")).not.toBeEmpty();
+    await cryptographyProjectPage.close();
+  }
+
+  {
+    const cryptographyLicensePagePromise = page.waitForEvent("popup");
+    await page
+      .getByRole("listitem")
+      .filter({ hasText: "cryptography (License)" })
+      .getByRole("link", { name: "License" })
+      .click();
+    const cryptographyLicensePage = await cryptographyLicensePagePromise;
+    await expect(cryptographyLicensePage).toHaveURL(new RegExp("cryptography"));
+    await expect(cryptographyLicensePage.locator("body")).not.toBeEmpty();
+    await cryptographyLicensePage.close();
+  }
+
+  await page.getByRole("button", { name: "Close", exact: true }).click();
+  await expect(
+    page.getByRole("heading", { name: "About TinyPilot" })
+  ).not.toBeVisible();
+
+  await page.close();
+});
--- a/e2e/debug-logs.spec.js
+++ b/e2e/debug-logs.spec.js
@@ -0,0 +1,53 @@
+import { test, expect } from "@playwright/test";
+
+/**
+ * Read the contents of the clipboard. To do this, create a new browser tab with
+ * an <input> element, simulate hitting Control+V to paste into the <input>, and
+ * read what it pasted.
+ *
+ * Playwright can't yet read or paste from the clipboard, so this is a workaround.
+ * https://github.com/microsoft/playwright/issues/15860
+ */
+async function readClipboardContents(page) {
+  // Create a new browser tab so that none of the event handlers in the
+  // tab-under-test prevent the test from pasting the clipboard contents.
+  const freshPage = await page.context().newPage();
+
+  // Create an input element so the test has a place to paste the clipboard
+  // contents into.
+  const input = await freshPage.evaluateHandle(() => {
+    return document.body.appendChild(document.createElement("input"));
+  });
+
+  const isMac = process.platform === "darwin";
+  const modifier = isMac ? "Meta" : "Control";
+  await input.press(`${modifier}+KeyV`);
+
+  return await input.inputValue();
+}
+
+test("loads debug logs and generates a shareable URL for them", async ({
+  page,
+}) => {
+  await page.goto("/");
+
+  await page.getByText("System", { exact: true }).hover();
+  await page.getByText("Logs", { exact: true }).click();
+  await expect(page.getByRole("heading", { name: "Debug Logs" })).toBeVisible();
+
+  await page.getByRole("button", { name: "Get Shareable URL" }).click();
+  await page.getByRole("button", { name: "Copy" }).click();
+  const copiedUrl = await readClipboardContents(page);
+  await expect(copiedUrl).toMatch(
+    new RegExp("^https://logs.tinypilotkvm.com/.*")
+  );
+
+  await expect(page.locator("#logs-success .logs-output")).toContainText(
+    "TinyPilot version: "
+  );
+
+  await page.getByRole("button", { name: "Close", exact: true }).click();
+  await expect(
+    page.getByRole("heading", { name: "Debug Logs" })
+  ).not.toBeVisible();
+});
--- a/package.json
+++ b/package.json
@@ -1,9 +1,11 @@
 {
  "type": "module",
  "devDependencies": {
+    "@playwright/test": "1.32.1",
    "eslint": "8.40.0",
    "eslint-plugin-html": "7.1.0",
    "mocha": "9.2.2",
+    "playwright": "1.32.1",
    "prettier": "2.0.5"
  }
 }
--- a/playwright.config.js
+++ b/playwright.config.js
@@ -0,0 +1,41 @@
+// @ts-check
+import { defineConfig, devices } from "@playwright/test";
+
+/**
+ * @see https://playwright.dev/docs/test-configuration
+ */
+export default defineConfig({
+  testDir: "./e2e",
+  fullyParallel: false,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  retries: 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+  reporter: "html",
+  use: {
+    baseURL: process.env.E2E_BASE_URL || "http://0.0.0.0:9000",
+    actionTimeout: 0,
+    trace: "on",
+    video: "on",
+  },
+
+  projects: [
+    {
+      name: "chromium",
+      use: { ...devices["Desktop Chrome"] },
+    },
+  ],
+
+  outputDir: "e2e-results/",
+
+  /* Do not start the local web server when running against a target TinyPilot server. */
+  webServer: process.env.E2E_BASE_URL
+    ? undefined
+    : {
+        command:
+          ". venv/bin/activate && export PORT=9000 && ./dev-scripts/serve-dev",
+        url: "http://0.0.0.0:9000",
+        reuseExistingServer: !process.env.CI,
+      },
+});