Add e2e tests

.github/workflows/update-font-list.yml

@@ -19,7 +19,7 @@ jobs:
           persist-credentials: true
 
       - name: Set up Deno
-        uses: denoland/setup-deno@v2
+        uses: denoland/setup-deno@909cc5acb0fdd60627fb858598759246509fa755 # v2.0.2
         with:
           deno-version: v2.x
 

.github/workflows/update-localizations.yml

@@ -20,7 +20,7 @@ jobs:
           ssh-key: ${{ secrets.SF_PUSH_KEY }}
 
       - name: Set up Deno
-        uses: denoland/setup-deno@v2
+        uses: denoland/setup-deno@909cc5acb0fdd60627fb858598759246509fa755 # v2.0.2
         with:
           deno-version: v2.x
 

.gitignore

@@ -33,3 +33,7 @@ coverage.opencover.xml
 junit.xml
 
 appsettings.user.json
+
+# e2e tests
+/src/SIL.XForge.Scripture/ClientApp/e2e/secrets.json
+/src/SIL.XForge.Scripture/ClientApp/e2e/test_output/

deploy/dev-server.playbook.yml

@@ -120,19 +120,6 @@
         - "/var/lib/xforge/avatars"
         - "{{ lookup('env', 'HOME') }}/.local/share/SIL/WritingSystemRepository/3"
 
-    - name: Add localhost to dnsmasq
-      ansible.builtin.lineinfile:
-        path: /etc/NetworkManager/dnsmasq.d/localhost-domain
-        line: "{{ item }}"
-        state: present
-        create: true
-        mode: "u=rw,g=r,o=r"
-      with_items:
-        - "address=/localhost/127.0.0.1"
-        - "address=/localhost/::1"
-      when: inventory_hostname == "localhost" and base_distribution_release == 'xenial'
-      notify: restart network
-
     - name: Enable convenient access to ng from commandline | bin dir
       become: false
       ansible.builtin.file:
@@ -172,9 +159,3 @@
         src: InternetSettings.xml
         dest: ~/.local/share/Paratext95/
         mode: "0644"
-
-  handlers:
-    - name: Restart network
-      ansible.builtin.service:
-        name: network-manager
-        state: restarted

src/SIL.XForge.Scripture/ClientApp/e2e/README.md

@@ -0,0 +1,141 @@
+# Scripture Forge End-to-End Tests
+
+## Setup
+
+Install [Deno](https://deno.com/).
+
+Playwright may need some dependencies installed. For example, on Ubuntu:
+
+```bash
+sudo apt install libavif-bin
+```
+
+Install other dependencies:
+
+```bash
+cd src/SIL.XForge.Scripture/ClientApp/e2e
+deno install
+npx playwright install
+```
+
+Populate file `secrets.json` with secrets for testing.
+
+## Run
+
+```bash
+cd src/SIL.XForge.Scripture/ClientApp/e2e
+./e2e.mts
+```
+
+## Testing philosophy
+
+### The testing pyramid
+
+The greater focus on integration tests rather than E2E tests in this version of Scripture Forge came from this Google
+developer blog post: https://testing.googleblog.com/2015/04/just-say-no-to-more-end-to-end-tests.html
+
+The main point is to use unit tests as much as possible, use integration tests for what unit tests can't cover, and use
+E2E tests for what only E2E tests can cover. This is mainly because unit tests are faster, more reliable, and pinpoint
+the source of the problem more accurately.
+
+It's not that E2E tests are bad, but that they come with trade-offs that should be considered.
+
+### A pyramid approach to the E2E tests themselves
+
+While the article above focuses on different types of tests, the same principle can be applied to the E2E tests
+themselves. Once a test is written, it may be possible to run that test across multiple browsers, viewport sizes,
+localization languages, and depending on the test type, different user roles. Being able to test every possible
+permutation of these variables is extremely powerful, but it also takes much longer to run (and the probability of
+failure due to flakey tests increases).
+
+Rather than choosing to always have a ton of tests or only a few tests, we can scale the number of tests based on the
+situation. In general, we should run as many tests as possible without sacrificing efficiency. In general this means:
+
+1. Pull requests should run as many E2E tests as can be run without slowing down the process (i.e. they need to be
+   reliable and take no longer than the other checks that are run on pull requests)
+2. Pull requests that make major changes should have more tests run (this could be a manual step or somehow configured
+   in the CI by setting a tag on the pull request).
+3. Release candidates should run as many tests as possible.
+
+### Other goals
+
+Instrumentation for E2E tests can be used for more than automated testing. We can also use it to create screenshots for
+visual regression testing, and to keep screenshots in documentation up to date and localized. Some of this would incur
+additional effort to implement, but the instrumentation should be designed with this future in mind.
+
+## Implementation
+
+Playwright is being used for the E2E tests. It comes with both a library for driving the browser, and a test runner. For
+the most part, I have avoided using the test runner, opting instead to use the library directly. This gives a lot more
+flexibility in controlling what tests are run. The Playwright test runner is powerful, allowing for permutations of
+tests to be run, and multiple browsers run in parallel. However, there are also scenarios where more flexibility is
+needed, such as when running smoke tests for each user role. The admin user needs to be able to create the share links
+that are then used for invitations, having one test use the output of another.
+
+I opted to use Deno rather than Node for the E2E tests, though this comes with some drawbacks (see the "Working with
+Deno" section below).
+
+There are two types of tests that have been created so far:
+
+- Smoke tests: The tests log in as each user role, navigate to each page, and take a screenshot on each.
+- Workflow: A specific workflow that a user may perform is tested from start to finish.
+
+## Running tests
+
+A test plan is defined in `e2e-globals.ts` as a "Preset". It defines which locales to test, which browser engines,
+user roles, whether to take screenshots, etc. It also defines which categories of tests should be run (e.g. smoke tests,
+generating a draft, community checking). When the tests are executed, the run sheet should be followed to the degree
+that is possible. For example, the smoke tests should test only the user roles specified in the run sheet, but
+certain tests are specific to a given role (for example, you have to be an admin to set up a community checking
+project), and won't need to consider the specified roles.
+
+To run the tests, make any necessary edits to the run sheet, then run `e2e.mts`.
+
+Screenshots are saved in the `screenshots` directory, in a subfolder specified by the run sheet. The default subfolder
+name is the timestamp when the tests started.
+
+A file named `run_log.json` is saved to the directory with information about the test run and metadata regarding each of
+the screenshots.
+
+## Other notes
+
+### Working with Deno
+
+Unfortunately, I have not found a good way to make Deno play nicely with Node and Angular. In VS Code, I always run the
+`Deno: Enable` command when working with files that will be run by Deno, and then run `Deno: Disable` when switching to
+other TypeScript files. When Deno is disabled the language server complains about problems in the files intended to be
+run by Deno, and when Deno is enabled the language server complains about problems in the other files.
+
+Hopefully a better solution is available.
+
+### Making utility functions wait for completion
+
+A utility function that performs an action should also wait for for any side effects of the action to complete before
+returning. For example, a function that deletes the current project should wait until the user is redirected to the my
+projects page before returning. This can be done by waiting for the URL to change. This has two main benefits:
+
+1. Whatever action runs next does not need to wait for the side effects of the previous action to complete.
+2. When failures occur (such as if the redirect following the deletion doesn't happen), it's much easier to determine
+   where things went wrong, because the failure will occur in the function where the problem originated.
+
+### Recording tests
+
+In general it does not work well to just record a test with Playwright and then consider it a finished test. However, it
+can be much quicker to have Playwright record a test and then use that as a starting point. You can record a test by
+running `npx playwright codegen`, or by calling `await page.pause()` in a test, which stops execution and opens a second
+inspector window, which allows recording of tests, or using Playwright's locator tool.
+
+## Future plans
+
+Workflow tests that should be created:
+
+- Community checking
+- Editing, including simultaneous editing and change in network status
+- Serval admins
+- Site admins
+
+Other use-cases for the E2E tests:
+
+- Automated screenshot comparison
+- Localized screenshots for documentation
+- Help videos

src/SIL.XForge.Scripture/ClientApp/e2e/characterize-tests.mts

@@ -0,0 +1,71 @@
+#!/usr/bin/env -S deno run --allow-run --allow-env --allow-sys --allow-read --allow-write --unstable-sloppy-imports
+import { chromium } from "npm:playwright";
+import { preset } from "./e2e-globals.ts";
+import { Utils } from "./e2e-utils.ts";
+import { numberOfTimesToAttemptTest } from "./pass-probability.ts";
+import { ScreenshotContext } from "./presets.ts";
+import { tests } from "./test-definitions.ts";
+
+const resultFilePath = "test_characterization.json";
+const testNames = Object.keys(tests) as (keyof typeof tests)[];
+let mostRecentResultData = JSON.parse(await Deno.readTextFile(resultFilePath));
+
+printRetriesForEachTest();
+
+while (true) {
+  const testName = nextTestToRun();
+  const testFunction = tests[testName];
+  const browser = await chromium.launch({ headless: true });
+  const browserContext = await browser.newContext();
+  await browserContext.grantPermissions(["clipboard-read", "clipboard-write"]);
+  await browserContext.tracing.start({ screenshots: true, snapshots: true, sources: true });
+  const page = await browserContext.newPage();
+  const screenshotContext: ScreenshotContext = { engine: "chromium" };
+  try {
+    const attempts = numberOfTimesToAttemptTest(testName, mostRecentResultData);
+    console.log(`Running test: ${testName}, which currently has ${attempts} attempts`);
+
+    await testFunction(chromium, page, screenshotContext);
+    await saveResult("success", testName);
+  } catch (error: unknown) {
+    console.error(error);
+    await saveResult("failure", testName);
+    const tracePath = `${preset.outputDir}/characterization-trace-${testName}-${Utils.formatDate(new Date())}.zip`;
+    console.log(`Saving trace to ${tracePath}`);
+    browserContext.tracing.stop({ path: tracePath });
+  } finally {
+    await browserContext.close();
+    await browser.close();
+    printRetriesForEachTest();
+  }
+}
+
+async function saveResult(result: "success" | "failure", testName: string): Promise<void> {
+  mostRecentResultData = JSON.parse(await Deno.readTextFile(resultFilePath));
+  mostRecentResultData[testName] ??= {};
+  mostRecentResultData[testName][result] ??= 0;
+  mostRecentResultData[testName][result]++;
+  await Deno.writeTextFile(resultFilePath, JSON.stringify(mostRecentResultData, null, 2) + "\n");
+  console.log(
+    `%c✔ Test ${testName} finished with result: ${result}`,
+    `color: ${result === "success" ? "green" : "red"}`
+  );
+}
+
+function nextTestToRun(): keyof typeof tests {
+  return testNames
+    .slice()
+    .sort(
+      (a, b) =>
+        numberOfTimesToAttemptTest(b, mostRecentResultData) - numberOfTimesToAttemptTest(a, mostRecentResultData)
+    )[0];
+}
+
+function printRetriesForEachTest(): void {
+  const info: { [key: string]: number } = {};
+  for (const testName of testNames) {
+    info[testName] = numberOfTimesToAttemptTest(testName, mostRecentResultData);
+  }
+  console.log("Retries for each test:");
+  console.table(info);
+}

src/SIL.XForge.Scripture/ClientApp/e2e/components/navigation.ts

@@ -0,0 +1,16 @@
+import { Locator, Page } from 'npm:playwright';
+
+const locatorStrings = {
+  translate_overview: `a[href$="/translate"]`,
+  edit_review: `a[href*="/translate/"]`,
+  generate_draft: `a[href$="/draft-generation"]`,
+  manage_questions: `a[href$="/checking"]`,
+  questions_answers: `a[href*="/checking/"]`,
+  sync: `a[href$="/sync"]`,
+  users: `a[href$="/users"]`,
+  settings: `a[href$="/settings"]`
+};
+
+export function navLocator(page: Page, menuItem: keyof typeof locatorStrings): Locator {
+  return page.locator('app-navigation').locator(locatorStrings[menuItem]);
+}

src/SIL.XForge.Scripture/ClientApp/e2e/copy_help_site_screenshots.mts

@@ -0,0 +1,67 @@
+#!/usr/bin/env -S deno run --allow-read --allow-write
+
+import { ScreenshotContext } from "./presets.ts";
+
+const runLogDir = Deno.args[0];
+const helpRepo = Deno.args[1];
+
+if (runLogDir == null || helpRepo == null) {
+  console.error("Usage: ./copy_help_site_screenshots.mts <run_log_dir> <help_repo>");
+  Deno.exit(1);
+}
+
+const runLog = JSON.parse(await Deno.readTextFile(`${runLogDir}/run_log.json`));
+if (!(await Deno.stat(helpRepo)).isDirectory) {
+  console.error(`Help repo "${helpRepo}" is not a directory`);
+  Deno.exit(1);
+}
+
+const pageNameToFileName = {
+  localized_page_sign_up: "1786056439.png",
+  localized_auth0_sign_up_with_pt: "1624359167.png",
+  localized_pt_registry_login: "448045579.png",
+  localized_my_projects: "1783795116.png",
+  localized_sync: "1990846672.png",
+  sign_up_for_drafting: "Draft-Generation/sign_up_for_drafting.png",
+  configure_sources_button: "Draft-Generation/configure_sources_button.png",
+  configure_sources_draft_source: "Draft-Generation/configure_sources_draft_source.png",
+  configure_sources_draft_reference: "Draft-Generation/configure_sources_draft_reference.png",
+  configure_sources_confirm_languages: "Draft-Generation/configure_sources_confirm_languages.png",
+  generate_draft_button: "Draft-Generation/generate_draft_button.png",
+  generate_draft_confirm_sources: "Draft-Generation/generate_draft_confirm_sources.png",
+  generate_draft_select_books_to_draft: "Draft-Generation/generate_draft_select_books_to_draft.png",
+  generate_draft_select_books_to_train: "Draft-Generation/generate_draft_select_books_to_train.png",
+  generate_draft_summary: "Draft-Generation/generate_draft_summary.png"
+};
+
+const screenshotsWithNoDestination = new Set();
+
+function getDirForLocale(localeCode: string): string {
+  return localeCode === "en"
+    ? `${helpRepo}/docs`
+    : `${helpRepo}/i18n/${localeCode}/docusaurus-plugin-content-docs/current`;
+}
+
+for (const screenshot of runLog.screenshotEvents) {
+  const context: ScreenshotContext = screenshot.context;
+  const pageName = context.pageName;
+  const localeCode = context.locale;
+  if (pageName == null || localeCode == null) {
+    console.error("Screenshot context is missing pageName or locale:", JSON.stringify(context));
+    continue;
+  }
+
+  if (pageName in pageNameToFileName) {
+    const currentFile = `${runLogDir}/${screenshot.fileName}`;
+    const newFile = `${getDirForLocale(localeCode)}/${pageNameToFileName[pageName as keyof typeof pageNameToFileName]}`;
+    console.log(`Moving ${currentFile} to ${newFile}`);
+    await Deno.copyFile(currentFile, newFile);
+  } else {
+    screenshotsWithNoDestination.add(pageName);
+  }
+}
+
+if (screenshotsWithNoDestination.size > 0) {
+  console.error("Screenshots with no destination:", Array.from(screenshotsWithNoDestination).join(", "));
+  console.error("Please update the script to handle these screenshots.");
+}