WIP: ExTester Runner

.github/workflows/main.yml

@@ -6,7 +6,6 @@ on:
     paths:
       - "**"
       - "!docs/**"
-      - "!packages/extester-runner/**"
   pull_request:
     branches: [main, extester-runner] # TODO remove 'extester-runner' branch before merge
     paths:
@@ -20,7 +19,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  test:
+  main:
     name: ${{ matrix.version }}
     strategy:
       fail-fast: false
@@ -31,9 +30,6 @@ jobs:
       version: ${{ matrix.version }}
 
   runner:
-    name: extester-runner
-    strategy:
-      fail-fast: false
     uses: ./.github/workflows/template-runner.yaml
     with:
       version: max
@@ -42,19 +38,19 @@ jobs:
     if: always()
     runs-on: ubuntu-latest
     name: 🚦 Status Check
-    needs: [test, runner]
+    needs: [main, runner]
     steps:
       - name: ℹ️ Test Matrix Result
         run: |
-          echo Main Test result = ${{ needs.test.result }}
+          echo Main Test result = ${{ needs.main.result }}
           echo Runner Test result = ${{ needs.runner.result }}
       - name: ✅ Status Check - success
-        if: ${{ needs.test.result == 'success' }} && ${{ needs.runner.result == 'success' }}
+        if: ${{ needs.main.result == 'success' && needs.runner.result == 'success' }}
         run: |
           echo "All tests successfully completed!"
           exit 0
       - name: ❌ Status Check - failure
-        if: ${{ needs.test.result != 'success' }} || ${{ needs.runner.result != 'success' }}
+        if: ${{ needs.main.result != 'success' || needs.runner.result != 'success' }}
         run: |
           echo "Status Check failed!"
           exit 1

.github/workflows/runner.yaml

@@ -1,10 +1,6 @@
 name: 🏗️ Runner CI
 
 on:
-  push:
-    branches: [main, extester-runner] # TODO remove 'extester-runner' branch before merge
-    paths:
-      - packages/extester-runner/**
   pull_request:
     branches: [main, extester-runner] # TODO remove 'extester-runner' branch before merge
     paths:
@@ -17,9 +13,6 @@ concurrency:
 
 jobs:
   runner:
-    name: extester-runner
-    strategy:
-      fail-fast: false
     uses: ./.github/workflows/template-runner.yaml
     with:
       version: max

.vscode/launch.json

@@ -70,15 +70,15 @@
       "name": "Launch Test Extension",
       "type": "extensionHost",
       "request": "launch",
-      "args": ["--extensionDevelopmentPath=${workspaceFolder}/tests/test-project"],
+      "args": ["--disable-extensions","--extensionDevelopmentPath=${workspaceFolder}/tests/test-project"],
       "outFiles": ["${workspaceFolder}/tests/test-project/out/**/*.js"],
       "preLaunchTask": "npm: watch"
     },
     {
       "name": "Launch ExTester Runner",
       "type": "extensionHost",
       "request": "launch",
-      "args": ["--extensionDevelopmentPath=${workspaceFolder}/packages/extester-runner"],
+      "args": ["--disable-extensions","--extensionDevelopmentPath=${workspaceFolder}/packages/extester-runner"],
       "outFiles": ["${workspaceFolder}/packages/extester-runner/out/**/*.js"],
       "preLaunchTask": "npm: watch"
     }

packages/extester-runner/package.json

@@ -368,7 +368,7 @@
     "@types/babel__traverse": "^7.20.7",
     "@vscode/test-cli": "^0.0.10",
     "@vscode/test-electron": "^2.5.2",
-    "esbuild": "^0.25.2",
+    "esbuild": "^0.25.3",
     "npm-run-all": "^4.1.5",
     "vscode-extension-tester": "*"
   },

packages/extester-runner/src/extension.ts

@@ -51,7 +51,7 @@ export async function activate(context: vscode.ExtensionContext) {
 
 	// Register the tree views:
 	// test view
-	const treeDataProvider = new ExtesterTreeProvider(logger);
+	const treeDataProvider = await ExtesterTreeProvider.create(logger);
 	logger.debug('Registering test tree view.');
 	vscode.window.createTreeView('extesterView', {
 		treeDataProvider: treeDataProvider,

packages/extester-runner/src/providers/extesterTreeProvider.ts

@@ -47,10 +47,21 @@ export class ExtesterTreeProvider implements vscode.TreeDataProvider<TreeItem> {
 	 *
 	 * @param {Logger} logger - The logging utility for debugging and tracking file operations.
 	 */
-	constructor(logger: Logger) {
+	private constructor(logger: Logger) {
 		this.logger = logger;
 		this.logger.debug('Initial tests tree provider constructed.');
-		void this.refresh(); // necessary to load initial data
+	}
+
+	/**
+	 * Creates and initializes a new instance of ExtesterTreeProvider.
+	 *
+	 * @param {Logger} logger - The logging utility for debugging and tracking file operations.
+	 * @returns {Promise<ExtesterTreeProvider>} - A promise that resolves to the initialized provider.
+	 */
+	static async create(logger: Logger): Promise<ExtesterTreeProvider> {
+		const provider = new ExtesterTreeProvider(logger);
+		await provider.refresh();
+		return provider;
 	}
 
 	/**

packages/extester-runner/src/tasks/RunAllTask.ts

@@ -67,7 +67,8 @@ export class RunAllTestsTask extends TestRunner {
 		const testFileGlob = configuration.get<string>('testFileGlob') || '**/ui-test/**/*.test.ts';
 		logger.debug(`RunAllTask: Using glob pattern: ${testFileGlob}`);
 		const testFileGlobSegments = testFileGlob.split(/[\/\\]/).filter(Boolean);
-		const testFilePattern = testFileGlobSegments[testFileGlobSegments.length - 1].replace(/\.ts$/, '.js');
+		const testFilePattern = testFileGlobSegments.pop()!.replace(/\.ts$/, '.js');
+
 		logger.debug(`RunAllTask: Test file pattern: ${testFilePattern}`);
 
 		// Split paths into segments.
@@ -124,20 +125,20 @@ export class RunAllTestsTask extends TestRunner {
  * @returns If no common path exists or the input is empty, returns an empty string.
  */
 function getCommonPath(paths: string[]): string {
-	if (paths.length === 0) {
+	if (!paths.length) {
 		return '';
 	}
-
-	const splitPaths = paths.map((p) => p.split('/'));
-	const firstPath = splitPaths[0];
-	let commonParts: string[] = [];
-
-	for (let i = 0; i < firstPath.length; i++) {
-		if (splitPaths.every((parts) => parts[i] === firstPath[i])) {
-			commonParts.push(firstPath[i]);
+	// split on either / or \
+	const splitPaths = paths.map((p) => p.split(/[/\\]/));
+	const first = splitPaths[0];
+	let common: string[] = [];
+	for (let i = 0; i < first.length; i++) {
+		if (splitPaths.every((sp) => sp[i] === first[i])) {
+			common.push(first[i]);
 		} else {
 			break;
 		}
 	}
-	return commonParts.join('/');
+	// join back with the platform's own separator
+	return common.join(path.sep);
 }

packages/extester-runner/src/test/suite/extension.test.ts

@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License", destination); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import * as assert from 'assert';
+import * as vscode from 'vscode';
+import * as path from 'path';
+import * as fs from 'fs';
+import { describe, it, before, after } from 'mocha';
+
+const EXTENSION_ID = 'extester-runner';
+const EXTENSION_PUBLISHER = 'ExTester';
+const FULL_EXTENSION_ID = `${EXTENSION_PUBLISHER}.${EXTENSION_ID}`;
+
+describe('Extension Test Suite', () => {
+	const testWorkspace = path.resolve(__dirname, '../../resources/testing-workspace');
+
+	before(async () => {
+		// create test workspace directory if it doesn't exist
+		if (!fs.existsSync(testWorkspace)) {
+			fs.mkdirSync(testWorkspace, { recursive: true });
+		}
+	});
+
+	after(() => {
+		// clean up test workspace
+		if (fs.existsSync(testWorkspace)) {
+			fs.rmSync(testWorkspace, { recursive: true, force: true });
+		}
+	});
+
+	it('should be present', () => {
+		const extension = vscode.extensions.getExtension(FULL_EXTENSION_ID);
+		assert.ok(extension, 'Extension should be present');
+		assert.strictEqual(extension.id, FULL_EXTENSION_ID, 'Extension ID should match');
+		assert.strictEqual(extension.packageJSON.name, EXTENSION_ID, 'Package name should match');
+		assert.strictEqual(extension.packageJSON.publisher, EXTENSION_PUBLISHER, 'Publisher should match');
+	});
+
+	it('should activate', async () => {
+		const extension = vscode.extensions.getExtension(FULL_EXTENSION_ID);
+		assert.ok(extension);
+		await extension?.activate();
+		assert.strictEqual(extension?.isActive, true);
+	});
+
+	it('should register commands', async () => {
+		const extension = vscode.extensions.getExtension(FULL_EXTENSION_ID);
+		assert.ok(extension);
+		await extension?.activate();
+
+		const commands = await vscode.commands.getCommands(true);
+		const extensionCommands = commands.filter((cmd) => cmd.startsWith(`${EXTENSION_ID}.`));
+		assert.ok(extensionCommands.length > 0, 'No extension commands were registered');
+	});
+
+	it('should have accessible test workspace', () => {
+		assert.ok(fs.existsSync(testWorkspace), 'Test workspace directory should exist');
+		assert.ok(fs.statSync(testWorkspace).isDirectory(), 'Test workspace should be a directory');
+	});
+
+	it('should handle workspace trust', async () => {
+		const workspaceTrust = vscode.workspace.isTrusted;
+		assert.ok(workspaceTrust !== undefined, 'Workspace trust state should be defined');
+	});
+
+	it('should have correct activation events', () => {
+		const extension = vscode.extensions.getExtension(FULL_EXTENSION_ID);
+		assert.ok(extension);
+		const activationEvents = extension.packageJSON.activationEvents;
+		assert.ok(activationEvents.includes('onStartupFinished'), 'Extension should activate on startup');
+	});
+
+	it('should have correct engine requirements', () => {
+		const extension = vscode.extensions.getExtension(FULL_EXTENSION_ID);
+		assert.ok(extension);
+		const engines = extension.packageJSON.engines;
+		assert.ok(engines.vscode, 'VS Code engine requirement should be defined');
+		assert.strictEqual(engines.vscode, '^1.96.0', 'VS Code engine version should match');
+	});
+
+	it('should create output channel', async () => {
+		const extension = vscode.extensions.getExtension(FULL_EXTENSION_ID);
+		assert.ok(extension);
+		await extension?.activate();
+
+		// create and verify output channel
+		const outputChannel = vscode.window.createOutputChannel('ExTester Runner');
+		assert.ok(outputChannel, 'Output channel should be created');
+		assert.strictEqual(outputChannel.name, 'ExTester Runner', 'Output channel name should match');
+
+		// clean up
+		outputChannel.dispose();
+	});
+});

packages/extester-runner/src/test/suite/index.ts

@@ -40,7 +40,7 @@ export function run(): Promise<void> {
 				}
 			});
 		} catch (err) {
-			e(err);
+			e(err instanceof Error ? err : new Error(String(err)));
 		}
 	});
 }

packages/extester-runner/src/test/suite/parser.test.ts

@@ -59,9 +59,15 @@ class DummyLogger extends Logger {
 		super(dummyOutputChannel);
 	}
 	// Override the logging methods to no-op implementations.
-	debug(): void {}
-	info(): void {}
-	error(): void {}
+	debug(): void {
+		// empty debug method since we don't want to output logs during tests
+	}
+	info(): void {
+		// empty info method since we don't want to output logs during tests
+	}
+	error(): void {
+		// empty error method since we don't want to output logs during tests
+	}
 }
 
 // Create an instance of DummyLogger to use in tests.

packages/extester-runner/src/test/suite/tasks.test.ts

@@ -51,9 +51,15 @@ class DummyLogger extends Logger {
 		super(dummyOutputChannel);
 	}
 	// Override the logging methods to no-op implementations.
-	debug(): void {}
-	info(): void {}
-	error(): void {}
+	debug(): void {
+		// empty debug method since we don't want to output logs during tests
+	}
+	info(): void {
+		// empty info method since we don't want to output logs during tests
+	}
+	error(): void {
+		// empty error method since we don't want to output logs during tests
+	}
 }
 
 // Create an instance of DummyLogger to use in tests.

packages/extester-runner/src/ui-test/utils/testUtils.ts

@@ -60,6 +60,7 @@ export async function extensionIsActivated(displayName: string): Promise<boolean
 			return false;
 		}
 	} catch (err) {
+		console.error('Error checking extension activation status:', err);
 		return false;
 	}
 }