ci: Adding minimal supported editor specific tests and improving trigger jobs coverage (#3409)

This PR aims to add minimal supported editor (6000.0) specific tests so
we can differentiate between those and trunk tests as well as adding
more variations to PR trigger job.

Additionally it introduces _**run_quick_checks**_ job that was added as
a requirement for package_pack and project_pack jobs. It will run as a
first check (also on PR trigger) to catch any basic errors (PVP and
project standards) and ONLY IF this job will pass, the rest of PR
trigger job dependencies will be executed.
This way provides faster feedback loop for developers and allow to
minimize resource usage when unnecessary

Failing result may look like this

![image](https://github.com/user-attachments/assets/bc89acc7-1fad-4330-92df-407bef45c308)
Where we can see that many jobs failed but in reality we can see that
only PVP jobs failed and others were never executed and failed due to
their dependencies

## Backport
This is specific change to develop-2.0.0 branch but it's somewhat
reflected in
#3408

Author: michalChrobot

.yamato/_run-all.yml

@@ -2,14 +2,23 @@
 ---
 
 # DESCRIPTION--------------------------------------------------------------------------
-  # Those jobs group together related test as dependencies to allow to easily manage running a given set of tests.
-  # This enables efficient test execution for different validation scenarios
+# Those jobs group together related test as dependencies to allow to easily manage running a given set of tests.
+# This enables efficient test execution for different validation scenarios
 
 # QUALITY CONSIDERATIONS---------------------------------------------------------------
-  # It's important to ensure that all dependencies exist (this can be verified in Yamato) since a modification in parameters may result in a given job not being generated, and thus we will not be able to run such erroneous job.
+# It's important to ensure that all dependencies exist (this can be verified in Yamato) since a modification in parameters may result in a given job not being generated, and thus we will not be able to run such erroneous job.
 
 #-----------------------------------------------------------------------------------
 
+# This job runs the fastest checks (PVP and code standards)
+# This is mainly used to quickly validate the easy mistakes before for example running PR trigger jobs (see _triggers.yml)
+run_quick_checks:
+  name: Run Quick Initial Checks
+  dependencies:
+    - .yamato/package-pack.yml#package_pack_-_ngo_ubuntu
+    - .yamato/project-standards.yml#standards_ubuntu_testproject_trunk
+
+
 # Runs all package tests
 run_all_package_tests:
   name: Run All Package Tests
@@ -31,6 +40,14 @@ run_all_package_tests_trunk:
 {% endfor -%}
 {% endfor -%}
 
+# Runs all package tests on mimimum supported editor (6000.0 in case of NGOv2.X)
+run_all_package_tests_6000:
+  name: Run All Package Tests [6000.0]
+  dependencies:
+{% for platform in test_platforms.desktop -%}
+    - .yamato/package-tests.yml#package_test_-_ngo_6000.0_{{ platform.name }}
+{% endfor -%}
+
 
 # Runs all projects tests
 run_all_project_tests:
@@ -61,6 +78,18 @@ run_all_project_tests_trunk:
 {% endif -%}
 {% endfor -%}
 
+# Runs all projects tests on mimimum supported editor (6000.0 in case of NGOv2.X)
+run_all_project_tests_6000:
+  name: Run All Project Tests [6000.0]
+  dependencies:
+{% for project in projects.all -%}
+{% if project.has_tests == "true" -%}
+{% for platform in test_platforms.desktop -%}
+    - .yamato/project-tests.yml#test_{{ project.name }}_{{ platform.name }}_6000.0
+{% endfor -%}
+{% endif -%}
+{% endfor -%}
+
 
 # Runs all project standards check
 run_all_projects_standards:
@@ -104,6 +133,18 @@ run_all_webgl_builds_trunk:
 {% endfor -%}
 {% endfor -%}
 
+# Runs all WebGL builds on 6000.0 editor
+run_all_webgl_builds_6000:
+  name: Run All WebGl Build [6000.0]
+  dependencies:
+{% for project in projects.default -%}
+{% if platform.name != "mac" -%} # There is an error about "Light baking could not be started because no valid OpenCL device could be found". Tracked in MTT-11726
+{% for platform in test_platforms.desktop -%}
+    - .yamato/webgl-build.yml#webgl_build_{{ project.name }}_{{ platform.name }}_6000.0
+{% endfor -%}
+{% endif -%}
+{% endfor -%}
+
 
 # Runs all Desktop tests
 run_all_project_tests_desktop_standalone:
@@ -134,6 +175,18 @@ run_all_project_tests_desktop_standalone_trunk:
 {% endfor -%}
 {% endfor -%}
 
+# Runs all Desktop tests on mimimum supported editor (6000.0 in case of NGOv2.X)
+run_all_project_tests_desktop_standalone_6000:
+  name: Run All Standalone Tests - Desktop [6000.0]
+  dependencies:
+{% for project in projects.default -%}
+{% for platform in test_platforms.desktop -%}
+{% for backend in scripting_backends -%}
+    - .yamato/desktop-standalone-tests.yml#desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_6000.0
+{% endfor -%}
+{% endfor -%}
+{% endfor -%}
+
 # Runs all Mobile tests
 run_all_project_tests_mobile_standalone:
   name: Run All Standalone Tests - Mobile
@@ -159,6 +212,16 @@ run_all_project_tests_mobile_standalone_trunk:
 {% endfor -%}
 {% endfor -%}
 
+# Runs all Mobile tests on mimimum supported editor (6000.0 in case of NGOv2.X)
+run_all_project_tests_mobile_standalone_6000:
+  name: Run All Standalone Tests - Mobile [6000.0]
+  dependencies:
+{% for project in projects.default -%}
+{% for platform in test_platforms.mobile_test -%}
+    - .yamato/mobile-standalone-test.yml#mobile_standalone_test_{{ project.name }}_{{ platform.name }}_6000.0
+{% endfor -%}
+{% endfor -%}
+
 
 # Runs all Console tests
 run_all_project_tests_console_standalone:
@@ -184,3 +247,13 @@ run_all_project_tests_console_standalone_trunk:
 {% endfor -%}
 {% endfor -%}
 {% endfor -%}
+
+# Runs all Console tests on mimimum supported editor (6000.0 in case of NGOv2.X)
+run_all_project_tests_console_standalone_6000:
+  name: Run All Standalone Tests - Console [6000.0]
+  dependencies:
+{% for project in projects.default -%}
+{% for platform in test_platforms.console_test -%}
+    - .yamato/console-standalone-test.yml#console_standalone_test_{{ project.name }}_{{ platform.name }}_6000.0
+{% endfor -%}
+{% endfor -%}

.yamato/_triggers.yml

@@ -2,55 +2,61 @@
 ---
 
 # DESCRIPTION--------------------------------------------------------------------------
-  # This configuration defines three main CI trigger patterns:
-    # 1. Pull Request Validation: Validation performed on PR basis
-    # 2. Nightly Development: Test set run nightly (validates most important test cases)
-    # 3. Weekly Full Validation: Test set run weekly (validates all test cases to prevent any suprises)
-  # Each pattern represents different balance between validation depth, execution time and CI resource usage
+# This configuration defines three main CI trigger patterns:
+# 1. Pull Request Validation: Validation performed on PR basis
+# 2. Nightly Development: Test set run nightly (validates most important test cases)
+# 3. Weekly Full Validation: Test set run weekly (validates all test cases to prevent any surprises)
+# Each pattern represents different balance between validation depth, execution time and CI resource usage
 
 # TRIGGER PATTERNS-------------------------------------------------------------------
-  # Pull Request:
-    # This test validaes Standards, Package tests, Project tests and Desktop standalone tests to ensure that main platforms are covered
-    # Triggers on PRs to develop, develop-2.0.0, and release branches
-    # Focuses on critical validation paths that we should validate before merging PRs
-    # Cancels previous runs on new commits
-    # Excludes draft PRs
+# Pull Request:
+# This test validates Standards, Package tests, Project tests and Desktop standalone tests to ensure that main platforms are covered
+# Triggers on PRs to develop, develop, and release branches
+# Focuses on critical validation paths that we should validate before merging PRs
+# Cancels previous runs on new commits
+# Excludes draft PRs
 
-  # Nightly:
-    # This test validaes same subset as pull_request_trigger with addition of mobile/console tests and webgl builds
-    # Runs daily on develop-2.0.0 (local configuration)
-    # Includes all test types but only on trunk. TODO: Add validation for minimum supported editor in nightly
-    # Adds platform-specific and APV validation
+# Nightly:
+# This test validates same subset as pull_request_trigger with addition of mobile/console tests and webgl builds
+# Runs daily on develop (local configuration)
+# Includes all test types but only on trunk.
+# Adds platform-specific and APV validation
 
-  # Weekly:
-  # This test validaes same subset as develop_nightly but runs per all supported editors as well as executes code coverage test and runs project standards per project
-    # Runs across all supported editor versions
-    # Includes code coverage analysis
-    # Validates all projects and standards
+# Weekly:
+# This test validates same subset as develop_nightly but runs per all supported editors as well as executes code coverage test and runs project standards per project
+# Runs across all supported editor versions
+# Includes code coverage analysis
+# Validates all projects and standards
 
 # CONFIGURATION STRUCTURE--------------------------------------------------------------
-  # Jobs configurations are generated by ensuring that all dependencies are successful.
-  # The dependencies are taken from _run-all.yml file where we can gather multiple tests into proper sets
+# Jobs configurations are generated by ensuring that all dependencies are successful.
+# The dependencies are taken from _run-all.yml file where we can gather multiple tests into proper sets
 
 # QUALITY CONSIDERATIONS---------------------------------------------------------------
 # It's important to ensure that all dependencies exist (this can be verified in Yamato) since a modification in parameters may result in a given job not being generated, and thus we will not be able to run such erroneous job.
 
 
-
 #-----------------------------------------------------------------------------------
 
 # Run all relevant tasks when a pull request targeting the develop or release branch is created or updated.
+# In order to have better coverage we run desktop standalone tests with different configurations which allows to mostly cover for different platforms, scripting backends and editor versions.
+# This job will FIRST run "run_quick_checks" jobs (defined in _run-all.yml) since it's the dependency of project pack jobs which is on the lowest dependency tier. This runs the fastest checks (like PVP or code standards) and ONLY IF those pass it will run the rest of the tests.
+# This optimization allows to speed up feedback look for any "obvious" errors and save resources.
+# Since standards job is a part of initial checks it's not present as direct dependency here
 pull_request_trigger:
   name: Pull Request Trigger (develop, develop-2.0.0, & release branches)
   dependencies:
-    # Run project standards to verify package/default project
-    - .yamato/project-standards.yml#standards_ubuntu_testproject_trunk
-    # Run package EditMode and Playmode tests on desktop platforms on trunk
+    # Run package EditMode and Playmode package tests on trunk
     - .yamato/_run-all.yml#run_all_package_tests_trunk
-    # Run project EditMode and PLaymode tests on desktop platforms on trunk 
+    # Run package EditMode and Playmode package tests on minimum supported editor (6000.0 in case of NGOv2.X)
+    - .yamato/_run-all.yml#run_all_package_tests_6000
+    # Run project EditMode and PLaymode project tests on trunk 
     - .yamato/_run-all.yml#run_all_project_tests_trunk
-    # Run one standalone test to make sure there are no obvious issues with most common platform (for example --fail-on-assert option is not present with package/project tests)
-    - .yamato/desktop-standalone-tests.yml#desktop_standalone_test_testproject_win_il2cpp_trunk
+    # Run project EditMode and PLaymode project tests on minimum supported editor (6000.0 in case of NGOv2.X)
+    - .yamato/_run-all.yml#run_all_project_tests_6000
+    # Run standalone test with mixture of parameters to make sure there are no obvious issues with most common platform (for example --fail-on-assert option is not present with package/project tests). We run 2 different combinations with different platform/editor/scripting backend.
+    - .yamato/desktop-standalone-tests.yml#desktop_standalone_test_testproject_mac_il2cpp_trunk
+    - .yamato/desktop-standalone-tests.yml#desktop_standalone_test_testproject_win_mono_6000.0
   triggers:
     cancel_old_ci: true
     pull_requests:
@@ -60,13 +66,13 @@ pull_request_trigger:
             - "develop-2.0.0"
             - "/release\/.*/"
       - drafts: false
-            
+
 
 # Run all tests on trunk on nightly basis.
 # Same subset as pull_request_trigger with addition of mobile/desktop/console tests and webgl builds
 # Those tests are all running on trunk editor (since it's daily and running all of them would add a lot of overhead)
 develop_nightly:
-  name: "\U0001F319 [Nightly] Run All Tests [Trunk]"
+  name: "\U0001F319 [Nightly] Run All Tests [Trunk and 6000]"
   triggers:
     recurring:
       - branch: develop-2.0.0
@@ -75,22 +81,30 @@ develop_nightly:
   dependencies:
     # Run project standards to verify package/default project
     - .yamato/project-standards.yml#standards_ubuntu_testproject_trunk
+    - .yamato/project-standards.yml#standards_ubuntu_testproject_6000.0
     # Run APV jobs to make sure the change won't break any dependants
     - .yamato/wrench/preview-a-p-v.yml#all_preview_apv_jobs
-    # Run package EditMode and Playmode tests on desktop platforms on trunk
+    # Run package EditMode and Playmode tests on desktop platforms on trunk and 6000.0
     - .yamato/_run-all.yml#run_all_package_tests_trunk
-    # Run project EditMode and PLaymode tests on desktop platforms on trunk 
+    - .yamato/_run-all.yml#run_all_package_tests_6000
+    # Run project EditMode and PLaymode tests on desktop platforms on trunk and 6000.0
     - .yamato/_run-all.yml#run_all_project_tests_trunk
-    # Run Runtime tests on desktop players on trunk
+    - .yamato/_run-all.yml#run_all_project_tests_6000
+    # Run Runtime tests on desktop players on trunk and 6000.0 editors
     - .yamato/_run-all.yml#run_all_project_tests_desktop_standalone_trunk
-    # Run Runtime tests on mobile players on trunk
+    - .yamato/_run-all.yml#run_all_project_tests_desktop_standalone_6000
+    # Run Runtime tests on mobile players on trunk and 6000.0 editors
     - .yamato/_run-all.yml#run_all_project_tests_mobile_standalone_trunk
-    # Run Runtime tests on console players on trunk
+    - .yamato/_run-all.yml#run_all_project_tests_mobile_standalone_6000
+    # Run Runtime tests on console players on trunk and 6000.0 editors
     - .yamato/_run-all.yml#run_all_project_tests_console_standalone_trunk
-    # Build player for webgl platform on trunk
+    - .yamato/_run-all.yml#run_all_project_tests_console_standalone_6000
+    # Build player for webgl platform on trunk and 6000.0 editors
     - .yamato/_run-all.yml#run_all_webgl_builds_trunk
-    # Build player for webgl platform on trunk
+    - .yamato/_run-all.yml#run_all_webgl_builds_6000
+    # Build player for webgl platform on trunk and 6000.0 editors
     - .yamato/project-updated-dependencies-test.yml#updated-dependencies_testproject_NGO_ubuntu_trunk
+    - .yamato/project-updated-dependencies-test.yml#updated-dependencies_testproject_NGO_win_6000.0
 
 
 # Run all tests on weekly bases

.yamato/package-tests.yml

@@ -54,6 +54,7 @@ package_test_-_ngo_{{ editor }}_{{ platform.name }}:
         - "upm-ci~/test-results/**/*"
         - "pvp-results/*"
   dependencies:
+    - .yamato/_run-all.yml#run_quick_checks # initial checks to perform fast validation of common errors
     - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }}
 {% endfor -%}
 {% endfor -%}

.yamato/project-pack.yml

@@ -35,6 +35,8 @@ project_pack_-_{{ project.name }}_{{ platform.name }}:
   commands:
     - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm # upm-ci is not preinstalled on the image so we need to download it
     - upm-ci project pack --project-path {{ project.path }}
+  dependencies:
+    - .yamato/_run-all.yml#run_quick_checks # initial checks to perform fast validation of common errors
   artifacts:
     packages:
       paths: