eng/tools/RepoTasks/BatchHelixWorkItems.cs - main - githubmirror/aspnetcore

eng/tools/RepoTasks/BatchHelixWorkItems.cs
228 строк10 KB
Matt Mitchell
Batch small Helix work items to reduce per-item overhead (-53% compute) (#66808)
19 июн 2026, 01:03
19 июн 2026, 01:03a097744
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Microsoft.Build.Framework;
using Microsoft.Build.Utilities;

namespace RepoTasks;

/// <summary>
/// Groups eligible Helix work items into batches to reduce per-item overhead.
/// Items marked with SkipHelixWorkItemBatching=true or that have pre-commands
/// are excluded from batching and passed through as-is.
/// Batched items use symlinks to assembly publish directories for fast payload creation.
/// </summary>
public class BatchHelixWorkItems : Microsoft.Build.Utilities.Task
{
    [Required]
    public ITaskItem[] WorkItems { get; set; }

    [Required]
    public int MaxBatchSize { get; set; }

    [Required]
    public string OutputDirectory { get; set; }

    [Required]
    public bool IsWindowsQueue { get; set; }

    /// <summary>
    /// Fallback work item timeout (HH:MM:SS) used when an item has no Timeout metadata.
    /// Batched items otherwise inherit the same timeout each individual work item was given.
    /// </summary>
    public string DefaultTimeout { get; set; } = "00:45:00";

    [Output]
    public ITaskItem[] BatchedWorkItems { get; set; }

    [Output]
    public ITaskItem[] UnbatchedWorkItems { get; set; }

    public override bool Execute()
    {
        var unbatched = new List<ITaskItem>();
        var batched = new List<ITaskItem>();
        var eligibleGroups = new Dictionary<string, List<ITaskItem>>(StringComparer.OrdinalIgnoreCase);
        var batchSize = MaxBatchSize > 0 ? MaxBatchSize : 20;
        var batchRoot = Path.Combine(OutputDirectory, "batched");
        Directory.CreateDirectory(batchRoot);

        foreach (var workItem in WorkItems ?? Array.Empty<ITaskItem>())
        {
            var payloadDirectory = workItem.GetMetadata("PayloadDirectory");
            var preCommands = workItem.GetMetadata("PreCommands");
            var skipBatching = workItem.GetMetadata("SkipHelixWorkItemBatching");
            var tfm = workItem.GetMetadata("TargetFrameworkMoniker");
            if (string.IsNullOrWhiteSpace(tfm))
            {
                var separatorIndex = workItem.ItemSpec.LastIndexOf("--", StringComparison.Ordinal);
                tfm = separatorIndex >= 0 ? workItem.ItemSpec.Substring(separatorIndex + 2) : string.Empty;
            }

            var shouldBatch = !string.IsNullOrWhiteSpace(tfm) &&
                string.IsNullOrWhiteSpace(preCommands) &&
                !string.Equals(skipBatching, "true", StringComparison.OrdinalIgnoreCase) &&
                Directory.Exists(payloadDirectory);

            if (!shouldBatch)
            {
                // Write a single-entry targets.txt for unbatched items so all items
                // use the same @targets.txt command format.
                if (Directory.Exists(payloadDirectory))
                {
                    var testAssembly = workItem.GetMetadata("TestAssembly");
                    File.WriteAllText(Path.Combine(payloadDirectory, "targets.txt"), testAssembly);
                }

                unbatched.Add(workItem);
                continue;
            }

            if (!eligibleGroups.TryGetValue(tfm, out var group))
            {
                group = new List<ITaskItem>();
                eligibleGroups.Add(tfm, group);
            }

            group.Add(workItem);
        }

        var batchNumber = 0;
        foreach (var group in eligibleGroups.OrderBy(pair => pair.Key, StringComparer.OrdinalIgnoreCase))
        {
            var orderedItems = group.Value.OrderBy(item => item.ItemSpec, StringComparer.OrdinalIgnoreCase).ToList();
            for (var index = 0; index < orderedItems.Count; index += batchSize)
            {
                batchNumber++;
                var chunk = orderedItems.Skip(index).Take(batchSize).ToList();
                var batchDirectory = Path.Combine(batchRoot, $"batch_{batchNumber}");
                if (Directory.Exists(batchDirectory))
                {
                    Directory.Delete(batchDirectory, recursive: true);
                }

                Directory.CreateDirectory(batchDirectory);
                var firstItem = chunk[0];

                // Copy shared content files (runtests.cmd/sh, NuGet.config, etc.) from the first
                // item's payload root into the batch directory. Only root-level files are copied,
                // not subdirectories (those are assembly-specific and handled via symlinks below).
                var firstPayload = firstItem.GetMetadata("PayloadDirectory");
                foreach (var file in Directory.GetFiles(firstPayload))
                {
                    var destPath = Path.Combine(batchDirectory, Path.GetFileName(file));
                    if (!File.Exists(destPath))
                    {
                        File.Copy(file, destPath);
                    }
                }

                // Create symbolic links from batch subdirectories to assembly publish directories.
                // This is orders of magnitude faster than copying files.
                var targets = new List<string>();
                foreach (var workItem in chunk)
                {
                    var testAssembly = workItem.GetMetadata("TestAssembly");
                    var assemblyName = Path.GetFileNameWithoutExtension(testAssembly);
                    var targetDirectory = Path.Combine(batchDirectory, assemblyName);
                    Directory.CreateSymbolicLink(targetDirectory, workItem.GetMetadata("PayloadDirectory"));

                    // Playwright tests should never be batched — they require special setup.
                    var installPlaywright = workItem.GetMetadata("InstallPlaywright");
                    if (string.Equals(installPlaywright, "true", StringComparison.OrdinalIgnoreCase))
                    {
                        Log.LogError("Work item '{0}' requires Playwright but is being batched. " +
                            "Set SkipHelixWorkItemBatching=true in the project to exclude it.", workItem.ItemSpec);
                        return false;
                    }

                    var relativeSeparator = IsWindowsQueue ? "\\" : "/";
                    targets.Add($"{assemblyName}{relativeSeparator}{testAssembly}");
                }

                File.WriteAllLines(Path.Combine(batchDirectory, "targets.txt"), targets);

                var batchedWorkItem = new TaskItem($"batch_{batchNumber}--{group.Key}");

                // Validate that all items in this batch share the same queue, architecture,
                // runtime, and quarantine settings. These must be consistent because the
                // batch produces a single command that applies to all assemblies.
                var runtimeVersion = firstItem.GetMetadata("RuntimeVersion");
                var queueName = firstItem.GetMetadata("QueueName");
                var arch = firstItem.GetMetadata("TestingArchitecture");
                var quarantined = firstItem.GetMetadata("RunQuarantined");

                foreach (var workItem in chunk)
                {
                    ValidateMetadataConsistency(workItem, firstItem, "RuntimeVersion");
                    ValidateMetadataConsistency(workItem, firstItem, "QueueName");
                    ValidateMetadataConsistency(workItem, firstItem, "TestingArchitecture");
                    ValidateMetadataConsistency(workItem, firstItem, "RunQuarantined");
                    ValidateMetadataConsistency(workItem, firstItem, "Timeout");
                }

                if (Log.HasLoggedErrors)
                {
                    return false;
                }

                // A batch inherits the same timeout each individual work item was given
                // (the repo's HelixTimeout, 45 minutes by default). Batches run in parallel
                // on the queue just like individual items did, and the per-test
                // --blame-hang-timeout (15 min) catches genuinely hung tests, so there is no
                // need to scale the work item timeout down for batched runs.
                var timeoutStr = firstItem.GetMetadata("Timeout");
                if (string.IsNullOrWhiteSpace(timeoutStr))
                {
                    timeoutStr = DefaultTimeout;
                }

                var script = IsWindowsQueue ? "call runtests.cmd" : "./runtests.sh";
                var command = $"{script} @targets.txt {runtimeVersion} {queueName} {arch} {quarantined} {timeoutStr} false";

                // Copy metadata from the first item, then override batch-specific values.
                foreach (System.Collections.DictionaryEntry metadataEntry in firstItem.CloneCustomMetadata())
                {
                    batchedWorkItem.SetMetadata((string)metadataEntry.Key, (string)metadataEntry.Value);
                }

                batchedWorkItem.SetMetadata("PayloadDirectory", batchDirectory);
                batchedWorkItem.SetMetadata("TestAssembly", "targets.txt");
                batchedWorkItem.SetMetadata("Timeout", timeoutStr);
                batchedWorkItem.SetMetadata("SkipHelixWorkItemBatching", "false");
                batchedWorkItem.SetMetadata("TargetFrameworkMoniker", group.Key);
                batchedWorkItem.SetMetadata("Command", command);

                Log.LogMessage(MessageImportance.High, "Created batched Helix work item {0} with {1} assemblies.", batchedWorkItem.ItemSpec, chunk.Count);
                batched.Add(batchedWorkItem);
            }
        }

        BatchedWorkItems = batched.ToArray();
        UnbatchedWorkItems = unbatched.ToArray();
        Log.LogMessage(
            MessageImportance.High,
            "Helix work item batching retained {0} unbatched items and created {1} batched items.",
            UnbatchedWorkItems.Length,
            BatchedWorkItems.Length);

        return !Log.HasLoggedErrors;
    }

    private void ValidateMetadataConsistency(ITaskItem item, ITaskItem reference, string metadataName)
    {
        var itemValue = item.GetMetadata(metadataName) ?? "";
        var refValue = reference.GetMetadata(metadataName) ?? "";
        if (!string.Equals(itemValue, refValue, StringComparison.OrdinalIgnoreCase))
        {
            Log.LogError(
                "Work item '{0}' has {1}='{2}' but batch expects '{3}' (from '{4}'). " +
                "All items in a batch must share the same {1}.",
                item.ItemSpec, metadataName, itemValue, refValue, reference.ItemSpec);
        }
    }
}