How to order xUnit tests belonging to one test collection but spread across multiple test classes?

Question

I have some test methods which are spread across multiple test classes but belonging to single test collection. I am using ITestCaseOrderer provided by xUnit but it is ordering only test methods within individual test classes.

[AttributeUsage(AttributeTargets.Method)]
public class TestPriorityAttribute : Attribute
{
    public TestPriorityAttribute(int priority)
    {
        this.Priority = priority;
    }

    public int Priority { get; }
}

I have implemented my priority orderer in the below fashion.

public class PriorityOrderer : ITestCaseOrderer
{
    public IEnumerable<TTestCase> OrderTestCases<TTestCase>(IEnumerable<TTestCase> testCases) where TTestCase : ITestCase
    {
        var sortedMethods = new Dictionary<int, TTestCase>();

        foreach (var testCase in testCases)
        {
            var attributeInfo = testCase.TestMethod.Method.GetCustomAttributes(typeof(TestPriorityAttribute).AssemblyQualifiedName)
                .SingleOrDefault();
            if (attributeInfo != null)
            {
                var priority = attributeInfo.GetNamedArgument<int>("Priority");
                sortedMethods.Add(priority, testCase);
            }
        }

        return sortedMethods.OrderBy(x => x.Key).Select(x => x.Value);
    }
}

My First test class looks like this.

[TestCaseOrderer("Integration.Tests.PriorityOrderer", "CompanyName.ProjectName.Integration.Tests")]
[Collection("StandardIntegrationTests")]
[Trait("Category", "Integration")]
public class StandardControllerTests1
{
    public StandardControllerTests1(StandardIntegrationTestFixture standardIntegrationTestFixture)
    {

    }

    [Fact, TestPriority(1)]
    public void TestMethod1()
    {
    }

    [Fact, TestPriority(2)]
    public void TestMethod2()
    {
    }
}

My Second test class looks like this

[TestCaseOrderer("Integration.Tests.PriorityOrderer", "CompanyName.ProjectName.Integration.Tests")]
[Collection("StandardIntegrationTests")]
[Trait("Category", "Integration")]
public class StandardControllerTests2
{
    public StandardControllerTests2(StandardIntegrationTestFixture standardIntegrationTestFixture)
    {

    }

    [Fact, TestPriority(3)]
    public void TestMethod3()
    {
    }

    [Fact, TestPriority(4)]
    public void TestMethod4()
    {
    }
}

I have other test classes also which belong to same test collection. When I run the tests, It is not ordering across the collection. How do I order these tests to run in order which are in same collection?

Answer 1

I have run into this same issue while building a ordering system to run tests in parallel with xUnit while also running some with dependencies within their own collections.

According to a Issue 898 on the xUnit board ordering across classes in the same collection is not possible at this time.

The work around I used was to organize the tests which needed sequenced into the same class with different files using partial class so that the tests could still be kept organized into different files.

[Trait("Order", "")]   
[TestCaseOrderer(DependencyOrderer.TypeName, DependencyOrderer.AssemblyName)]
public partial class OrderTests
{
    [Fact]        
    public void Test0()
    {
        Thread.Sleep(TimeSpan.FromSeconds(1));
    }

    [Fact]
    [TestDependency("Test1", "Test0")]
    public void Test3()
    {
        Thread.Sleep(TimeSpan.FromSeconds(1));
    }
 }

[Trait("Order", "")]
public partial class OrderTests
{
    [Fact]
    public void Test2()
    {
        Thread.Sleep(TimeSpan.FromSeconds(1));            
    }

    [Fact]
    [TestDependency("Test0")]
    public void Test1()
    {
        Thread.Sleep(TimeSpan.FromSeconds(1));
    }
}