Meta_TestSuite_Utilities_Client.cs

Go to the documentation of this file.

using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System.Threading;
using System.Threading.Tasks;
 
using AnalyzeRe;
using AnalyzeRe.APIResourceView;
using AnalyzeRe.APITypes;
using AnalyzeRe.Layers;
using AnalyzeRe.LossSets;
using AnalyzeRe.Utilities;
using AnalyzeReTesting.Base;
using AnalyzeReTesting.Utilities;
 
#if MSTEST
using Microsoft.VisualStudio.TestTools.UnitTesting;
#elif NUNIT
using NUnit.Framework;
using TestClass = NUnit.Framework.TestFixtureAttribute;
using TestMethod = NUnit.Framework.TestAttribute;
using TestCategory = NUnit.Framework.CategoryAttribute;
#endif
 
namespace AnalyzeReTesting.Tests.Utilities
{
    [TestClass]
    public sealed class Meta_TestSuite_Utilities_Client : BaseClientTestSuite
    {
        private const string Category = "Test Utilities";
 
        [TestMethod, TestCategory(Category)]
        public void Test_API_CleanupMethod()
        {
            if (!EnvironmentSettings.ROOT_DELETE_SUPPORTED &&
                !EnvironmentSettings.RESOURCE_DELETE_SUPPORTED)
                Assert.Inconclusive("DELETE is disabled.");
            if (EnvironmentSettings.EXECUTING_IN_PARALLEL)
                Assert.Inconclusive("Cannot DELETE while tests running in parallel.");
 
            // Hack: If we're going to be calling a live server method,
            //       this should be in a BaseServerTest class.
            BaseServerTestSuite.ApplyConfiguredCredentials();
            Distribution _ = Samples.Distribution_BinomialDistribution.Posted;
            // Test
            CleanUp.ForceCleanServer();
            Assert.IsTrue(CleanUp.CheckServerIsClean(), "Server did not end up clean.");
        }
 
        [TestMethod, TestCategory(Category)]
        public void Test_DoublesAreEqual()
        {
            List<Tuple<bool, double, double>> tests = new List<Tuple<bool, double, double>>
            {
                // Test the behaviour when a difference exists at various significant figures.
                Tuple.Create(true,  1.00000000000001, 1.0), // Difference at 15th
                Tuple.Create(false, 1.0000000000001,  1.0), // Difference at 14th
                Tuple.Create(true,  10000000000000.1, 10000000000000.0), // 15th
                Tuple.Create(false, 1000000000000.1,  1000000000000.0), // 14th
                Tuple.Create(true,  0.123456789012345, 0.123456789012346), // 15th
                Tuple.Create(false, 0.12345678912345,  0.12345678912346), // 14th
                // Same test for very large doubles
                Tuple.Create(true,  0.123456789012345*1E100, 0.123456789012346*1E100),
                Tuple.Create(false, 0.12345678912345*1E100,  0.12345678912346*1E100),
                // Same test for very small doubles
                Tuple.Create(true,  0.123456789012345/1E100, 0.123456789012346/1E100),
                Tuple.Create(false, 0.12345678912345/1E100,  0.12345678912346/1E100),
                Tuple.Create(true, Double.MaxValue, Double.MaxValue - Math.PI),
                // Real world round trip examples that caused a problem
                Tuple.Create(true, 817393214.3568030, 817393214.35680345),
                Tuple.Create(true, 0.408297614850242462, 0.408297614850242018)
            };
            foreach (Tuple<bool, double, double> test in tests)
            {
                if (test.Item1)
                {
                    AssertApi.DoublesAreEqual(test.Item2, test.Item3);
                    AssertApi.DoublesAreEqual(test.Item3, test.Item2);
                }
                else
                {
                    Assert.IsFalse(AssertApi.AlmostEquals(test.Item2, test.Item3), "Expected " +
                        Output.AutoFormat(test.Item2) + " and " +
                        Output.AutoFormat(test.Item3) + " to be not equal.");
                }
            }
        }
 
        #region Test TypeResolver
        [TestMethod, TestCategory(Category)]
        public void Test_TypeResolver_ResolveBaseTypeByURL()
        {
            string testURL = "http://localhost:8000/layer_views/ae53378c-9a94-39f4-898a-4d888d43ae73";
            Assert.AreEqual(typeof(LayerView<ILayer>), TypeResolver.ResolveBaseTypeByURL(testURL));
            testURL = "https://dev-master-api.analyzere.net/portfolios/ae53378c-9a94-39f4-898a-4d888d43ae73/";
            Assert.AreEqual(typeof(Portfolio), TypeResolver.ResolveBaseTypeByURL(testURL));
            testURL = "https://host/api/loss_sets/dead-beef";
            Assert.AreEqual(typeof(LossSet), TypeResolver.ResolveBaseTypeByURL(testURL));
        }
 
        [TestMethod, TestCategory(Category)]
        public void Test_TypeResolver_GetCovariantBase_GenericTypes()
        {
            // Test References
            Assert.AreEqual(typeof(IReference<CatXL>),
                TypeResolver.GetGenericCovariantBase(typeof(Reference<CatXL>)));
            Assert.AreEqual(typeof(IReference<APIResource>),
                TypeResolver.GetGenericCovariantBase(typeof(Reference<APIResource>)));
            Assert.AreEqual(typeof(IReference<IAPIResource>),
                TypeResolver.GetGenericCovariantBase(typeof(Reference<IAPIResource>)));
            // Test LayerViews
            Assert.AreEqual(typeof(ILayerView<CatXL>),
                TypeResolver.GetGenericCovariantBase(typeof(LayerView<CatXL>)));
            Assert.AreEqual(typeof(ILayerView<ILayer>),
                TypeResolver.GetGenericCovariantBase(typeof(LayerView<>)));
        }
 
        [TestMethod, TestCategory(Category)]
        public void Test_TypeResolver_GetCovariantBase_AlreadyInterfaces()
        {
            // Test Generic Interface Types as input
            Assert.AreEqual(typeof(ILayerView<CatXL>),
                TypeResolver.GetGenericCovariantBase(typeof(ILayerView<CatXL>)));
            Assert.AreEqual(typeof(ILayerView<ILayer>),
                TypeResolver.GetGenericCovariantBase(typeof(ILayerView<ILayer>)));
            Assert.AreEqual(typeof(IReference<LayerView<Nested>>),
                TypeResolver.GetGenericCovariantBase(typeof(Reference<LayerView<Nested>>)));
        }
 
        [TestMethod, TestCategory(Category)]
        public void Test_TypeResolver_MakeGenericTypeFast()
        {
            ILayer[] layers =
            {
                new CatXL(), new AggXL(), new Generic(), new SurplusShare(),
                new QuotaShare(), new AggregateQuotaShare(), new IndustryLossWarranty(),
                new Nested(), new Filter()
            };
            Type genericContainer = typeof(LayerView<>);
 
            // Resolve all types once to ignore the effect of built in assembly caching
            Enumerable.Range(0, layers.Length).ToList().ForEach(i =>
                _ = genericContainer.MakeGenericType(layers[i].GetType()));
 
            const int samples = 1000000;
 
            // Test threaded access with caching
            DateTime start = DateTime.UtcNow;
            Enumerable.Range(0, samples).AsParallel().ForAll(i =>
            {
                Type t = layers[i % layers.Length].GetType();
                Type _ = genericContainer.MakeGenericTypeFast(t);
            });
            TimeSpan cachedReflectionElapsed = DateTime.UtcNow - start;
            Console.WriteLine(cachedReflectionElapsed.TotalSeconds + " s for caching.");
 
            // Test threaded access without added caching
            start = DateTime.UtcNow;
            Enumerable.Range(0, samples).AsParallel().ForAll(i =>
            {
                Type t = layers[i % layers.Length].GetType();
                Type _ = genericContainer.MakeGenericType(t);
            });
            TimeSpan reflectionElapsed = DateTime.UtcNow - start;
            Console.WriteLine(reflectionElapsed.TotalSeconds + " s for just Reflecting each time.");
 
            Assert.IsTrue(cachedReflectionElapsed < reflectionElapsed, "Caching is slower " +
                "than reflecting each time, so it should be removed.");
        }
        #endregion Test TypeResolver
 
        #region Test GetCollectionName
        [TestMethod, TestCategory(Category)]
        public void Test_API_GetCollectionName_BaseTypes()
        {
            Assert.AreEqual("layers", API.GetCollectionName(typeof(Layer),
                out Type declaringType));
            Assert.AreEqual(typeof(Layer), declaringType);
            Assert.AreEqual("portfolios", API.GetCollectionName(typeof(Portfolio),
                out declaringType));
            Assert.AreEqual(typeof(Portfolio), declaringType);
            Assert.AreEqual("loss_sets", API.GetCollectionName(typeof(LossSet),
                out declaringType));
            Assert.AreEqual(typeof(LossSet), declaringType);
        }
 
        [TestMethod, TestCategory(Category)]
        public void Test_API_GetCollectionName_DerivedTypes()
        {
            Assert.AreEqual("layers", API.GetCollectionName(typeof(CatXL),
                out Type declaringType));
            Assert.AreEqual(typeof(Layer), declaringType);
            Assert.AreEqual("portfolios", API.GetCollectionName(typeof(StaticPortfolio),
                out declaringType));
            Assert.AreEqual(typeof(Portfolio), declaringType);
            Assert.AreEqual("loss_sets", API.GetCollectionName(typeof(ELTLossSet),
                out declaringType));
            Assert.AreEqual(typeof(LossSet), declaringType);
        }
 
        [TestMethod, TestCategory(Category)]
        public void Test_API_GetCollectionName_Generic()
        {
            Assert.AreEqual("layer_views", API.GetCollectionName(typeof(LayerView<ILayer>),
                out Type declaringType));
            Assert.AreEqual(typeof(LayerView<ILayer>), declaringType);
            Assert.AreEqual("layer_views", API.GetCollectionName(typeof(LayerView<CatXL>),
                out declaringType));
            Assert.AreEqual(typeof(LayerView<ILayer>), declaringType);
        }
 
        [TestMethod, TestCategory(Category)]
        public void Test_API_GetCollectionName_Interfaces()
        {
            Assert.AreEqual("layers", API.GetCollectionName(typeof(ILayer),
                out Type declaringType));
            Assert.AreEqual(typeof(Layer), declaringType);
            Assert.AreEqual("layers", API.GetCollectionName(typeof(ILayer_WithTerms),
                out declaringType));
            Assert.AreEqual(typeof(Layer), declaringType);
        }
 
        [TestMethod, TestCategory(Category)]
        public void Test_API_GetCollectionName_GenericInterfaces()
        {
            Assert.AreEqual("layer_views", API.GetCollectionName(typeof(ILayerView<ILayer>),
                out Type declaringType));
            Assert.AreEqual(typeof(LayerView<ILayer>), declaringType);
            Assert.AreEqual("layer_views", API.GetCollectionName(typeof(ILayerView<CatXL>),
                out declaringType));
            Assert.AreEqual(typeof(LayerView<ILayer>), declaringType);
        }
 
        [TestMethod, TestCategory(Category)]
        public void Test_API_GetCollectionName_NoneAvailable()
        {
            // Setup a test helper for testing various invalid types
            void TestInvalidBaseType<T>() where T : IAPIResource
            {
                Type runtimeType = typeof(T);
 
                // Test returning null using the optional argument to avoid throwing
                Assert.IsNull(API.GetCollectionName(runtimeType, out Type declaring, true));
                Assert.IsNull(declaring);
 
                string expectedError = $"Invalid type {runtimeType.NiceTypeName()}. " +
                    "Type can be linked to more than one resource";
                Action<ArgumentException> exceptionTest = ex =>
                    Assert.IsTrue(ex.Message.StartsWith(expectedError), ex.Message);
                // Test throwing an error using the compile-time-type overloads
                AssertApi.ExceptionThrown(() => API.GetCollectionName<T>(), exceptionTest);
                AssertApi.ExceptionThrown(() => API.GetCollectionName<T>(out Type _), exceptionTest);
                // Test throwing an error using the runtime-type overloads
                AssertApi.ExceptionThrown(() => API.GetCollectionName(runtimeType), exceptionTest);
                AssertApi.ExceptionThrown(() => API.GetCollectionName(runtimeType, out Type _), exceptionTest);
            }
 
            TestInvalidBaseType<IAPIResource>();
            TestInvalidBaseType<APIResource>();
            TestInvalidBaseType<IAPIResourceView>();
            TestInvalidBaseType<BaseAPIResourceView>();
            TestInvalidBaseType<IAPIResource_WithDataEndpoint>();
            TestInvalidBaseType<IAPIResource_WithDataEndpoint>();
            TestInvalidBaseType<APIResource_WithDataEndpoint>();
            TestInvalidBaseType<ILayerSource>();
 
            // Check a types that doesn't meet boxing requirements against the runtime-type variant
            // Should still not throw when suppress-throw is set to true.
            Assert.IsNull(API.GetCollectionName(typeof(IAPIType), out Type shouldBeNull, true));
            Assert.IsNull(shouldBeNull);
            // Throws a different error message if suppress-throw is false
            AssertApi.ExceptionThrown(() => API.GetCollectionName(typeof(IAPIType)),
                (ArgumentException ex) => Assert.AreEqual(
                    "IAPIType does not derive from IAPIResource.", ex.Message));
        }
        #endregion Test GetCollectionName
 
        #region Test API.PollUntilReady
        private const int DefaultRetryTime = 10; //ms
        private const int DefaultQueuePosition = 1;
        public static int ThrowMockRetryAfter()
        {
            return ThrowMockRetryAfter(DefaultRetryTime / 1000d, DefaultQueuePosition);
        }
 
        public static int ThrowMockRetryAfter(double? retry_time, int? queue_position)
        {
            throw new APIRequestException("Retry", null, MockResponse.RetryAfter(retry_time, queue_position), null);
        }
 
        [TestMethod, TestCategory(Category)]
        public void Test_API_PollUntilReady()
        {
            const int returned = 5;
            int call_counter = 0;
            int TestFunction() => ++call_counter < returned ? ThrowMockRetryAfter() : call_counter;
            // If not supplied any PollingOptions as arguments, PollUntilReady should keep retrying.
            Assert.AreEqual(returned, API.PollUntilReady(TestFunction, PollingOptions.Default),
                "Function was not retried as many times as expected.");
        }
 
        [TestMethod, TestCategory(Category)]
        public void Test_API_PollUntilReady_NotWaiting()
        {
            const int returned = 5;
            int call_counter = 0;
            int TestFunction() => ++call_counter < returned ? ThrowMockRetryAfter() : call_counter;
            // A NotWaitingException is raised if the wait time is exceeded.
            AssertApi.ExceptionThrown(() => API.PollUntilReady(TestFunction,
                new PollingOptions(maxPollTotalTime: 0)),
                (NotWaitingException ex) =>
                {
                    Assert.AreEqual($"Simulation is queued at position {DefaultQueuePosition}. " +
                        $"Server suggests retrying after {DefaultRetryTime / 1000d} " +
                        $"seconds. Please try again later.", ex.Message);
                    Assert.IsNotNull(ex.RetryTime);
                    AssertApi.DoublesAreEqual(DefaultRetryTime / 1000d, ex.RetryTime.Value);
                });
 
            // The message includes the maximum wait time if it was supplied
            call_counter = 0;
            AssertApi.ExceptionThrown(
                () => API.PollUntilReady(TestFunction, new PollingOptions(
                    DefaultRetryTime, maxPollTotalTime: DefaultRetryTime / 10)),
                (NotWaitingException ex) =>
                {
                    Assert.AreEqual($"Simulation is queued at position {DefaultQueuePosition}. " +
                        $"Server suggests retrying after {DefaultRetryTime / 1000d} seconds. " +
                        $"The maximum wait time of {DefaultRetryTime / 10000d} seconds has been exceeded. " +
                        "Please try again later.", ex.Message);
                    Assert.IsNotNull(ex.RetryTime);
                    AssertApi.DoublesAreEqual(DefaultRetryTime / 1000d, ex.RetryTime.Value);
                });
        }
 
        [TestMethod, TestCategory(Category)]
        public void Test_API_PollUntilReady_TimeoutOptionsRespected()
        {
            DateTime start = DateTime.UtcNow;
            const int oneMillisecond = 1;
            const int timeToWait = 100 * oneMillisecond;
            const int retryTime = timeToWait * 8 / 10;
            const int minPollingTime = timeToWait * 1 / 10;
            const int tolerance = minPollingTime * 2 - oneMillisecond;
            try
            {
                API.PollUntilReady(() => ThrowMockRetryAfter(1, null),
                    new PollingOptions(minPollingTime, retryTime, timeToWait));
                Assert.Fail("Expected a NotWaitingException to be thrown.");
            }
            catch (NotWaitingException ex)
            {
                double elapsed = (DateTime.UtcNow - start).TotalMilliseconds / 1000;
                Console.WriteLine("PollUntilReady reports time waited was: " + ex.TimeWaited);
                Console.WriteLine("Time until control was returned to caller: " + elapsed);
                // The following assertions are millisecond time sensitive and do not work
                // well in a massive parallel or continuous integration testing environment,
                // so they are disabled but left to demonstrate the expected behaviour and
                // permit future debugging.
                if (ex.TimeWaited - elapsed <= tolerance)
                {
                    Console.WriteLine("Warning: PollUntilReady took " + (elapsed - ex.TimeWaited) *
                        1000 + " ms longer than desired to stop polling.");
                    //Assert.Fail("PollUntilReady took " + (elapsed - ex.TimeWaited) * 1000 +
                    //    " ms longer than desired to stop polling.");
                }
                // Assert that we didn't retry for longer than we should have
                if (ex.TimeWaited > timeToWait + tolerance)
                {
                    Console.WriteLine("Warning: Waited too long (" + ex.TimeWaited + "s >> " +
                        timeToWait + "s) to give up.");
                    //Assert.Fail("Waited too long (" + ex.TimeWaited + "s >> " +
                    //    time_to_wait + "s) to give up.");
                }
                // Assert that we didn't give up when we still had time to poll once more.
                if (ex.TimeWaited < timeToWait - minPollingTime)
                {
                    Console.WriteLine("Warning: Didn't wait long enough before giving up (" +
                        ex.TimeWaited + "s << " + timeToWait + "s - " + minPollingTime + "s).");
                    //Assert.Fail("Didn't wait long enough before giving up (" +
                    //    ex.TimeWaited + "s << " + time_to_wait + "s - " + min_polling_time + "s).");
                }
            }
        }
 
        [TestMethod, TestCategory(Category)]
        public void Test_API_PollUntilReady_QueuePositionReported()
        {
            const int mockQueuePosition = 5;
            try
            {
                API.PollUntilReady(() => ThrowMockRetryAfter(null, mockQueuePosition), new PollingOptions(maxPollTotalTime: 0));
                Assert.Fail("Expected a NotWaitingException to be thrown.");
            }
            catch (NotWaitingException ex)
            {
                Assert.AreEqual(mockQueuePosition, ex.QueuePosition, "Queue position not reported correctly.");
                Assert.AreEqual($"Simulation is queued at position {mockQueuePosition}. " +
                    "Please try again later.", ex.Message);
            }
        }
 
        [TestMethod, TestCategory(Category)]
        public void Test_API_PollUntilReady_CancellationRespected()
        {
            CancellationTokenSource canceler = new CancellationTokenSource();
            // Set up the cancellation token to trigger after roughly 10 retries.
            canceler.CancelAfter(DefaultRetryTime * 10);
            // Invoking cancellation should stop the polling with an OperationCanceledException
            AssertApi.ExceptionThrown<OperationCanceledException>(
                () => API.PollUntilReady(ThrowMockRetryAfter, new PollingOptions(
                    maxPollInterval: DefaultRetryTime, cancellationToken: canceler.Token)));
        }
        #endregion Test API.PollUntilReady
 
        #region Test AsyncRequestMonitor
        [TestMethod, TestCategory(Category)]
        public void Test_API_AsyncRequestMonitor()
        {
            const int returned = 5;
            int call_counter = 0;
            int TestFunction() => ++call_counter < returned ? ThrowMockRetryAfter() : call_counter;
 
            using (AsyncRequestMonitor<int> monitor = new AsyncRequestMonitor<int>(TestFunction,
                new PollingOptions(DefaultRetryTime, DefaultRetryTime)))
            {
                monitor.Start();
                int result = monitor.Wait();
                Assert.AreEqual(returned, result, "Function was not retried as many times as expected.");
                Assert.AreEqual(returned, monitor.Result);
            }
        }
 
        [TestMethod, TestCategory(Category)]
        public void Test_API_AsyncRequestMonitor_Cancel_Method()
        {
            // Assert that cancellation using the Stop() method works.
            using (AsyncRequestMonitor<int> monitor = new AsyncRequestMonitor<int>(ThrowMockRetryAfter,
                new PollingOptions(DefaultRetryTime, DefaultRetryTime)))
            {
                monitor.Start();
                monitor.Stop();
                AssertApi.ExceptionThrown(() => monitor.Wait(), (AggregateException ex) =>
                {
                    Assert.IsNotNull(ex.InnerException);
                    Assert.AreEqual(typeof(TaskCanceledException), ex.InnerException.GetType());
                });
                Assert.IsNotNull(monitor.ResultException);
                Assert.AreEqual(typeof(TaskCanceledException), monitor.ResultException.GetType());
            }
        }
 
        [TestMethod, TestCategory(Category)]
        public void Test_API_AsyncRequestMonitor_CancellationToken_Respected()
        {
            // Assert that cancellation using a user-supplied cancellation token works
            CancellationTokenSource canceler = new CancellationTokenSource();
            using (AsyncRequestMonitor<int> monitor = new AsyncRequestMonitor<int>(ThrowMockRetryAfter,
                new PollingOptions(DefaultRetryTime, DefaultRetryTime, cancellationToken: canceler.Token)))
            {
                monitor.Start();
                canceler.Cancel();
                AssertApi.ExceptionThrown(() => monitor.Wait(), (AggregateException ex) =>
                {
                    Assert.IsNotNull(ex.InnerException);
                    Assert.AreEqual(typeof(TaskCanceledException), ex.InnerException.GetType());
                });
                Assert.IsNotNull(monitor.ResultException);
                Assert.AreEqual(typeof(TaskCanceledException), monitor.ResultException.GetType());
            }
 
            // Assert that cancellation using the Stop() method works
            // when the user supplied their own cancellation token
            canceler = new CancellationTokenSource();
            using (AsyncRequestMonitor<int> monitor = new AsyncRequestMonitor<int>(ThrowMockRetryAfter,
                new PollingOptions(DefaultRetryTime, DefaultRetryTime, cancellationToken: canceler.Token)))
            {
                monitor.Start();
                monitor.Stop();
                AssertApi.ExceptionThrown(() => monitor.Wait(), (AggregateException ex) =>
                {
                    Assert.IsNotNull(ex.InnerException);
                    Assert.AreEqual(typeof(TaskCanceledException), ex.InnerException.GetType());
                });
                Assert.IsNotNull(monitor.ResultException);
                Assert.AreEqual(typeof(TaskCanceledException), monitor.ResultException.GetType());
            }
        }
        #endregion Test AsyncRequestMonitor
 
        #region Test Reflection
        [TestMethod, TestCategory(Category), TestCategory("Randomized")]
        public void Test_Reflection_RandomizedReference_ValidationDisabled()
        {
            Reflection reflection = new Reflection(Samples) { Validation_Enabled = false };
            DateTime start = DateTime.UtcNow;
            // Generating a random instance for each type with validation disabled
            // should not require any server traffic. All references should be invalid.
            foreach (Type subtype in Samples.GetAPIResourceTypes()
                .SelectMany(Reflection.GetAllInstantiableSubtypes))
            {
                // Generate a random instance of this type
                object test = reflection.CreateRandomizedInstance(subtype);
                if (!(test is IAPIResource testResource))
                {
                    Assert.Fail("CreateRandomizedInstance did not generate a valid " +
                        $"{subtype.NiceTypeName()} instance. Result was \"{test ?? "(null)"}\"");
                    continue;
                }
 
                Console.WriteLine($"{DateTime.UtcNow:mm:ss.ff} Generated a random " +
                    $"{subtype.NiceTypeName()} with runtime type {test.GetType().NiceTypeName()}");
                // Check that none of the reference properties are resolved, which confirms that
                // the reflection utility did not unnecessarily POST any new resources to reference.
                List<PropertyInfo> writableProperties = test.GetType()
                    .GetUserFacingProperties(true, true).ToList();
                foreach (PropertyInfo prop in writableProperties.Where(p => 
                    typeof(IReference<IAPIResource>).IsAssignableFrom(p.PropertyType)))
                {
                    IReference<IAPIResource> reference = (IReference<IAPIResource>)prop.GetValue(test);
                    Assert.IsFalse(reference?.resolved ?? false);
                    Console.WriteLine($"{DateTime.UtcNow:mm:ss.ff} Verified that " +
                        $"the reference property {prop.DeclaringType.NiceTypeName()}.{prop.Name} " +
                        "wasn't resolved.");
                }
                // Check that every writable property can be re-randomized without issue.
                foreach (PropertyInfo prop in writableProperties)
                {
                    Console.WriteLine($"{DateTime.UtcNow:mm:ss.ff} Randomizing the " +
                        $"{test.GetType().NiceTypeName()} property {prop.PropertyType.NiceTypeName()} " +
                        $"{prop.DeclaringType.NiceTypeName()}.{prop.Name}");
                    reflection.ChangePropertyValueRandomly(testResource, prop);
                }
            }
            // The entire test should have taken well under a second to run if there was truly
            // no server requests made as is intended when ValidationEnabled is set to false.
            // We see that it is common now for the test to spend > 2 seconds, especially with the parallel
            // testing (can be caused by the NUnit), therefore increasing timeout to 10 seconds.
            TimeSpan elapsed = DateTime.UtcNow - start;
            Assert.IsTrue(elapsed.TotalSeconds < 10, "Expected this test to take significantly " +
                "less than 10 seconds to run since it should be making no server requests. " +
                $"Actual time taken was {elapsed.TotalMilliseconds}ms.");
        }
        #endregion Test Reflection
    }
}