Meta_TestSuite_Utilities_Server.cs

Go to the documentation of this file.

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Net;
using System.Reflection;
using System.Threading;
using System.Threading.Tasks;
 
using AnalyzeRe;
using AnalyzeRe.APITypes;
using AnalyzeRe.Attributes;
using AnalyzeRe.Utilities;
using AnalyzeReTesting.Base;
using AnalyzeReTesting.Utilities;
 
#if MSTEST
using Microsoft.VisualStudio.TestTools.UnitTesting;
#elif NUNIT
using NUnit.Framework;
using TestClass = NUnit.Framework.TestFixtureAttribute;
using TestMethod = NUnit.Framework.TestAttribute;
using TestCategory = NUnit.Framework.CategoryAttribute;
using AssertFailedException = NUnit.Framework.AssertionException;
#endif
 
namespace AnalyzeReTesting.Tests.Utilities
{
    [TestClass]
    public class Meta_TestSuite_Utilities_Server : BaseServerTestSuite
    {
        private const string Category = "Test Utilities";
 
        #region Test Sample Injection
        [TestMethod, TestCategory(Category), TestCategory("Randomized")]
        public void Test_Concurrent_Resource_Injection()
        {
            // Assert that tests will not fail if multiple threads are simultaneously
            // trying to use the same injected samples.
            IEnumerable<PropertyInfo> toInject =
                typeof(Samples).GetPublicProperties_Fast().Where(p =>
                    typeof(IInjectableResource<IAPIResource>).IsAssignableFrom(p.PropertyType))
                    .OrderBy(random => Guid.NewGuid());
            ConcurrentBag<Exception> thread_failures = new ConcurrentBag<Exception>();
            ParallelLoopResult result = Parallel.ForEach(toInject, property =>
            {
                // Do not attempt to POST resources that are not saveable.
                if (property.PropertyType.GetGenericArguments()[0]
                    .IsAttributeDefinedFast<NotSaveableAttribute>())
                    return;
                // Overkill, spin off 2 additional threads to do the same action
                new Thread(() => TestPostAction(property, thread_failures)).Start();
                TestPostAction(property, thread_failures);
                new Thread(() => TestPostAction(property, thread_failures)).Start();
            });
            while (!result.IsCompleted)
                Thread.Sleep(50);
            if (thread_failures.Any())
                Assert.Fail(thread_failures.Count + " assertions failed during " +
                    "concurrent resource injection:\n" + String.Join("\n", thread_failures));
        }
 
        private static void TestPostAction(PropertyInfo property, ConcurrentBag<Exception> errors)
        {
            try
            {
                Debug.WriteLine("Sample Resource \"" + property.Name +
                    "\" (type " + property.PropertyType.NiceTypeName() + ") concurrent request received.");
                IInjectableResource<IAPIResource> resource =
                    (IInjectableResource<IAPIResource>)property.GetValue(Samples, null);
                Debug.WriteLine("Sample Resource \"" + property.Name +
                    "\" (type " + property.PropertyType.NiceTypeName() + ") acquired. Invoking Post.");
                IAPIResource posted = resource.Posted;
                Debug.WriteLine("Sample Resource \"" + property.Name +
                    "\" (type " + property.PropertyType.NiceTypeName() + ") Post Complete.");
                AssertApi.IsValidUUID(posted.id);
            }
            catch (Exception ex)
            {
                errors.Add(ex);
            }
        }
 
        [TestMethod, TestCategory(Category), TestCategory("Randomized")]
        public void Test_Reflection_RandomizedReference_ValidationEnabled()
        {
            // Generate a random instance of each type, and re-randomize each property
            IEnumerable<Type> tests = Samples.GetAPIResourceTypes().SelectMany(
                Reflection.GetAllInstantiableSubtypes);
            Console.WriteLine("Note: Configured Target Analysis Profile is: " +
                Reflection.Target_AnalysisProfile.id);
            Console.WriteLine("      Configured Target Event Catalog is:    " +
                Reflection.Target_AnalysisProfile.event_catalogs[0].ref_id);
            // Toggle the below constant to simplify test debugging.
            const bool runInParallel = true;
            // ReSharper disable once ConditionIsAlwaysTrueOrFalse
#pragma warning disable 162
            if (runInParallel)
                tests.AsParallel().ForAll(RandomTypeTest);
            else
                tests.ToList().ForEach(RandomTypeTest);
#pragma warning restore 162
        }
 
        // Helper method for the above run on each subtype discovered via reflection.
        private void RandomTypeTest(Type subtype)
        {
            // Generate a random instance of this type
            object test = Reflection.CreateRandomizedInstance(subtype);
            if (!(test is IAPIResource testResource))
            {
                Assert.Fail("CreateRandomizedInstance did not generate a valid " +
                    $"{subtype.NiceTypeName()} instance. Result was \"{test ?? "(null)"}\"");
                return;
            }
 
            string threadId = $"T{Thread.CurrentThread.ManagedThreadId} ";
            Console.WriteLine($"{threadId}{DateTime.UtcNow:mm:ss.ff} Generated a random " +
                $"{subtype.NiceTypeName()} with runtime type {test.GetType().NiceTypeName()}");
            // POST should succeed, because Validation was enabled.
            IAPIResource posted = null;
            try
            {
                posted = testResource.Post();
            }
            catch (Exception ex)
            {
                string issue = ex is APIRequestException apiError &&
                    apiError.RestResponse.StatusCode == HttpStatusCode.BadRequest ?
                        $"generated an invalid {subtype.NiceTypeName()} resulting in a rejected POST" :
                        $"encountered an unexpected POST error for a random {subtype.NiceTypeName()}";
                string msg = $"{threadId}{DateTime.UtcNow:mm:ss.ff} CreateRandomizedInstance {issue}." +
                    $"\nThe posted JSON was:\n{testResource.Serialize()}" +
                    $"\nThe response error was:\n{ex}";
                Console.WriteLine(msg);
                // The Assert.Fail throw has been observed ending the Task before logs are flushed
                Console.Out.Flush();
                Assert.Fail(msg);
            }
 
            // Give this resource data if necessary.
            posted = Reflection.FinalizePostedResource(posted);
 
            // Check that every writable property can be re-randomized without issue.
            List<PropertyInfo> writableProperties = test.GetType()
                .GetUserFacingProperties(true, true).ToList();
            // Bonus: Let's make sure each change to the resource results in a new id.
            HashSet<string> resourceIds = new HashSet<string> { posted.id };
            foreach (PropertyInfo prop in writableProperties)
            {
                object originalValue = prop.GetValue(testResource);
                string strProperty = $"the {test.GetType().NiceTypeName()} property " +
                    $"{prop.PropertyType.NiceTypeName()} {prop.DeclaringType.NiceTypeName()}.{prop.Name}";
                Console.WriteLine($"{threadId}{DateTime.UtcNow:mm:ss.ff} Randomizing {strProperty}");
                IAPIResource modified = testResource.DeepCopy().Change(r => r.id, null);
                object newValue;
                try { newValue = Reflection.ChangePropertyValueRandomly(modified, prop); }
                catch (PropertyGenerationException ex)
                {
                    Console.WriteLine($"{threadId}{DateTime.UtcNow:mm:ss.ff} RandomizePropertyValue " +
                        $"reported an error generating {strProperty}:\n{ex.Message}");
                    // TODO: Handle all cases where this throws because some properties aren't
                    //       randomized with validation turned on.
                    //throw;
                    continue;
                }
 
                string DebugValues(object v1, object v2 = null) =>
                    $"\n    Original: {Output.AutoFormat(v1)}" +
                    $"\n    Modified: {Output.AutoFormat(v2)}";
 
                // Ensure the new random value was valid (i.e. POST still succeeds)
                try { modified = modified.Post(); }
                catch (Exception ex)
                {
                    string issue = ex is APIRequestException apiError &&
                        apiError.RestResponse.StatusCode == HttpStatusCode.BadRequest ?
                            $"generated an invalid value for {strProperty} resulting in a rejected POST" :
                            $"encountered an unexpected POST error after randomly modifying {strProperty}";
                    string msg = $"{threadId}{DateTime.UtcNow:mm:ss.ff} RandomizePropertyValue {issue}." +
                        $"Property values are:{DebugValues(originalValue, newValue)}\n\n" +
                        $"Entire resource definitions:{DebugValues(testResource, modified)}\n";
                    Console.WriteLine($"{msg}{ex.Message}");
                    // The Assert.Fail throw has been observed ending the Task before logs are flushed
                    Console.Out.Flush();
                    Assert.Fail($"{msg}{ex}");
                }
 
                // Ensure the property value was changed
                if (newValue?.Equals(originalValue) ?? originalValue == null)
                    Console.WriteLine($"{threadId}{DateTime.UtcNow:mm:ss.ff} RandomizePropertyValue " +
                        $"failed to change {strProperty} - possibly because validation rules are " +
                        $"overly restrictive.{DebugValues(originalValue, newValue)}");
                // Check that if we did manage to change the property value, that the server
                // returned a different id from all previous versions of this resource.
                else if (resourceIds.Contains(modified.id))
                    Assert.Fail($"The server returned the same resource id ({modified.id}) after " +
                        $"randomly changing {strProperty}.{DebugValues(originalValue, newValue)}");
                resourceIds.Add(modified.id);
                Reflection.FinalizePostedResource(modified);
            }
        }
        #endregion Test Sample Injection
 
        [TestMethod, TestCategory(Category)]
        public virtual void Test_API_Detect_BadAPITest()
        {
            if (EnvironmentSettings.RUN_OFFLINE)
                Assert.Inconclusive("RUN_OFFLINE = true");
            Distribution _ = Samples.Distribution_BinomialDistribution.Posted;
            // Test that the server is identified as being unclean
            bool serverIsClean = CleanUp.CheckServerIsClean(true);
            Assert.IsFalse(serverIsClean,
                "The test utility did not properly detect that the server is not clean.");
        }
 
        [TestMethod, TestCategory(Category)]
        public virtual void Test_API_Seed_And_SeedCleanup()
        {
            if (EnvironmentSettings.EXECUTING_IN_PARALLEL)
                Assert.Inconclusive("Cannot test wiping server while running tests in parallel.");
            Scripts.SeedServer();
            Scripts.WipeServer();
            if (EnvironmentSettings.ROOT_DELETE_SUPPORTED ||
                EnvironmentSettings.RESOURCE_DELETE_SUPPORTED)
            {
                Assert.IsTrue(CleanUp.CheckServerIsClean(), "Server did not end up clean.");
            }
        }
 
        [TestMethod, TestCategory(Category)]
        public void Test_API_PollUntilReady_TimeoutException()
        {
            AssertApi.ExceptionThrown(() =>
                {
                    DateTime start = DateTime.UtcNow;
                    try
                    {
                        API.PollUntilReady(() => API.GetResourceList<Layer>(timeout: 1), PollingOptions.Default);
                    }
                    catch (APIRequestException ex)
                    {
                        Console.WriteLine(ex);
                        throw;
                    }
 
                    TimeSpan elapsed = DateTime.UtcNow - start;
                    if (elapsed.TotalMilliseconds < 10)
                        Assert.Inconclusive("This test relies on timing out before a request completes, " +
                            "but this request appears to have completed so quickly (< 10ms) " +
                            "that timeout couldn't be enforced.");
                },
                (RequestTimeoutException ex) =>
                {
                    string messageIfFailed = "The returned error message isn't as expected:\n" + ex.Message;
                    Assert.IsTrue(ex.Message.StartsWith("The request timed out"), messageIfFailed);
                    Assert.IsTrue(ex.Message.EndsWith(
                            "If this is a large request, you may need to wait longer for a response. " +
                            "Check your API.DefaultRequestTimeout or set a timeout directly in your API call."),
                        messageIfFailed);
                });
        }
    }
}