GenericTest.cs

Go to the documentation of this file.

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Net;
using System.Reflection;
using System.Runtime.Serialization;
 
using AnalyzeRe;
using AnalyzeRe.APITypes;
using AnalyzeRe.Attributes;
using AnalyzeRe.Rest;
using AnalyzeRe.Utilities;
 
#if MSTEST
using Microsoft.VisualStudio.TestTools.UnitTesting;
#elif NUNIT
using NUnit.Framework;
#endif
 
namespace AnalyzeReTesting.Utilities
{
    public static class GenericTest
    {
        #region GET Generic Tests
        public static void GET_Collection<T>(IResourceCollection<T> collection_source)
            where T : IAPIResource
        {
            if (EnvironmentSettings.RUN_OFFLINE)
                Assert.Inconclusive("RUN_OFFLINE = true");
            void ValidateResponse(ICollectionResponse<T> response)
            {
                Assert.IsNotNull(response.meta);
                Assert.AreEqual(0, response.meta.offset);
                Debug.WriteLine("List contains " + response.meta.total_count + " items.");
                Assert.IsNotNull(response.items);
                Assert.IsFalse(response.items.Any(), "Expected no items to be returned.");
            }
 
            // Test the GET on the collection works with a page size of zero.
            ValidateResponse(API.GetResourceList<T>(API.Parameters.Page(0, 0)));
 
            // Test retrieval using the IResourceCollection works.
            ValidateResponse(collection_source.List(0, 0));
        }
 
        public static T Try_POST_Random_Resource<T>(Type subtype,
            Reflection reflection, int attempts = 3)
            where T : IAPIResource
        {
            if (!EnvironmentSettings.RANDOMIZED_TESTS_ENABLED)
                Assert.Inconclusive("RANDOMIZED_TESTS_ENABLED = false");
            if (EnvironmentSettings.RUN_OFFLINE)
                Assert.Inconclusive("RUN_OFFLINE = true");
            string generation_error = null;
            // Try to generate a POSTable resource of this type.
            return Reflection.LimitAttempts(() =>
            {
                try
                {
                    generation_error = null;
                    T randomizedResource = (T)reflection.CreateRandomizedInstance(subtype);
                    randomizedResource = randomizedResource.Post();
                    Console.WriteLine($"Posted a random new {randomizedResource.GetType().NiceTypeName()} " +
                        $"with id {randomizedResource.id}");
                    return reflection.FinalizePostedResource(randomizedResource);
                }
                catch (APIRequestException ex) { generation_error = ex.ToString(); }
                return default(T);
            }, arg => !Equals(arg, default(T)), () => generation_error, max_attempts: attempts);
        }
 
        public static void GET_Collection_AllSubtypes<T>(
            IResourceCollection<T> collection_source,
            Reflection reflection,
            long offset = 0, long limit = 100,
            RequestParameters additionalParameters = null,
            Action<ICollectionResponse<T>> additionalActions = null,
            int? timeout = null)
            where T : IAPIResource
        {
            if (!EnvironmentSettings.RANDOMIZED_TESTS_ENABLED)
                Assert.Inconclusive("RANDOMIZED_TESTS_ENABLED = false");
            if (EnvironmentSettings.RUN_OFFLINE)
                Assert.Inconclusive("RUN_OFFLINE = true");
 
            // Attempt to POST one new resource of every sub-type
            List<T> posted = Reflection.GetAllInstantiableSubtypes(typeof(T)).AsParallel()
                .Select(subtype => Try_POST_Random_Resource<T>(subtype, reflection))
                .ToList();
            foreach (T item in posted)
                Assert.IsFalse(String.IsNullOrWhiteSpace(item.id), "The " +
                    item.GetType().NiceTypeName() + " just posted didn't return a valid id ");
            List<string> ids = posted.Select(i => i.id).ToList();
 
            RequestParameters parameters =
                API.Parameters.Order<StoredAPIResource>(res => res.created, false)
                .AddParameters(API.Parameters.Page(offset, limit))
                // Add in any additional parameters from the caller.
                .AddParameters(additionalParameters);
 
            // Get the collection in each supported way
            ICollectionResponse<T> result_collection =
                collection_source.GetItems(ids, parameters, timeout);
            ICollectionResponse<T> result_reflection = API.GetResourceList<T>(
                parameters.AddParameters(API.Parameters.Ids(ids)), timeout: timeout);
 
            // Set up the routine that will validate our response
            void ValidateResponse(ICollectionResponse<T> response)
            {
                Assert.IsNotNull(response);
                Assert.IsNotNull(response.meta);
                // Assert that either the max page size was reached, or that
                // the collection returned as many or more items than we posted.
                long total_count = response.meta.total_count;
                int result_count = response.items.Count();
 
                Assert.AreEqual(limit, response.meta.limit, "Limit was not respected by the server.");
                Assert.AreEqual(offset, response.meta.offset, "Offset was not respected by the server.");
                if (total_count - offset >= limit)
                    Assert.AreEqual(limit, result_count, $"Expected a full page of {limit} " +
                        $"results to be returned, but only {result_count} items were.");
                else
                {
                    Assert.IsTrue(total_count >= posted.Count, $"Expected the collection to contain " +
                        $"at least as many items as were posted ({posted.Count}), but the server " +
                        $"only reports having {total_count} items.");
                    Assert.AreEqual(total_count - offset, result_count,
                        $"Expected all {(total_count - offset)} items in the collection after " +
                        $"offset {offset} to be in our results list, but only {result_count} items " +
                        $"were returned. Server total is {total_count}");
                }
 
                // Check if any specific items are missing from the list
                // (only applies if the offset wasn't set)
                if (offset == 0)
                {
                    List<T> missingFromList = posted.Where(
                        item => response.items.All(res => res.id != item.id)).ToList();
                    Assert.AreEqual(0, missingFromList.Count,
                        "One or more items that were successfully posted " +
                        $"weren't found in the list of {total_count} {typeof(T).NiceTypeName()} " +
                        $"objects retrieved with offset {offset} and limit {limit}:\n" +
                        String.Join("\n", missingFromList.Select(
                            r => $"{r.GetType().NiceTypeName()} with id {r.id}")));
                }
 
                // Perform any additional caller actions.
                additionalActions?.Invoke(response);
            }
 
            // Validate both responses
            ValidateResponse(result_collection);
            ValidateResponse(result_reflection);
        }
 
        public static void GET_ExistingResource_Succeeds<T>(T existingResource)
            where T : IAPIResource
        {
            if (EnvironmentSettings.RUN_OFFLINE)
                Assert.Inconclusive("RUN_OFFLINE = true");
            T getResult = existingResource.Get();
            AssertApi.AllPropertiesEqual(existingResource, getResult);
        }
 
        public static void GET_AllPropertiesRecognized<T>(T existingResource)
            where T : IAPIResource
        {
            if (EnvironmentSettings.RUN_OFFLINE)
                Assert.Inconclusive("RUN_OFFLINE = true");
            T resultTyped = existingResource.Get();
            Dictionary<string, object> resultRaw = existingResource.Get_AsType<Dictionary<string, object>>();
            Dictionary<string, object> unrecognizedProperties = new Dictionary<string, object>();
            Help_Test_Resource_GET_AllPropertiesRecognized(
                unrecognizedProperties, typeof(T), resultTyped, resultRaw);
 
            // Give an error for any unrecognized properties
            if (unrecognizedProperties.Count > 0)
            {
                Assert.Inconclusive("The following properties were in the JSON response, " +
                    "but were ignored because they had no corresponding property " +
                    "to deserialize to in the .NET object model:\n  " +
                    String.Join("\n  ", unrecognizedProperties.Select(kvp =>
                        "\"" + kvp.Key + "\": " + (kvp.Value?.ToString() ?? "(null)"))));
            }
        }
 
        private static void Help_Test_Resource_GET_AllPropertiesRecognized(
            IDictionary<string, object> unrecognizedProperties, Type toCheck,
            object resultTyped, IDictionary<string, object> resultRaw, string prefix = "")
        {
            if (String.IsNullOrEmpty(prefix))
                prefix = toCheck.NiceTypeName() + ".";
            Dictionary<string, PropertyInfo> serializedProperties = new Dictionary<string, PropertyInfo>();
 
            // Include all user-facing properties that aren't marked as ignored by the serializer.
            foreach (PropertyInfo property in toCheck.GetUserFacingProperties(excludeNotSerialized:true))
            {
                DataMemberAttribute dataMember =
                    property.GetCustomAttributeFast<DataMemberAttribute>();
                // Only properties that have a DataMember attribute count,
                // otherwise the property risks being ignored by the serializer.
                if (dataMember == null) continue;
                // The property is inserted with its DataMemberAttribute name if available
                string propertyName = dataMember.Name ?? property.Name;
                serializedProperties[propertyName] = property;
            }
            // Note: Newtonsoft can serialize to private properties as well, so if we can't
            //       find a property, we will check this list.
            List<PropertyInfo> allProperties = toCheck.GetProperties(
                BindingFlags.Public | BindingFlags.NonPublic |
                BindingFlags.Instance | BindingFlags.FlattenHierarchy).ToList();
 
            foreach (KeyValuePair<string, object> jsonProperty in resultRaw)
            {
                Type expectedType = resultTyped.GetType();
                object jsonValue = jsonProperty.Value;
 
                // If the strongly typed property (from the object model) is a dictionary
                // or list of objects, then it can contain anything it wants, so no need
                // to recurse and check all items in the dictionary returned by the JSON
                if (expectedType.IsGenericType &&
                    expectedType.GetGenericTypeDefinition() == typeof(Dictionary<,>))
                    return;
                // Otherwise, check that all properties in the json are in the object.
                // If the property is missing, add it to the list of unrecognized properties
                if (!serializedProperties.ContainsKey(jsonProperty.Key))
                {
                    // See if any non-user-facing properties have a matching DataMemberAttribute
                    PropertyInfo ignored = allProperties
                        .Where(pi => pi.IsAttributeDefinedFast<DataMemberAttribute>())
                        .FirstOrDefault(pi => jsonProperty.Key ==
                            (pi.GetCustomAttributeFast<DataMemberAttribute>().Name ?? pi.Name));
                    // If not, see if any properties have a name that would match
                    if (ignored == null)
                        ignored = allProperties.FirstOrDefault(pi => jsonProperty.Key == pi.Name);
                    // If there was no matching property, add it to the list of failed properties.
                    if (ignored == null)
                    {
                        unrecognizedProperties.Add(prefix + jsonProperty.Key, jsonValue);
                        continue;
                    }
                    // If there was a matching property on the object, just not a "user-facing"
                    // property, make a note of it in the test output, but don't necessarily fail the test.
                    string message = $"The response contained a property \"{jsonProperty.Key}\" " +
                        "that does not correspond to a user-facing deserialized data member, but " +
                        $"which corresponds to the {ignored.ReflectedType.NiceTypeName()} " +
                        $"property {ignored.PropertyType.NiceTypeName()} " +
                        $"{ignored.DeclaringType.NiceTypeName()}.{ignored.Name} ";
                    if (ignored.IsAttributeDefinedFast<ObsoleteAttribute>())
                        Console.WriteLine($"{message} - but this property is marked as Obsolete, " +
                            "so it is safe to assume it is intentionally being ignored.");
                    else if (ignored.IsAttributeDefinedFast<InternalMemberAttribute>())
                        Console.WriteLine($"{message} - but this property is marked as Internal, " +
                            "so it is safe to assume it is being surfaced to the user via " +
                            "another property or is intentionally not user-facing.");
                    else
                    {
                        Console.WriteLine($"{message} - but this property is not-user facing " +
                            "for reasons that don't seem deliberate. This should be clarified.");
                        unrecognizedProperties.Add(prefix + jsonProperty.Key, jsonValue);
                    }
                    continue;
                }
 
                PropertyInfo property = serializedProperties[jsonProperty.Key];
                object propertyValue = property.GetValue(resultTyped, null);
                // If the property is not missing, check its sub-properties if any are recognized.
                if (jsonValue is IDictionary<string, object> jsonDictionary)
                {
                    Type typeToCompare = propertyValue.GetType();
                    object propertyToCompare = propertyValue;
                    // If the property we're getting back from the server is a resolved reference,
                    // we should instead be comparing it to the reference type.
                    if (propertyToCompare is IReference<IAPIResource> &&
                        !jsonDictionary.ContainsKey(nameof(IReference.ref_id)))
                    {
                        typeToCompare = typeToCompare.GetGenericArguments()[0];
                        propertyToCompare = ((IReference<IAPIResource>)propertyToCompare).GetValue();
                    }
 
                    // Recurse and test the properties of any nested objects.
                    Help_Test_Resource_GET_AllPropertiesRecognized(unrecognizedProperties,
                        typeToCompare, propertyToCompare, jsonDictionary,
                        prefix + jsonProperty.Key + ".");
                }
                // Otherwise, check that we retrieved its value correctly
                else if (property.PropertyType.IsPrimitive || property.PropertyType == typeof(string))
                {
                    // JSON may interpret a numeric value as a being type other than the target
                    // property type (resulting from our custom deserialization). We should attempt
                    // to convert that value to the desired type before doing an equivalency test.
                    string if_error_message = $"We have a property to capture \"{jsonProperty.Key}\"" +
                        $", but the value didn't match. JSON property value: {jsonProperty.Value} " +
                        $"Deserialized Value {propertyValue}";
                    if (jsonValue != null && jsonValue.GetType() != property.PropertyType)
                    {
                        if_error_message += ". Note that the native JSON interpretation of the " +
                            $"value was as the type {jsonValue.GetType()}. This value was converted " +
                            $"to the type {property.PropertyType} to perform this equivalency test.";
                        jsonValue = Convert.ChangeType(jsonValue, property.PropertyType);
                    }
                    Assert.AreEqual(jsonValue, propertyValue, if_error_message);
                }
            }
        }
        #endregion GET Generic Tests
 
        #region POST Generic Tests and Assertions
        public static void POST_AllSubtypes_Succeeds<T>(Reflection reflection) where T : IAPIResource
        {
            if (!EnvironmentSettings.RANDOMIZED_TESTS_ENABLED)
                Assert.Inconclusive("RANDOMIZED_TESTS_ENABLED = false");
            foreach (Type subType in Reflection.GetAllInstantiableSubtypes(typeof(T)))
            {
                T posted = Try_POST_Random_Resource<T>(subType, reflection);
                Debug.WriteLine("Successfully posted a new " + typeof(T).NiceTypeName() +
                    " instance of type " + subType.NiceTypeName() + " with the id: " + posted.id);
            }
        }
 
        public static T POST_ValidResource<T>(T toPost) where T : IAPIResource
        {
            if (EnvironmentSettings.RUN_OFFLINE)
                Assert.Inconclusive("RUN_OFFLINE = true");
            T posted = default(T);
 
            AssertApi.MethodIsAllowed(() => posted = toPost.Post(), "POST", true);
            Console.WriteLine($"Posted a new {posted.GetType().NiceTypeName()} with id {posted.id}");
            Assert.IsFalse(String.IsNullOrWhiteSpace(posted.id),
                "No id returned after posting the " + posted.GetType().NiceTypeName());
            AssertApi.PostResponseMatches(toPost, posted);
            return posted;
        }
 
        public static T POST_ValidResourceWithData<T>(T toPost, string data)
            where T : IAPIResource_WithDataEndpoint
        {
            return UploadData(POST_ValidResource(toPost), data);
        }
 
        public static T UploadData<T>(T resource, string data)
            where T : IAPIResource_WithDataEndpoint
        {
            AssertApi.MethodIsAllowed(() =>
                resource.data.UploadString(data, Samples.UploadParams), "Data Upload", true);
            // If uploading data changed the posted object, we need to get it again
            return resource.Get();
        }
 
        public static T POST_ThenDoAction<T>(T toPost, Action<T> toExecute)
            where T : IAPIResource
        {
            T posted = POST_ValidResource(toPost);
            ExecuteActionAfterPost(toPost, posted, toExecute);
            return posted;
        }
 
        public static T POST_WithData_ThenDoAction<T>(
            T toPost, string data, Action<T> toExecute)
            where T : IAPIResource_WithDataEndpoint
        {
            T posted = POST_ValidResourceWithData(toPost, data);
            ExecuteActionAfterPost(toPost, posted, toExecute);
            return posted;
        }
 
        public static void POST_InvalidResource_Fails<T>(T invalidToPost) where T : IAPIResource =>
            POST_InvalidResource_Fails(invalidToPost, AssertApi.ApiExceptionTest(HttpStatusCode.BadRequest));
 
        public static void POST_InvalidResource_Fails<T, TException>(
            T invalidToPost, Action<TException> exceptionTest)
            where T : IAPIResource
            where TException : Exception
        {
            Perform_InvalidPost(invalidToPost, null, exceptionTest);
        }
 
        public static void POST_InvalidResourceWithData_Fails<T>(T toPost, string data)
            where T : IAPIResource_WithDataEndpoint
        {
            POST_InvalidResourceWithData_Fails(toPost, data,
                AssertApi.ApiExceptionTest(HttpStatusCode.BadRequest));
        }
 
        public static void POST_InvalidResourceWithData_Fails<T, TException>(
            T toPost, string data, Action<TException> exceptionTest)
            where T : IAPIResource_WithDataEndpoint
            where TException : Exception
        {
            Perform_InvalidPost(toPost, data, exceptionTest);
        }
 
        #region Private Helper Methods
        private static void ExecuteActionAfterPost<T>(T unposted, T posted, Action<T> toExecute)
            where T : IAPIResource
        {
            ExecuteWithConsoleErrorLogging(() => { toExecute(posted); }, () =>
                "The POST that was successfully before performing additional actions was:\n" +
                $"Request:  {unposted.Serialize()}\nResponse: {posted.Serialize()}");
        }
 
        private static void Perform_InvalidPost<T, TException>(
            T toPost, string data, Action<TException> exceptionTest)
            where T : IAPIResource
            where TException : Exception
        {
            if (EnvironmentSettings.RUN_OFFLINE)
                Assert.Inconclusive("RUN_OFFLINE = true");
            long numBefore = 0;
            if (!EnvironmentSettings.EXECUTING_IN_PARALLEL)
            {
                // Track the number of resources on the server before attempting the post.
                numBefore = API.GetResourceList<T>(API.Parameters.Page(0, 0)).meta.total_count;
            }
            // Flag determines whether exception was thrown after during POST or data PUT
            bool noExceptionOnPost = false;
 
            // Post the invalid resource and expect to catch an exception
            AssertApi.ExceptionThrown(() =>
            {
                T posted = toPost.Post();
                noExceptionOnPost = true;
                // Regardless of whether the Post was meant to succeed or not, if there is data to
                // be uploaded, do it now to avoid leaving an broken resource on the server
                if (data != null && posted is IAPIResource_WithDataEndpoint asWithData)
                    asWithData.data.UploadString(data, Samples.UploadParams);
                else if (data != null)
                    throw new ArgumentException($"Data was provided for upload, but the " +
                        $"supplied resource type {toPost.GetType()} does not appear to " +
                        $"implement the {typeof(IAPIResource_WithDataEndpoint)} interface.");
                Console.WriteLine(
                    $"The following request succeeded (but should have failed):\n{toPost.Serialize(true)}\n" +
                    $"The resulting posted resource was:\n{posted.Serialize(true)}");
            }, exceptionTest);
 
            if (!EnvironmentSettings.EXECUTING_IN_PARALLEL)
            {
                // Check the number of resources on the server after the attempted post.
                long numAfter = API.GetResourceList<T>(API.Parameters.Page(0, 0)).meta.total_count;
                if (!noExceptionOnPost)
                {
                    Assert.AreEqual(numBefore, numAfter, "The POST was unsuccessful, yet the " +
                        "resource collection count changed from " + numBefore + " to " + numAfter);
                }
                else
                {
                    Assert.IsTrue((numBefore > 0 && numAfter == numBefore) || numAfter == numBefore + 1,
                        "After a successful POST, followed by a failed PUT of data, we expected the " +
                        "number of items on the server to either increase by 1, or stay the same " +
                        "(if the resource was already on the server), but the count went from " +
                        numBefore + " to " + numAfter);
                }
            }
        }
        #endregion Private Helper Methods
        #endregion POST Generic Tests and Assertions
 
        #region PUT Generic Tests and Assertions
        public static void PUT_Fails<T, TException>(T toPut,
            Action<TException> exceptionTest = null)
            where T : IAPIResource
            where TException : Exception
        {
            if (EnvironmentSettings.RUN_OFFLINE)
                Assert.Inconclusive("RUN_OFFLINE = true");
            AssertApi.ExceptionThrown(() => toPut.Put(), exceptionTest);
        }
        #endregion PUT Generic Tests and Assertions
 
        #region Helper Methods
        public static T Mock<T>(string id) where T : IAPIResource
        {
            T test = (T)Activator.CreateInstance(
                Reflection.GetAllInstantiableSubtypes(typeof(T)).First());
            test.id = id;
            return test;
        }
 
        public static void ExecuteWithConsoleErrorLogging(Action action,
            Func<string> additionalContext = null)
        {
            try
            {
                action();
            }
            catch (Exception ex)
            {
                // Provide some additional context information to logs before re-throwing
                AssertApi.LogExceptionDetails(ex);
                if (additionalContext != null)
                    Console.WriteLine(additionalContext());
                throw;
            }
        }
        #endregion Private Helper Methods
    }
}