BaseServerAPITestSuite.cs

Go to the documentation of this file.

using System;
using System.Diagnostics;
using System.Net;
using System.Threading;
 
using AnalyzeRe;
using AnalyzeRe.Authentication;
using AnalyzeReTesting.Properties;
using AnalyzeReTesting.Utilities;
#if MSTEST
using Microsoft.VisualStudio.TestTools.UnitTesting;
#elif NUNIT
using NUnit.Framework;
using TestClass = NUnit.Framework.TestFixtureAttribute;
using AssemblyCleanup = NUnit.Framework.OneTimeTearDownAttribute;
#endif
 
namespace AnalyzeReTesting.Base
{
    [TestClass]
    public abstract class BaseServerTestSuite : BaseClientTestSuite
    {
        #region Static Properties
        public static string API_URL { get; set; }
 
        public static IAccessToken API_AuthenticationToken { get; set; }
 
        private static bool? _lastTestCleanupSucceeded;
 
        private static readonly object SuiteInitializeLock = new object();
 
        private static volatile int _suiteInitializeCalled = 0;
        #endregion Static Properties
 
        #region Instance Properties
        public bool DeletingResourcesBetweenTests { get; private set; }
        #endregion Instance Properties
 
        #region Static Test Suite Constructor
        static BaseServerTestSuite()
        {
            API_URL = EnvironmentSettings.API_URL;
            // Create the authentication token (it will only be used if the server requires authentication)
            API_AuthenticationToken = new BasicAuthenticationToken(
                EnvironmentSettings.HTTPS_USER, EnvironmentSettings.HTTPS_PASSWORD);
 
            // Apply the configured properties to the API
            ApplyConfiguredCredentials();
            API.DefaultRequestTimeout = EnvironmentSettings.REQUEST_TIMEOUT;
            API.DefaultRequestTimeoutCollections = EnvironmentSettings.COLLECTION_REQUEST_TIMEOUT;
 
            // Optionally ignore SSL certificate errors since we often test against servers with
            // self-signed certificates
            if (!EnvironmentSettings.VALIDATE_SERVER_CERTIFICATE)
            {
                ServicePointManager.ServerCertificateValidationCallback =
                    (sender, certificate, chain, errors) => true;
            }
        }
 
#if NUNIT
        [OneTimeSetUp]
        public void TestSuiteInitialize() => TestSuiteInitialize(new FakeTestContext());
#elif MSTEST
        [AssemblyInitialize]
#endif
        public static void TestSuiteInitialize(TestContext context)
        {
            lock (SuiteInitializeLock)
            {
                if (_suiteInitializeCalled != 0)
                {
                    Console.WriteLine("WARNING: Tried to initialize the test suite on thread " +
                        Thread.CurrentThread.ManagedThreadId + " but another test thread (" +
                        _suiteInitializeCalled + ") has already invoked TestSuiteInitialize.");
                    return;
                }
                _suiteInitializeCalled = Thread.CurrentThread.ManagedThreadId;
            }
 
            // Log the test settings.
            context.WriteLine("Test Settings:");
            context.WriteLine("Connecting to server at " + EnvironmentSettings.API_URL +
                " with username " + EnvironmentSettings.HTTPS_USER);
            if (EnvironmentSettings.EXECUTING_IN_PARALLEL)
                context.WriteLine("Tests will be run in parallel.");
 
            context.WriteLine("Polling Interval: " + EnvironmentSettings.POLLING_INTERVAL);
            context.WriteLine("Request Timeout: " + EnvironmentSettings.REQUEST_TIMEOUT);
            context.WriteLine("Collection Request Timeout: " + EnvironmentSettings.COLLECTION_REQUEST_TIMEOUT);
            context.WriteLine("Data Upload Timeout: " + EnvironmentSettings.DATA_UPLOAD_TIMEOUT);
            context.WriteLine("Simulation Timeout: " + EnvironmentSettings.SIMULATION_TIMEOUT);
            context.WriteLine("Optimization Timeout: " + EnvironmentSettings.OPTIMIZATION_TIMEOUTS);
 
            if (!EnvironmentSettings.ROOT_DELETE_SUPPORTED)
                context.WriteLine("Root DELETE is set to \"not supported\". Root DELETE will not be called.");
            else
            {
                context.WriteLine("DELETE Timeout: " + EnvironmentSettings.DELETE_TIMEOUT);
                if (EnvironmentSettings.ENFORCE_CLEANUP_BEFORE_RUN)
                    context.WriteLine("Root DELETE will be called " +
                        (EnvironmentSettings.EXECUTING_IN_PARALLEL ? "once before tests begin." :
                        "before each test is run."));
                if (EnvironmentSettings.ENFORCE_CLEANUP_AFTER_RUN)
                    context.WriteLine("Root DELETE will be called " +
                        (EnvironmentSettings.EXECUTING_IN_PARALLEL ? "once after tests finish running." :
                        "after each test is run."));
            }
            if (!EnvironmentSettings.VALIDATE_SERVER_CERTIFICATE)
                context.WriteLine("Certificate Verification has been Disabled.");
            if (!EnvironmentSettings.RANDOMIZED_TESTS_ENABLED)
                context.WriteLine("Randomized Tests have been Disabled.");
            if (!EnvironmentSettings.SKIPS_ENABLED)
                context.WriteLine("Skipping tests has been Disabled, skipped tests will be run.");
            else if (EnvironmentSettings.FAIL_ON_EXPIRED_SKIP)
                context.WriteLine("Any skipped tests past their expiry will raise an error.");
            else
                context.WriteLine("Any skipped tests past their expiry will be run.");
#if DEBUG
            if (EnvironmentSettings.ENABLE_REQUEST_DEBUG_LOGGING)
            {
                context.WriteLine("Debug logs for all API requests have been enabled.");
                API.RequestLogging = true;
            }
            else
                API.RequestLogging = false;
#else
            if (EnvironmentSettings.ENABLE_REQUEST_DEBUG_LOGGING)
                context.WriteLine("Debug logs for all API requests is unavailable in Release mode.");
#endif
 
            // Setup the server
            CheckConnection();
            if (EnvironmentSettings.ENFORCE_CLEANUP_BEFORE_RUN &&
                EnvironmentSettings.ROOT_DELETE_SUPPORTED)
            {
                ForceServerCleanupAction();
            }
        }
 
        [AssemblyCleanup]
        public static void TestSuiteCleanup()
        {
            if (EnvironmentSettings.ENFORCE_CLEANUP_AFTER_RUN &&
                EnvironmentSettings.ROOT_DELETE_SUPPORTED)
            {
                ForceServerCleanupAction();
            }
        }
        #endregion Static Test Suite Constructor
 
        #region Constructor
        protected BaseServerTestSuite(bool? CleanBeforeEveryTest = null,
                                    bool? CleanAfterEveryTest = null)
        {
            if (CleanBeforeEveryTest ?? EnvironmentSettings.ENFORCE_CLEANUP_BEFORE_RUN &&
                EnvironmentSettings.ROOT_DELETE_SUPPORTED &&
                !EnvironmentSettings.EXECUTING_IN_PARALLEL)
            {
                AddCommonTestInitializeAction(DeleteAllResources);
            }
            AddCommonTestCleanupAction(delegate
            {
                if (_lastTestCleanupSucceeded == false &&
                    TestContext.CurrentTestOutcome != UnitTestOutcome.Passed)
                {
                    // Add a message to the failed test output to indicate that the test may have
                    // failed for reasons outside of its control.
                    Assert.Fail("Warning: This test did not pass, but this may have been " +
                        "due to the fact that the previous test failed to successfully clean " +
                        "up after itself, so the server was not clean when this test started.");
                }
            });
            if (CleanAfterEveryTest ?? EnvironmentSettings.ENFORCE_CLEANUP_AFTER_RUN &&
                EnvironmentSettings.ROOT_DELETE_SUPPORTED &&
                !EnvironmentSettings.EXECUTING_IN_PARALLEL)
            {
                AddCommonTestCleanupAction(DeleteAllResources);
            }
 
            // Determine whether we are deleting resources between tests based on the settings.
            DeletingResourcesBetweenTests =
                (CleanBeforeEveryTest ?? EnvironmentSettings.ENFORCE_CLEANUP_BEFORE_RUN &&
                EnvironmentSettings.ROOT_DELETE_SUPPORTED &&
                !EnvironmentSettings.EXECUTING_IN_PARALLEL) ||
                (CleanAfterEveryTest ?? EnvironmentSettings.ENFORCE_CLEANUP_AFTER_RUN &&
                EnvironmentSettings.ROOT_DELETE_SUPPORTED &&
                !EnvironmentSettings.EXECUTING_IN_PARALLEL);
 
            // It doesn't make sense to create a new samples class on each new test if
            // we aren't cleaning up between tests.
            if (DeletingResourcesBetweenTests)
                Samples.Reset();
        }
        #endregion Constructor
 
        #region Setup/Teardown methods
        public static void ApplyConfiguredCredentials()
        {
            // Don't override local default setting on this machine.
            API.ManagedSettingsEnabled = false;
            API.ServerURL = API_URL;
            API.AuthenticationToken = API_AuthenticationToken;
        }
 
        public static void CheckConnection()
        {
            ApplyConfiguredCredentials();
            DateTime start = DateTime.UtcNow;
            // Check whether this server is responding to OPTIONS requests as expected
            try
            {
                API.Options();
            }
            catch (Exception ex)
            {
                if (EnvironmentSettings.RUN_OFFLINE)
                {
                    // If we have no connection and RUN_OFFLINE is enabled, shrink the timeouts
                    // to allow tests that require a connection to fail faster.
                    API.DefaultRequestTimeout = API.DefaultRequestTimeoutCollections =
                        Settings.Default.REQUEST_TIMEOUT = Settings.Default.COLLECTION_REQUEST_TIMEOUT =
                            Settings.Default.DATA_UPLOAD_TIMEOUT = Settings.Default.DELETE_TIMEOUT =
                                Settings.Default.OPTIMIZATION_TIMEOUTS = Settings.Default.POLLING_INTERVAL =
                                    Settings.Default.SIMULATION_TIMEOUT = 1;
                    API.MaxRequestRetries = 0;
                    EnvironmentSettings.Reload();
                    return;
                }
                Assert.Fail("Could not communicate with server at " + API_URL +
                    ". Test cannot run. (" + (DateTime.UtcNow - start).TotalSeconds +
                    " seconds)\nError Details: " + ex);
            }
        }
        
        private void DeleteAllResources()
        {
            // Reset the samples class because the previously injected resources will be deleted.
            Samples.Reset();
            ForceServerCleanupAction();
        }
 
        private static void ForceServerCleanupAction()
        {
            Debug.WriteLine("LastTestCleanupSucceeded: " + (_lastTestCleanupSucceeded ?? (object)"(null)"));
            // Force cleanup of the server (tries to get it to DELETE for a while).
            _lastTestCleanupSucceeded = CleanUp.ForceCleanServer(
                // If the cleanup fails (times out) fail this test, but if the last test
                // failed cleanup, don't fail this one because of it. Just try to clean up the
                // test again and move on if cleanup is still not working.
                _lastTestCleanupSucceeded == false,
                // Give the server some time to perform the delete, but if the last delete
                // timeout or otherwise failed, it's probably broken for all future tests,
                // so use a reduced timeout to avoid dragging out the tests.
                _lastTestCleanupSucceeded == false ? 1 : EnvironmentSettings.DELETE_TIMEOUT);
        }
        #endregion Setup/Teardown methods
    }
}