LargeDataUpload_ExtensionMethods.cs

Go to the documentation of this file.

using System;
using System.Diagnostics;
using System.IO;
using System.Net;
using System.Threading;
 
using AnalyzeRe.APITypes;
using AnalyzeRe.LargeDataUpload;
using AnalyzeRe.LossSets;
using AnalyzeRe.Rest;
using AnalyzeRe.Utilities;
using RestSharp;
using Utils = AnalyzeRe.LargeDataUpload.Utilities;
 
// ReSharper disable once CheckNamespace (Extension methods classes must be on root namespace)
namespace AnalyzeRe
{
    public static class LargeDataUpload_ExtensionMethods
    {
        public const int DefaultBufferSize = 4096;
 
        #region Extension Methods
        #region Asynchronous Invocations
        public static void LargeFileUpload_Async(this LargeDataSubResource dataEndpoint,
            string filePath, AsyncParameters parameters = null)
        {
            dataEndpoint.ValidateOwnerHasId();
            parameters = parameters ?? new AsyncParameters();
            new Thread((ThreadStart)delegate
            {
                try
                {
                    using (FileStream fs = Utils.OpenReadOnlyFileStream(filePath))
                    {
                        UploadDataFromStream(dataEndpoint, fs, parameters);
                    }
                }
                catch (Exception ex)
                {
                    parameters.OnError?.Invoke(ex);
                }
            }).Start();
        }
 
        public static void LargeStreamUpload_Async(this LargeDataSubResource dataEndpoint,
            Stream dataStream, AsyncParameters parameters = null, long? dataSize = null)
        {
            dataEndpoint.ValidateOwnerHasId();
            parameters = parameters ?? new AsyncParameters();
            new Thread((ThreadStart)delegate
            {
                try
                {
                    UploadDataFromStream(dataEndpoint, dataStream, parameters, dataSize);
                }
                catch (Exception ex)
                {
                    parameters.OnError?.Invoke(ex);
                }
            }).Start();
        }
        #endregion Asynchronous Invocations
 
        #region Synchronous Invocations
        public static void LargeFileUpload(
            this LargeDataSubResource dataEndpoint, string filePath, Parameters parameters = null)
        {
            dataEndpoint.ValidateOwnerHasId();
            using (FileStream fs = Utils.OpenReadOnlyFileStream(filePath))
            {
                LargeStreamUpload(dataEndpoint, fs, parameters);
            }
        }
 
        public static void LargeStreamUpload(
            this LargeDataSubResource dataEndpoint, Stream dataStream,
            Parameters parameters = null, long? dataSize = null)
        {
            dataEndpoint.ValidateOwnerHasId();
            // Wrap any user-supplied parameters in our own to handle errors synchronously.
            AsyncParameters uploadParameters = new AsyncParameters(parameters ?? new Parameters(),
                onError: e =>
                {
                    // Make sure to call the original OnError callback first if there is one.
                    (parameters as AsyncParameters)?.OnError?.Invoke(e);
                    throw e;
                });
            UploadDataFromStream(dataEndpoint, dataStream, uploadParameters, dataSize);
        }
        #endregion Synchronous Invocations
        #endregion Extension Methods
 
        #region Segmented Upload Implementation
        private static void UploadDataFromStream(LargeDataSubResource dataEndpoint,
            Stream dataStream,
            AsyncParameters parameters,
            long? dataSize = null)
        {
            // Verify that all required parameters are set to non-null values.
            if (dataEndpoint == null) throw new ArgumentNullException(nameof(dataEndpoint));
            if (dataStream == null) throw new ArgumentNullException(nameof(dataStream));
            if (parameters == null) throw new ArgumentNullException(nameof(parameters));
 
            // Get the size of the file
            // If no file size was supplied, derive it from the stream's properties
            // If no file size is available, the server cannot validate the entire file was received.
            long? fileSize = dataSize ?? DetermineStreamLength(dataStream);
            
            // ReSharper disable once RedundantAssignment (debug mode requires this assignment)
            DateTime start = DateTime.UtcNow;
            if (Utils.EnableDebugLogging)
            {
                Debug.WriteLine($"{start.ToString(API.DebugTimeFormat)} " +
                    $"{dataEndpoint.Owner.GetType().NiceTypeName()} Large File Upload Initiated " +
                    $"({(fileSize.HasValue ? fileSize + " bytes)" : "unknown size)")}");
                // Allows for extensive debugging of large file upload progress.
                // (Hooking this event is expensive, so don't do it in Release mode)
                void OnProgressUpdate(StatusResponse s)
                {
                    DateTime now = DateTime.UtcNow;
                    Debug.WriteLine($"{now.ToString(API.DebugTimeFormat)} Upload Progress Update. " +
                        $"Time Elapsed: {(now - start).TotalMilliseconds}ms, " +
                        $"Status: {s.status} ({s.status.ToAPIString()}), " +
                        $"Uploaded: {s.bytes_uploaded} of {s.total_bytes?.ToString() ?? "(unknown)"}" +
                        $"{(s.commit_progress.HasValue ? $", Commit: {s.commit_progress}" : "")}" +
                        $"{(String.IsNullOrEmpty(s.commit_error) ? "" : $", Error: {s.commit_error}")}");
                }
 
                // Wrap the existing parameters to add our own debug hooks.
                AsyncParameters oldParameters = parameters;
                parameters = new AsyncParameters(oldParameters, onProgressUpdate: s =>
                {
                    OnProgressUpdate(s);
                    oldParameters.OnProgressUpdate?.Invoke(s);
                });
            }
            
#pragma warning disable 618
            // TODO: Deprecate binary YELT format and related logic.
            bool isBinaryYELTUpload = (dataEndpoint.Owner as YELTLossSet)?.data_type == YELTLossSet.DataType.binary;
            // Hack: If this is a binary YELT upload, the final file uploaded will be a zip file
            //       (not just temporarily zipped for transmission). We won't know the final file size,
            //       so we must set it to null now even if it can be determined from the stream.
            if (isBinaryYELTUpload)
            {
                fileSize = null;
                if (Utils.EnableDebugLogging)
                    Debug.WriteLine($"{DateTime.UtcNow.ToString(API.DebugTimeFormat)} - LFU " +
                        $"binary YELTLossSet detected, file size ignored.");
            }
#pragma warning restore 618
 
            // Initialize the upload session. If we are resuming a previous upload, get the offset
            // we should start streaming data from.
            long currentStreamPosition = 0;
            // TODO: What if the user's stream has already been advanced? Can this be configured?
            long? bytesToSkip = InitiateUploadSession(dataEndpoint, parameters, fileSize);
 
            // If the file isn't already fully uploaded, begin the process of uploading data
            if (bytesToSkip != fileSize)
            {
                // If we are resuming a previous upload, advance the stream to where uploading will resume
                if (bytesToSkip.HasValue && bytesToSkip.Value > 0)
                {
                    currentStreamPosition = AdvanceStream(dataStream, bytesToSkip.Value);
                    if (Utils.EnableDebugLogging)
                        Debug.WriteLine($"{DateTime.UtcNow.ToString(API.DebugTimeFormat)} - " +
                            $"LFU ignored the first {bytesToSkip.Value} bytes of the stream's " +
                            $"{fileSize} total bytes in order to resume a previous upload.");
                }
 
                // Upload the entire file starting from the current point in the data stream.
                currentStreamPosition = UploadBytes(dataEndpoint, dataStream, parameters, currentStreamPosition, fileSize);
                if (Utils.EnableDebugLogging)
                {
                    DateTime now = DateTime.UtcNow;
                    Debug.WriteLine($"{now.ToString(API.DebugTimeFormat)} Upload Complete: " +
                        $"{(now - start).TotalMilliseconds}ms");
                }
            }
            // If the bytesToSkip = fileSize, there's nothing to upload, so go straight to the commit phase.
            else
            {
                if (fileSize.HasValue)
                    currentStreamPosition = fileSize.Value;
                if (Utils.EnableDebugLogging)
                    Debug.WriteLine($"{DateTime.UtcNow.ToString(API.DebugTimeFormat)} - LFU ignored all " +
                        $"{fileSize?.ToString() ?? "(unknown length)"} bytes of the stream in order to " +
                        $"resume monitoring the status of a previous upload that has already been committed.");
            }
 
            // Notify that we have 100% completed the upload.
            parameters.OnProgressUpdate?.Invoke(new StatusResponse
            {
                status = TaskStatus.Uploading,
                bytes_uploaded = currentStreamPosition,
                total_bytes = currentStreamPosition
            });
 
            // Commit the file to begin server-side processing and validation
            CommitUpload(dataEndpoint, parameters);
            if (Utils.EnableDebugLogging)
            {
                DateTime now = DateTime.UtcNow;
                Debug.WriteLine($"{now.ToString(API.DebugTimeFormat)} Commit Complete. " +
                    $"Total Large file upload time: {(now - start).TotalMilliseconds}ms");
            }
        }
 
        // Helper method that determines the length of the Stream from it's properties
        // or throws an exception if that's not possible.
        private static long? DetermineStreamLength(Stream dataStream)
        {
            long dataStreamStart = dataStream.CanSeek ? dataStream.Position : 0;
            // Get the file size from the stream (minus the current position)
            return dataStream.CanSeek ? dataStream.Length - dataStreamStart : (long?)null;
        }
 
        private static long AdvanceStream(Stream dataStream, long bytes)
        {
            // Skip ahead in the supplied stream now if we need to and are able to.
            if (dataStream.CanSeek)
                return dataStream.Position += bytes;
            // If the stream doesn't support seeking, we must use Read() to advance :(
            byte[] ignored = new byte[Math.Min(bytes, DefaultBufferSize)];
            long streamPosition = 0;
            while (streamPosition < bytes && dataStream.CanRead)
            {
                int bytesRead = dataStream.Read(ignored, 0,
                    (int)Math.Min(ignored.Length, bytes - streamPosition));
                // Some types of streams simply return no bytes when you reach EOF.
                if (bytesRead == 0)
                    break;
                streamPosition += bytesRead;
            }
            if (streamPosition < bytes)
                throw new EndOfStreamException($"AdvanceStream was asked to advance {bytes} " +
                    $"bytes, but the stream ended after reading {streamPosition} bytes.");
            return streamPosition;
        }
 
        // Helper method that streams data from the current offset until the end of the file.
        // Returns the final stream position after upload completion.
        private static long UploadBytes(LargeDataSubResource dataEndpoint, Stream dataStream,
            AsyncParameters parameters, long streamPosition, long? streamLength)
        {
            // The following properties are only used for purposes of debugging and timing the upload.
            DateTime checkpoint = DateTime.UtcNow;
            DateTime start = checkpoint;
            long initialStreamPosition = streamPosition;
            if (Utils.EnableDebugLogging)
            {
                Debug.WriteLine($"{start.ToString(API.DebugTimeFormat)} - LFU " +
                    $"starting from stream position of {streamPosition} bytes out of " +
                    $"{(streamLength.HasValue ? streamLength + " bytes." : "(unknown size).")}");
            }
 
            // Set the maximum source buffer length equal to the number of bytes
            // we want to upload in each PATCH request to the server.
            int max_buffer_length = parameters.max_chunk_size;
            // If we know the length of the entire file, we can potentially shrink the max buffer size.
            if (streamLength.HasValue && streamLength.Value < max_buffer_length)
                max_buffer_length = (int)streamLength.Value;
            int min_buffer_length = Math.Min(parameters.min_chunk_size, max_buffer_length);
 
            // Set up the producer-consumer chain of buffered bytes to be uploaded.
            // The basic source is a buffer for the file stream
            IProducerConsumerBuffer<BufferedBytes> restSource = new BufferedBytesFromStreamProducer(
                dataStream, streamLength, min_buffer_length, max_buffer_length);
            // Add on the gzip layer if applicable
            if (parameters.enable_compression)
                restSource = new GZipBufferedBytesProducer(restSource, min_buffer_length, max_buffer_length);
 
#pragma warning disable 618
            // TODO: Deprecate binary YELT format and related logic.
            bool isBinaryYELTUpload = (dataEndpoint.Owner as YELTLossSet)?.data_type == YELTLossSet.DataType.binary;
            // Automatic conversion can be toggled off in the parameters to support
            // e.g. downloading and re-uploading a file that has already been converted to binary.
            if (isBinaryYELTUpload && parameters.binary_yelt_options != Parameters.BinaryYELTUploadOptions.None)
            {
                if (parameters.binary_yelt_options != Parameters.BinaryYELTUploadOptions.AutomaticCompressionOnly)
                    restSource = new BinaryYELTConverter(restSource);
                restSource = new GZipBufferedBytesProducer(restSource, min_buffer_length, max_buffer_length);
            }
#pragma warning restore 618
 
            using (restSource)
            {
                // If this is a GZipStream, we need to keep track of *uncompressed* bytes
                // uploaded because that's what the server expects the offset to be.
                GZipBufferedBytesProducer sourceAsGZipProducer = restSource as GZipBufferedBytesProducer;
                long currentChunkNumber = 0;
 
                // Tracks the overhead in setting up the producer/consumer chain.
                if (Utils.EnableDebugLogging)
                {
                    DateTime now = DateTime.UtcNow;
                    Debug.WriteLine($"{now.ToString(API.DebugTimeFormat)} - LFU " +
                        $"time to setup: {(now - checkpoint).TotalSeconds}s");
                    checkpoint = now;
                    currentChunkNumber = 1;
                }
 
                // Tell the producer/consumer threads to begin buffering data.
                restSource.Start(parameters.commit_polling_options.CancellationToken);
 
                // Loop until we've uploaded the remaining bytes in the file.
                // If the file size is unknown, loop until an inner exit condition is met
                while (!streamLength.HasValue || streamPosition < streamLength)
                {
                    // Check to see if the upload has been cancelled before each chunk.
                    Utils.CheckForCancellation(parameters);
 
                    // Invoke the OnProgressUpdate callback (if any) with a mocked
                    // status response to avoid a round trip request to the server.
                    parameters.OnProgressUpdate?.Invoke(new StatusResponse
                    {
                        status = TaskStatus.Uploading,
                        bytes_uploaded = streamPosition,
                        total_bytes = streamLength
                    });
                    // Raise an error if the end of stream is encountered too early
                    // Need to check in this order to avoid race condition where
                    // item is added to collection and marked complete after the "Any()" check.
                    if (!restSource.CanTake)
                    {
                        // If the file length is unknown, this is the expected way of breaking the loop.
                        if (!streamLength.HasValue)
                            break;
                        // Otherwise, if we haven't received the full file we were expecting, raise an error.
                        throw new EndOfStreamException($"Expected to receive {streamLength} bytes of data, " +
                            $"but the end of the data source was reached after {streamPosition} bytes.");
                    }
 
                    // Try to get the next buffered chunk (or wait for one to become available)
                    BufferedBytes nextChunk = restSource.TryTake(out bool success,
                        parameters.commit_polling_options.CancellationToken);
                    // If no new data was returned, jump back to the start of the loop.
                    if (!success) continue;
                    
                    double elapsedSeconds, megabytes = 0;
                    if (Utils.EnableDebugLogging)
                    {
                        // Tracks how long it took to produce the last chunk.
                        DateTime now = DateTime.UtcNow;
                        elapsedSeconds = (now - checkpoint).TotalSeconds;
                        megabytes = nextChunk.LengthFilled / 1048580d;
                        Debug.WriteLine($"{now.ToString(API.DebugTimeFormat)} - LFU chunk " +
                            $"{currentChunkNumber} size: {nextChunk.LengthFilled} bytes ({megabytes:0.00} MiB)");
                        Debug.WriteLine($"{now.ToString(API.DebugTimeFormat)} - LFU time to " +
                            $"produce chunk {currentChunkNumber}: {elapsedSeconds}s (" +
                            (Math.Abs(elapsedSeconds) < 1E-4 ? "was ready)" : $"{megabytes / elapsedSeconds:0.0} MiBps)"));
                        checkpoint = now;
                    }
                    // Try to upload this chunk
                    UploadChunk(dataEndpoint, nextChunk.Bytes, nextChunk.LengthFilled, streamPosition, parameters);
                    if (Utils.EnableDebugLogging)
                    {
                        // Tracks how long it took to upload the last chunk.
                        DateTime now = DateTime.UtcNow;
                        elapsedSeconds = (now - checkpoint).TotalSeconds;
                        Debug.WriteLine($"{now.ToString(API.DebugTimeFormat)} - LFU time " +
                            $"to upload chunk {currentChunkNumber}: {elapsedSeconds}s " +
                            $"({megabytes / elapsedSeconds:0.0} MiBps)");
                        checkpoint = now;
                        currentChunkNumber++;
                    }
                    // Update the stream position to reflect what has been uploaded to the server
                    streamPosition = initialStreamPosition + nextChunk.TotalBytesRead;
                }
                // If data is being compressed, we can output the compression ratio achieved.
                if (Utils.EnableDebugLogging && sourceAsGZipProducer != null)
                    Debug.WriteLine($"Compressed a total of {sourceAsGZipProducer.TotalBytesIn} bytes " +
                        $"into {sourceAsGZipProducer.TotalBytesOut} bytes (" +
                        $"{sourceAsGZipProducer.TotalBytesOut / (double)sourceAsGZipProducer.TotalBytesIn:P2} the original size).");
            }
 
            if (Utils.EnableDebugLogging)
            {
                // Tracks how long the entire file took to upload, and the effective throughput.
                DateTime now = DateTime.UtcNow;
                double total_elapsed = (now - start).TotalSeconds;
                long total_bytes = streamPosition - initialStreamPosition;
                double total_megabytes = total_bytes / 1048580d;
                Debug.WriteLine($"{now.ToString(API.DebugTimeFormat)} - LFU " +
                    $"total upload time: {total_elapsed}s for {total_bytes} bytes " +
                    $"({total_megabytes:0.0} MiB - {total_megabytes / total_elapsed:0.0} MiBps)");
            }
            return streamPosition;
        }
 
        private static void CommitUpload(LargeDataSubResource dataEndpoint, AsyncParameters parameters)
        {
            StatusResponse status = null;
 
            // Check to see if the upload has been cancelled before attempting the commit.
            Utils.CheckForCancellation(parameters);
 
            // HACK: Since the user is willing to wait up until "MaxPollTotalTime" for the result
            //       to be ready, in theory any request leading up to polling can take this long
            //       before timing out and still be acceptable. This is a hack because we aren't
            //       keeping track of the total time taken from beginning to end of the "commit"
            //       process - we should really only allow polling for MaxPollTotalTime minus
            //       the total time taken of all requests before we started actually polling.
            int requestTimeout = parameters.commit_polling_options.MaxPollTotalTime;
 
            // Finalize the upload session.
            // In most cases, it's faster to invoke commit immediately and expect a successful
            // response. In the few scenarios where it is not appropriate to invoke commit
            // (we are resuming a previously committed upload, or there was an error and
            // the chunks are incomplete) we can follow this up with a status get to clarify
            try
            {
                IRestResponse response = dataEndpoint.commit.POST(requestTimeout);
                if (response.StatusCode != HttpStatusCode.Accepted)
                    throw new APIRequestException("Unexpected Server response to commit.", null, response);
            }
            catch (APIRequestException e)
            {
                if (e.RestResponse.StatusCode != HttpStatusCode.MethodNotAllowed)
                    throw;
                // If the POST is 405 not allowed, we must be resuming another commit or
                // in an incomplete state. Get the status from the server
                status = dataEndpoint.status.Get(timeout: requestTimeout);
                // If we're not in one of the below states, re-throw the error that was encountered
                if (!status.status.IsBusy() && !status.status.IsProcessingComplete())
                    throw;
                // Otherwise continue with the logic for monitoring the processing
            }
 
            // Poll the status endpoint with GET until processing is complete.
            if (status == null || status.status.IsBusy())
                status = Utils.PollUntilDataProcessed(dataEndpoint, parameters);
            // If the processing is complete, report either success or error.
            if (status.status == TaskStatus.Success)
                parameters.OnSuccess?.Invoke(status);
            else if (status.status.IsError())
                parameters.OnError?.Invoke(new CommitFailedException(status.commit_error));
            else
                parameters.OnError?.Invoke(new CommitFailedException(
                    $"The data endpoint entered an unexpected state \"{status.status}\" " +
                    $"while attempting to commit the upload. Server status is: {status.status}"));
        }
 
        public static RequestParameters GzipEncodingHeader = new RequestParameters(new[]
        {
            new Parameter
            {
                Type = ParameterType.HttpHeader,
                Name = "Content-Encoding",
                Value = "gzip"
            }
        });
 
        private static void UploadChunk(LargeDataSubResource dataEndpoint, byte[] data, int count,
            long offset, AsyncParameters parameters)
        {
            int uploadAttempts = 0;
            while (true)
            {
                uploadAttempts++;
                try
                {
                    dataEndpoint.Patch(data, count, offset, parameters.chunk_timeout,
                        parameters.enable_compression ? GzipEncodingHeader : null);
                    return;
                }
                // If the error was a timeout, try again
                catch (TimeoutException)
                {
                    if (uploadAttempts >= parameters.max_retries_per_chunk)
                        throw;
                }
                // If the error is an APIRequestException:
                catch (APIRequestException e)
                {
                    if (e.RestResponse.StatusCode != HttpStatusCode.MethodNotAllowed)
                        throw;
 
                    // Get the status from the server
                    var requestTimeout = parameters.commit_polling_options.MaxPollTotalTime;
                    var status = dataEndpoint.status.Get(timeout: requestTimeout);
                    
                    // If we're not in one of the below states, re-throw the error that was encountered
                    if (!status.status.IsBusy() && !status.status.IsProcessingComplete())
                        throw;
 
                    return;
                }
                Utils.CheckForCancellation(parameters);
            }
        }
 
        private static long? InitiateUploadSession(
            LargeDataSubResource dataEndpoint, Parameters parameters, long? fileSize)
        {
            // Request the current status of the data endpoint
            StatusResponse status = dataEndpoint.status.Get();
            TaskStatus state = status.status;
 
            // How we proceed usually depends on our conflict resolution strategy
            HandleUploadInSessionStrategy strategy = parameters.handle_existing_upload_strategy;
 
            // Check for the simplest statuses which don't require much additional work
            switch (state)
            {
                case TaskStatus.Awaiting_Upload:
                    // Initiate the large upload and that's it.
                    dataEndpoint.Post(fileSize);
                    return 0;
                case TaskStatus.Success:
                    // Previous upload finalized and successful, raise an error if we aren't resuming
                    if (strategy != HandleUploadInSessionStrategy.AttemptResume)
                        throw new InvalidOperationException("Cannot upload data for this resource " +
                            "because existing data has already been successfully uploaded.");
                    break;
                case TaskStatus.Unknown:
                    // Hopefully this never happens.
                    throw new APIRequestException("Could not determine the current state " +
                        "of this data upload process. Cannot continue.");
            }
 
            if (strategy == HandleUploadInSessionStrategy.RaiseError)
                throw new UploadInProgressException();
            if (strategy == HandleUploadInSessionStrategy.CancelPriorUpload)
            {
                if (state != TaskStatus.Uploading && state != TaskStatus.Failed)
                    throw new UploadInProgressException("An conflicting data upload is currently " +
                        "being processed and cannot be deleted. Please wait until processing is " +
                        "of the file is complete.");
                // Delete the current data endpoint, then initiate the new large upload.
                dataEndpoint.Delete();
                dataEndpoint.Post(fileSize);
                return 0;
            }
            // (This should never happen unless the enum is changed without revising this method.)
            if (strategy != HandleUploadInSessionStrategy.AttemptResume)
                throw new ArgumentException($"Unexpected upload conflict strategy: {strategy}");
 
            // Raise an error if we aren't in one of the expected possible states
            // (shouldn't happen if the above conditional logic is sound)
            if (!(state == TaskStatus.Uploading || state == TaskStatus.Queued
                || state == TaskStatus.Processing || state == TaskStatus.Success
                || state == TaskStatus.Failed))
            {
                throw new Exception($"Internal error. Unexpected status: {state}");
            }
            // At this point the UploadStatus is one of the following:
            // UploadingChunks, Processing, Processing_Failed, or Processing_Successful
            // In all cases, advance the stream to match the server reported offset (in the
            // 'processed' cases this should be EOF) and let the upload process commit or re-commit.
 
            // If supplied, ensure the total_bytes matches our file's length
            if (fileSize != null && status.total_bytes != null && status.total_bytes.Value != fileSize)
            {
                throw new UploadInProgressException(
                    $"Attempted to resume an existing upload session with a file of different size. " +
                    $"Prior file size: {status.total_bytes.Value} This file size: {fileSize}");
            }
 
            // Special case - if the state is "Processing" the /status endpoint does not
            // report anything that tells us the size of the file, so we have to assume
            // it matches our file size and inform the caller to advance to the end of the file.
            // TODO: Verify whether this is still the case. File a complaint if so ;)
            if (state == TaskStatus.Processing || state == TaskStatus.Queued)
            {
                return fileSize;
            }
 
            // Ensure the bytes_uploaded lines up with this file.
            if (status.bytes_uploaded == null)
                throw new APIRequestException(
                    "Server status response was missing the expected bytes_uploaded value.");
            if (status.bytes_uploaded.Value > fileSize)
            {
                throw new UploadInProgressException($"Server gave unexpected Offset value of: " +
                    $"{status.bytes_uploaded} which exceeds this file's size of: {fileSize}");
            }
 
            // Resume the file upload at the position specified by the offset header:
            return status.bytes_uploaded.Value;
        }
        #endregion Segmented Upload Implementation
    }
}