Commit ecb91ceb99 for freeswitch.com
commit ecb91ceb99e97385e849abefb3a6043c7861932f
Author: Andrey Volk <andywolk@gmail.com>
Date: Mon Sep 28 16:24:09 2026 +0300
[Build-System] Windows: Retry failed package downloads and fail the build when a package never arrives (#3194)
diff --git a/w32/downloadpackage.task b/w32/downloadpackage.task
index 0223e98cb0..6132131ac2 100644
--- a/w32/downloadpackage.task
+++ b/w32/downloadpackage.task
@@ -38,6 +38,9 @@ using System.Diagnostics;
{
public string filename;
public int progress;
+ public long lastActivityTicks;
+ public Exception error;
+ public bool cancelled;
}
protected ManualResetEvent TaskCanceled { get; private set; }
@@ -70,6 +73,8 @@ using System.Diagnostics;
basedir = Path.GetFullPath(@"$(BaseDir)");
TaskCanceled = new ManualResetEvent(false);
+ ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12;
+ bool result = true;
//Log.LogMessage(MessageImportance.High,
// "Checking for package \"" + package + "\".");
@@ -96,6 +101,8 @@ using System.Diagnostics;
Log.LogMessage(MessageImportance.High,
"Start downloading package \"" + package + "\".");
+ bool downloaded = true;
+
using (var client = new System.Net.WebClient())
{
Uri uri = new Uri(package);
@@ -111,41 +118,12 @@ using System.Diagnostics;
} else
//if (!File.Exists(output)) // Uncomment to skip download if exists
{
- var syncObject = new State
- {
- filename = urifilename,
- progress = -1
- };
- lock (syncObject)
- {
- client.DownloadFileCompleted += new AsyncCompletedEventHandler(DownloadFileCompleted);
- client.DownloadProgressChanged += new DownloadProgressChangedEventHandler(DownloadProgressCallback);
- client.DownloadFileAsync(uri, output, syncObject);
-
- while (!Monitor.Wait(syncObject, 1000))
- {
- if (TaskCanceled.WaitOne(0))
- {
- client.CancelAsync();
- Monitor.Wait(syncObject);
-
- if (File.Exists(output))
- {
- Log.LogMessage(MessageImportance.High,
- "Deleting incomplete file " + output + " for package \"" + package + "\".");
- File.Delete(output);
- }
-
- Log.LogMessage(MessageImportance.High,
- "Downloading canceled for package \"" + package + "\".");
-
- break;
- }
- }
- }
+ client.DownloadFileCompleted += new AsyncCompletedEventHandler(DownloadFileCompleted);
+ client.DownloadProgressChanged += new DownloadProgressChangedEventHandler(DownloadProgressCallback);
+ downloaded = Download(client, uri, output, urifilename);
}
- if (File.Exists(output))
+ if (downloaded && File.Exists(output))
{
// Successful download.
@@ -181,7 +159,11 @@ using System.Diagnostics;
}
}
- if (!TaskCanceled.WaitOne(0))
+ if (!downloaded && !TaskCanceled.WaitOne(0))
+ {
+ result = false;
+ }
+ else if (!TaskCanceled.WaitOne(0))
{
Log.LogMessage(MessageImportance.High,
"Downloading finished for package \"" + package + "\".");
@@ -190,7 +172,129 @@ using System.Diagnostics;
m.ReleaseMutex();
- return true;
+ return result;
+ }
+
+ // Every failure is retried -- an HTTP status of any kind, 404 and 403
+ // included, a dropped or stalled connection, an empty file. The URL a sheet
+ // names is taken to be right, so a failure is something passing, and a
+ // missing file on a mirror or a release CDN is often exactly that. Only
+ // cancelling the build stops the retries early.
+ private static readonly int[] retryDelaysSeconds = { 2, 5, 10, 20, 30 };
+ private const int stallTimeoutSeconds = 60;
+
+ private bool Download(WebClient client, Uri uri, string output, string urifilename)
+ {
+ int attempts = retryDelaysSeconds.Length + 1;
+ string reason = null;
+
+ for (int attempt = 1; attempt <= attempts; attempt++)
+ {
+ if (attempt > 1)
+ {
+ int delay = retryDelaysSeconds[attempt - 2];
+ Log.LogWarning("Downloading package \"" + package + "\" failed (" + reason + "), attempt " +
+ (attempt - 1) + " of " + attempts + "; retrying in " + delay + " s.");
+ if (TaskCanceled.WaitOne(delay * 1000))
+ {
+ Log.LogMessage(MessageImportance.High,
+ "Downloading canceled for package \"" + package + "\".");
+ return false;
+ }
+ }
+
+ reason = DownloadOnce(client, uri, output, urifilename);
+ if (reason == null)
+ {
+ return true;
+ }
+
+ if (File.Exists(output))
+ {
+ Log.LogMessage(MessageImportance.High,
+ "Deleting incomplete file " + output + " for package \"" + package + "\".");
+ File.Delete(output);
+ }
+
+ if (TaskCanceled.WaitOne(0))
+ {
+ Log.LogMessage(MessageImportance.High,
+ "Downloading canceled for package \"" + package + "\".");
+ return false;
+ }
+ }
+
+ Log.LogError("Downloading package \"" + package + "\" failed after " + attempts + " attempts: " + reason + ".");
+ return false;
+ }
+
+ // One attempt: null when the file arrived, otherwise why it did not.
+ private string DownloadOnce(WebClient client, Uri uri, string output, string urifilename)
+ {
+ var syncObject = new State
+ {
+ filename = urifilename,
+ progress = -1,
+ lastActivityTicks = DateTime.UtcNow.Ticks
+ };
+
+ lock (syncObject)
+ {
+ client.DownloadFileAsync(uri, output, syncObject);
+
+ while (!Monitor.Wait(syncObject, 1000))
+ {
+ if (TaskCanceled.WaitOne(0))
+ {
+ client.CancelAsync();
+ Monitor.Wait(syncObject);
+ return "canceled";
+ }
+
+ // DownloadFileAsync ignores WebRequest timeouts, so a connection
+ // that stops delivering would otherwise wait forever.
+ long idle = DateTime.UtcNow.Ticks - Interlocked.Read(ref syncObject.lastActivityTicks);
+ if (idle > TimeSpan.FromSeconds(stallTimeoutSeconds).Ticks)
+ {
+ client.CancelAsync();
+ Monitor.Wait(syncObject);
+ return "no data for " + stallTimeoutSeconds + " s";
+ }
+ }
+ }
+
+ if (syncObject.error != null)
+ {
+ return Describe(syncObject.error);
+ }
+ if (syncObject.cancelled)
+ {
+ return "canceled";
+ }
+ if (!File.Exists(output))
+ {
+ return "no file written";
+ }
+ if (new FileInfo(output).Length == 0)
+ {
+ return "0 bytes";
+ }
+ return null;
+ }
+
+ private static string Describe(Exception error)
+ {
+ var web = error as WebException;
+ if (web != null)
+ {
+ var http = web.Response as HttpWebResponse;
+ if (http != null)
+ {
+ return "HTTP " + (int)http.StatusCode + " " + http.StatusDescription;
+ }
+ return web.Status + ": " + web.Message;
+ }
+ return error.GetType().Name + ": " + error.Message;
}
private void Extract(string filename, string extracttofolder)
@@ -244,10 +348,13 @@ using System.Diagnostics;
private void DownloadFileCompleted(object sender, AsyncCompletedEventArgs e)
{
- lock (e.UserState)
+ var state = (State)e.UserState;
+ lock (state)
{
+ state.error = e.Error;
+ state.cancelled = e.Cancelled;
//releases blocked thread
- Monitor.Pulse(e.UserState);
+ Monitor.Pulse(state);
}
}
@@ -267,6 +374,8 @@ using System.Diagnostics;
private void DownloadProgressCallback(object sender, DownloadProgressChangedEventArgs e)
{
+ Interlocked.Exchange(ref ((State)e.UserState).lastActivityTicks, DateTime.UtcNow.Ticks);
+
if (((State)e.UserState).progress < e.ProgressPercentage)
{
((State)e.UserState).progress = e.ProgressPercentage;