WebClient does not download the entire file if the size is above 50 MB
I am using WebClient to upload files. But something fishy is going on when it tries to download larger files over 50MB, the download will complete right after 39kb and won't download it all the way. Does anyone know what the problem is?
using (webClient = new WebClient())
{
webClient.Proxy = null;
webClient.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)");
webClient.DownloadFileCompleted += new AsyncCompletedEventHandler(Completed);
webClient.DownloadProgressChanged += new DownloadProgressChangedEventHandler(ProgressChanged);
// The variable that will be holding the url address (making sure it starts with http://)
Uri URL = urlAddress.StartsWith("http://", StringComparison.OrdinalIgnoreCase) ? new Uri(urlAddress) : new Uri("http://" + urlAddress);
// Start the stopwatch which we will be using to calculate the download speed
sw.Start();
try
{
// Start downloading the file
webClient.DownloadFileAsync(URL, _mainWindow.outputDirComboBox.SelectedItem.ToString() + "\\" + _compressedClientFileDownloadRelativePath, new List<object>
{
_compressedClientFileDownloadRelativePath,
_decompressedClientFileDownloadRelativePath,
_lastDownloadedFilePath,
_compressedSize,
_lastCompressedFilePartRelativePath,
_filePartsRelativePath,
urlAddress,
_clientFilePack,
_clientFile
});
}
catch (Exception ex)
{
MessageBox.Show(ex.Message);
}
}
DownloadFileCompleted only checks for errors and exits, but no error messages occur. Just that it finishes the download as if the download size was only 39kb
Edit1: While digging, I end up here: Downloading Large Google Drive Files from WebClient in C #
It looks like Google Drive has some kind of confirmation link, but they haven't been able to figure out how it all works yet.
Edit2: I just validated 39kb - html page validation
+3
source to share
2 answers
you have to wait for webClient.DownloadFileAsync process
otherwise your execution crashes to dispose when the file is still loading
Example:
public class DownloadManager
{
public void DownloadFile(string sourceUrl, string targetFolder)
{
WebClient downloader = new WebClient();
// fake as if you are a browser making the request.
downloader.Headers.Add("User-Agent", "Mozilla/4.0 (compatible; MSIE 8.0)");
downloader.DownloadFileCompleted += new AsyncCompletedEventHandler(Downloader_DownloadFileCompleted);
downloader.DownloadFileAsync(new Uri(sourceUrl), targetFolder);
}
private void Downloader_DownloadFileCompleted(object sender, AsyncCompletedEventArgs e)
{
// display completion status.
if (e.Error != null)
Console.WriteLine(e.Error.Message);
else
Console.WriteLine("Download Completed!!!");
}
}
+2
source to share
Thanks everyone!
After checking this issue: Downloading large files from Google Drive with WebClient in C # I was looking for getting large files from Google Drive using C # and found this :
using System;
using System.IO;
using System.Net;
public static class FileDownloader
{
private const string GOOGLE_DRIVE_DOMAIN = "drive.google.com";
private const string GOOGLE_DRIVE_DOMAIN2 = "https://drive.google.com";
// Normal example: FileDownloader.DownloadFileFromURLToPath( "http://example.com/file/download/link", @"C:\file.txt" );
// Drive example: FileDownloader.DownloadFileFromURLToPath( "http://drive.google.com/file/d/FILEID/view?usp=sharing", @"C:\file.txt" );
public static FileInfo DownloadFileFromURLToPath( string url, string path )
{
if( url.StartsWith( GOOGLE_DRIVE_DOMAIN ) || url.StartsWith( GOOGLE_DRIVE_DOMAIN2 ) )
return DownloadGoogleDriveFileFromURLToPath( url, path );
else
return DownloadFileFromURLToPath( url, path, null );
}
private static FileInfo DownloadFileFromURLToPath( string url, string path, WebClient webClient )
{
try
{
if( webClient == null )
{
using( webClient = new WebClient() )
{
webClient.DownloadFile( url, path );
return new FileInfo( path );
}
}
else
{
webClient.DownloadFile( url, path );
return new FileInfo( path );
}
}
catch( WebException )
{
return null;
}
}
// Downloading large files from Google Drive prompts a warning screen and
// requires manual confirmation. Consider that case and try to confirm the download automatically
// if warning prompt occurs
private static FileInfo DownloadGoogleDriveFileFromURLToPath( string url, string path )
{
// You can comment the statement below if the provided url is guaranteed to be in the following format:
// https://drive.google.com/uc?id=FILEID&export=download
url = GetGoogleDriveDownloadLinkFromUrl( url );
using( CookieAwareWebClient webClient = new CookieAwareWebClient() )
{
FileInfo downloadedFile;
// Sometimes Drive returns an NID cookie instead of a download_warning cookie at first attempt,
// but works in the second attempt
for( int i = 0; i < 2; i++ )
{
downloadedFile = DownloadFileFromURLToPath( url, path, webClient );
if( downloadedFile == null )
return null;
// Confirmation page is around 50KB, shouldn't be larger than 60KB
if( downloadedFile.Length > 60000 )
return downloadedFile;
// Downloaded file might be the confirmation page, check it
string content;
using( var reader = downloadedFile.OpenText() )
{
// Confirmation page starts with <!DOCTYPE html>, which can be preceeded by a newline
char[] header = new char[20];
int readCount = reader.ReadBlock( header, 0, 20 );
if( readCount < 20 || !( new string( header ).Contains( "<!DOCTYPE html>" ) ) )
return downloadedFile;
content = reader.ReadToEnd();
}
int linkIndex = content.LastIndexOf( "href=\"/uc?" );
if( linkIndex < 0 )
return downloadedFile;
linkIndex += 6;
int linkEnd = content.IndexOf( '"', linkIndex );
if( linkEnd < 0 )
return downloadedFile;
url = "https://drive.google.com" + content.Substring( linkIndex, linkEnd - linkIndex ).Replace( "&", "&" );
}
downloadedFile = DownloadFileFromURLToPath( url, path, webClient );
return downloadedFile;
}
}
// Handles 3 kinds of links (they can be preceeded by https://):
// - drive.google.com/open?id=FILEID
// - drive.google.com/file/d/FILEID/view?usp=sharing
// - drive.google.com/uc?id=FILEID&export=download
public static string GetGoogleDriveDownloadLinkFromUrl( string url )
{
int index = url.IndexOf( "id=" );
int closingIndex;
if( index > 0 )
{
index += 3;
closingIndex = url.IndexOf( '&', index );
if( closingIndex < 0 )
closingIndex = url.Length;
}
else
{
index = url.IndexOf( "file/d/" );
if( index < 0 ) // url is not in any of the supported forms
return string.Empty;
index += 7;
closingIndex = url.IndexOf( '/', index );
if( closingIndex < 0 )
{
closingIndex = url.IndexOf( '?', index );
if( closingIndex < 0 )
closingIndex = url.Length;
}
}
return string.Format( "https://drive.google.com/uc?id={0}&export=download", url.Substring( index, closingIndex - index ) );
}
}
// Web client used for Google Drive
public class CookieAwareWebClient : WebClient
{
private class CookieContainer
{
Dictionary<string, string> _cookies;
public string this[Uri url]
{
get
{
string cookie;
if( _cookies.TryGetValue( url.Host, out cookie ) )
return cookie;
return null;
}
set
{
_cookies[url.Host] = value;
}
}
public CookieContainer()
{
_cookies = new Dictionary<string, string>();
}
}
private CookieContainer cookies;
public CookieAwareWebClient() : base()
{
cookies = new CookieContainer();
}
protected override WebRequest GetWebRequest( Uri address )
{
WebRequest request = base.GetWebRequest( address );
if( request is HttpWebRequest )
{
string cookie = cookies[address];
if( cookie != null )
( (HttpWebRequest) request ).Headers.Set( "cookie", cookie );
}
return request;
}
protected override WebResponse GetWebResponse( WebRequest request, IAsyncResult result )
{
WebResponse response = base.GetWebResponse( request, result );
string[] cookies = response.Headers.GetValues( "Set-Cookie" );
if( cookies != null && cookies.Length > 0 )
{
string cookie = "";
foreach( string c in cookies )
cookie += c;
this.cookies[response.ResponseUri] = cookie;
}
return response;
}
protected override WebResponse GetWebResponse( WebRequest request )
{
WebResponse response = base.GetWebResponse( request );
string[] cookies = response.Headers.GetValues( "Set-Cookie" );
if( cookies != null && cookies.Length > 0 )
{
string cookie = "";
foreach( string c in cookies )
cookie += c;
this.cookies[response.ResponseUri] = cookie;
}
return response;
}
}
With this stackoverflow question + this github code, I figured it was related to both issues, as google sends a confirmation page when the file is over 50MB. So after implementing this github code, I can now upload large files using my webClient.
+2
source to share