using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Text;
using System.Windows.Forms;
using System.Net;
using System.Text.RegularExpressions;
using System.IO;
using System.Threading;
using System.Xml;
namespace DownloadSVN
{
/// <summary>
/// SVN 2016-05-13
/// Geovin Du edit
/// </summary>
public partial class MainForm : Form
{
ManualResetEvent _waitingForStop;
ManualResetEvent _finishedReadingTree;
List<FileDownloadData> _filesToDownload;
Thread _readingThread;
String _selectedSourceType;
public MainForm()
{
InitializeComponent();
this.comboBoxSourceType.SelectedIndex = 0;
}
private void buttonBrowseTargetFolder_Click(object sender, EventArgs e)
{
using (FolderBrowserDialog fbd = new FolderBrowserDialog())
{
if (fbd.ShowDialog() == DialogResult.OK)
this.textBoxTargetFolder.Text = fbd.SelectedPath;
}
}
private void buttonGo_Click(object sender, EventArgs e)
{
if (_readingThread == null)
{
_selectedSourceType = this.comboBoxSourceType.Text;
Thread t = new Thread(new ThreadStart(Run));
t.Start();
_readingThread = t;
SetButtonGoText("Stop");
}
else
{
Stop();
}
}
void Run()
{
// Start downloading threads
_finishedReadingTree = new ManualResetEvent(false);
_waitingForStop = new ManualResetEvent(false);
_filesToDownload = new List<FileDownloadData>();
List<Thread> downloadThreads = new List<Thread>();
for (int i = 0; i < 5; i++)
{
Thread t = new Thread(new ThreadStart(DownloadFilesThread));
t.Start();
downloadThreads.Add(t);
}
try
{
if ((this.textBoxTargetFolder.Text != "") && (this.textBoxSourceSvnUrl.Text != ""))
{
string url = this.textBoxSourceSvnUrl.Text;
if (_selectedSourceType == "GIT")
RunSvn(this.textBoxTargetFolder.Text, this.textBoxSourceSvnUrl.Text, RepositoryType.GIT);
else // "SVN"
RunSvn(this.textBoxTargetFolder.Text, this.textBoxSourceSvnUrl.Text, RepositoryType.SVN);
}
else
WriteToScreen("Parameters not set.");
}
catch (Exception ex)
{
WriteToScreen("Failed: " + ex);
lock (_filesToDownload)
{
_filesToDownload.Clear();
}
}
finally
{
_finishedReadingTree.Set();
}
// Wait for downloading threads
WriteToScreen("Waiting for file downloading threads to finish");
for (int i = 0; i < downloadThreads.Count; i++)
downloadThreads[i].Join();
WriteToScreen("Done.");
MessageBox.Show("Done", "Done");
_readingThread = null;
SetButtonGoText("Start");
}
delegate void SetButtonGoTextDelegate(string text);
/// <summary>
///
/// </summary>
/// <param name="text"></param>
void SetButtonGoText(string text)
{
if (InvokeRequired == true)
{
this.Invoke(new SetButtonGoTextDelegate(SetButtonGoText), text);
return;
}
buttonGo.Text = text;
}
/// <summary>
///
/// </summary>
void DownloadFilesThread()
{
while (true)
{
FileDownloadData fileDownloadData = null;
lock (_filesToDownload)
{
if (_filesToDownload.Count > 0)
{
fileDownloadData = _filesToDownload[0];
_filesToDownload.RemoveAt(0);
}
}
if ((fileDownloadData == null) && (_finishedReadingTree.WaitOne(0, false) == true))
return;
if (fileDownloadData != null)
{
bool retry = true;
while (retry == true)
{
if (_waitingForStop.WaitOne(0, false) == true)
return;
try
{
DownloadFile(fileDownloadData.Url, fileDownloadData.FileName);
retry = false;
}
catch (Exception ex)
{
WriteToScreen("Failed to download: " + ex.Message);
}
}
}
else
{
Thread.Sleep(100);
}
}
}
/// <summary>
///
/// </summary>
public enum RepositoryType
{
SVN,
GIT
}
/// <summary>
///
/// </summary>
/// <param name="baseFolder"></param>
/// <param name="baseUrl"></param>
/// <param name="repositoryType"></param>
void RunSvn(string baseFolder, string baseUrl, RepositoryType repositoryType)
{
if (repositoryType == RepositoryType.SVN)
{
if (baseUrl.EndsWith("/") == false)
baseUrl += "/";
}
if (baseFolder.EndsWith("\\") == false)
baseFolder += "\\";
List<FolderLinkData> urls = new List<FolderLinkData>();
urls.Add(new FolderLinkData(baseUrl, ""));
while (urls.Count > 0)
{
if (_waitingForStop.WaitOne(0, false) == true)
{
WriteToScreen("Stopping...");
lock (_filesToDownload)
{
_filesToDownload.Clear();
}
break;
}
FolderLinkData targetUrlData = urls[0];
string targetUrl = targetUrlData.Url;
urls.RemoveAt(0);
// Create the folder
string relative;
if (targetUrlData.RelativePath == null)
relative = targetUrl.Substring(baseUrl.Length);
else
relative = targetUrlData.RelativePath;
relative = relative.Replace("/", "\\");
string targetFolder = Path.Combine(baseFolder, relative);
if (Directory.Exists(targetFolder) == false)
Directory.CreateDirectory(targetFolder);
// Download target page
string page = null;
bool retry = true;
while (retry == true)
{
if (_waitingForStop.WaitOne(0, false) == true)
return;
try
{
page = DownloadUrl(targetUrl);
retry = false;
}
catch (Exception ex)
{
WriteToScreen("Failed to download: " + ex.Message);
}
}
if (repositoryType == RepositoryType.SVN)
{
List<string> links = ParseLinks(page);
foreach (string link in links)
{
string linkFullUrl = targetUrl + link;
if (linkFullUrl.EndsWith("/") == true)
{
urls.Add(new FolderLinkData(linkFullUrl, null));
}
else // file - download
{
string fileName = targetFolder + link;
lock (_filesToDownload)
{
_filesToDownload.Add(new FileDownloadData(linkFullUrl, fileName));
}
}
}
}
else if (repositoryType == RepositoryType.GIT)
{
List<PageLink> links = ParseGitLinks(page);
int pos = targetUrl.IndexOf("/?");
string serverUrl = targetUrl.Substring(0, pos);
foreach (PageLink link in links)
{
string linkFullUrl = serverUrl + link.Url;
if (link.IsFolder == true)
urls.Add(new FolderLinkData(linkFullUrl, targetUrlData.RelativePath + link.Name + "\\"));
else
{
string fileName = targetFolder + link.Name;
lock (_filesToDownload)
{
_filesToDownload.Add(new FileDownloadData(linkFullUrl, fileName));
}
}
}
}
}
}
/// <summary>
///
/// </summary>
/// <param name="page"></param>
/// <returns></returns>
List<string> ParseLinks(string page)
{
try
{
return ParseLinksFromXml(page);
}
catch
{
return ParseLinksFromHtml(page);
}
}
/// <summary>
///
/// </summary>
/// <param name="page"></param>
/// <returns></returns>
List<string> ParseLinksFromXml(string page)
{
List<string> list = new List<string>();
XmlDocument doc = new XmlDocument();
doc.LoadXml(page);
XmlNode svnNode = doc.SelectSingleNode("/svn");
if (svnNode == null)
throw new Exception("Not a valid SVN xml");
foreach (XmlNode node in doc.SelectNodes("/svn/index/dir"))
{
string dir = node.Attributes["href"].Value;
list.Add(dir);
}
foreach (XmlNode node in doc.SelectNodes("/svn/index/file"))
{
string file = node.Attributes["href"].Value;
list.Add(file);
}
return list;
}
/// <summary>
///
/// </summary>
/// <param name="page"></param>
/// <returns></returns>
List<string> ParseLinksFromHtml(string page)
{
List<string> links = new List<string>();
string listArea = null;
// Find list area: <ul> ... </ul>
int pos = page.IndexOf("<ul>");
if (pos >= 0)
{
int lastPos = page.IndexOf("</ul>", pos);
if (lastPos >= 0)
listArea = page.Substring(pos + 4, lastPos - (pos + 4));
}
if (listArea != null)
{
string[] lines = listArea.Split('\n');
string linePattern = "<a [^>]*>([^<]*)<";
for (int i = 0; i < lines.Length; i++)
{
Match match = Regex.Match(lines[i], linePattern);
if (match.Success == true)
{
string linkRelUrl = match.Groups[1].Value;
if (linkRelUrl != "..")
links.Add(linkRelUrl);
}
}
}
return links;
}
/// <summary>
///
/// </summary>
/// <param name="page"></param>
/// <returns></returns>
List<PageLink> ParseGitLinks(string page)
{
List<PageLink> links = new List<PageLink>();
string dataStartMarker = "<td class=\"mode\">";
string nameMarker = "hb=HEAD\">";
using (StringReader sr = new StringReader(page))
{
string line;
while ((line = sr.ReadLine()) != null)
{
if (line.StartsWith(dataStartMarker) == false)
continue;
bool isFolder = false;
if (line[dataStartMarker.Length] == 'd')
isFolder = true;
line = sr.ReadLine();
// Get name
int pos = line.IndexOf(nameMarker);
int endPos = line.IndexOf("<", pos);
pos += nameMarker.Length;
string name = line.Substring(pos, endPos - pos);
if ((name == "..") || (name == "."))
continue;
// Get URL
pos = line.IndexOf("href=\"");
endPos = line.IndexOf("\">", pos);
pos += "href=\"".Length;
string url = line.Substring(pos, endPos - pos);
if (isFolder == false)
{
url = url.Replace(";a=blob;", ";a=blob_plain;");
pos = url.IndexOf(";h=");
url = url.Substring(0, pos);
url = url + ";hb=HEAD";
}
if (url.Contains(";a=tree;"))
isFolder = true;
links.Add(new PageLink(name, url, isFolder));
}
}
return links;
}
#region Download helper functions
/// <summary>
///
/// </summary>
/// <param name="url"></param>
/// <param name="fileName"></param>
void DownloadFile(string url, string fileName)
{
WriteToScreen("Downloading File: " + url);
WebRequest webRequest = WebRequest.Create(url);
webRequest.ContentType = "text/html; charset=utf-8"; //考虑乱码问题
webRequest.Timeout = 50000;
WebResponse webResponse = null;
Stream responseStream = null;
try
{
webResponse = webRequest.GetResponse();
responseStream = webResponse.GetResponseStream();
// string page = new StreamReader(responseStream, Encoding.UTF8, true).ReadToEnd();
using (FileStream fs = new FileStream(fileName, FileMode.Create))
{
byte[] buffer = new byte[1024];
int readSize;
while ((readSize = responseStream.Read(buffer, 0, buffer.Length)) > 0)
{
fs.Write(buffer, 0, readSize);
}
}
}
finally
{
if (responseStream != null)
responseStream.Close();
if (webResponse != null)
webResponse.Close();
}
}
/// <summary>
///
/// </summary>
/// <param name="url"></param>
/// <returns></returns>
string DownloadUrl(string url)
{
WriteToScreen("Downloading: " + url);
using (WebClient client = new WebClient())
{
client.Encoding = System.Text.Encoding.UTF8; //GetEncoding("utf-8"); //考虑乱码问题
string data = client.DownloadString(url);
return data;
}
}
#endregion
delegate void WriteToScreenDelegate(string str);
/// <summary>
///
/// </summary>
/// <param name="str"></param>
void WriteToScreen(string str)
{
if (this.InvokeRequired)
{
this.Invoke(new WriteToScreenDelegate(WriteToScreen), str);
return;
}
this.richTextBox1.AppendText(str + "\n");
this.richTextBox1.ScrollToCaret();
}
private void buttonClose_Click(object sender, EventArgs e)
{
this.Close();
}
private void Stop()
{
_waitingForStop.Set();
}
private void MainForm_FormClosing(object sender, FormClosingEventArgs e)
{
if (_readingThread != null)
{
Stop();
e.Cancel = true;
}
}
}
public class PageLink
{
string _name;
string _url;
bool _isFolder;
public string Name { get { return _name; } }
public string Url { get { return _url; } }
public bool IsFolder { get { return _isFolder; } }
public PageLink(string name, string url, bool isFolder)
{
_name = name;
_url = url;
_isFolder = isFolder;
}
}
/// <summary>
///
/// </summary>
public class FolderLinkData
{
string _url;
string _relativePath;
public string Url { get { return _url; } }
public string RelativePath { get { return _relativePath; } }
public FolderLinkData(string url, string relativePath)
{
_url = url;
_relativePath = relativePath;
}
}
/// <summary>
///
/// </summary>
public class FileDownloadData
{
string _url;
string _fileName;
public string Url
{
get { return _url; }
}
public string FileName
{
get { return _fileName; }
}
public FileDownloadData(string url, string fileName)
{
_url = url;
_fileName = fileName;
}
}
}