How can we download data from website using Script task with C# code
/*
We have customer details in CSV files on monthly basis in http site of jcubefinance.com.
We need to download CSV files on the basis of date and create same date folder and load the files.
Example of files in http:
http://jcubefinance.com/customer/2012-01-01/customer.csv,
http://jcubefinance.com/customer/2012-02-01/customer.csv,
http://jcubefinance.com/customer/2012-03-01/customer.csv
*/
using System;
using System.Data;
using Microsoft.SqlServer.Dts.Runtime;
using System.Windows.Forms;
using System.IO;
using System.Web;
using System.Net;
using System.Collections.Generic;
namespace ST_b30a48fe352a48ddbef16d2da49c11ae.csproj
{
[System.AddIn.AddIn("ScriptMain", Version = "1.0", Publisher = "", Description = "")]
public partial class ScriptMain : Microsoft.SqlServer.Dts.Tasks.ScriptTask.VSTARTScriptObjectModelBase
{
#region VSTA generated code
enum ScriptResults
{
Success = Microsoft.SqlServer.Dts.Runtime.DTSExecResult.Success,
Failure = Microsoft.SqlServer.Dts.Runtime.DTSExecResult.Failure
};
#endregion
public void Main()
{
WebClient mySSISWebClient = new WebClient();
mySSISWebClient.Credentials = new NetworkCredential("username", "password"); // Give Credentails username and password
String URI = @"http://jcubefinance.com/customer/"; // Give http site name.
string URIPath = String.Empty;
string URIFullpath = String.Empty;
string sfile1 = @"C:\Test\Test.html";
string sfile2 = @"C:\Test\NewTest.html";
string sfile3 = @"C:\Test\";
mySSISWebClient.DownloadFile(URI, sfile1); // It download data from this http://jcubefinance.com/customer/ website and save into C:\Test\Test.html
//Sample data <a href="2012-01-01/">2012-01-01/</a>
// <a href="2012-02-01/">2012-02-01/</a>
string data = string.Empty;
StreamReader reader = new StreamReader(sfile1); // read the data from sfile1
string line;
int len = 0; ;
int len1 = 0; ;
while ((line = reader.ReadLine()) != null) // read line by line
{
if (line.StartsWith("<a href"))
{
len = line.IndexOf(">");
len = len + 1;
len1 = line.IndexOf("</a>");
data = line.Substring(len, len1 - len); // data : 2012-01-01/
URIPath = URI + data; //URIPath : http://jcubefinance.com/customer/2012-01-01/
mySSISWebClient.DownloadFile(URIPath, sfile2); // It download data from this http://jcubefinance.com/customer/2012-01-01/ website and save into C:\Test\NewTest.html
string path = string.Empty;
path = sfile3 + data;
if (!Directory.Exists(path)) // Creating Date folder in local Server according to Date if directory is not exists. path: C:\Test\2012-01-01
{
Directory.CreateDirectory(path);
}
StreamReader sreader = new StreamReader(sfile2);
string sdata = string.Empty;
string sline;
int slen = 0; ;
int slen1 = 0; ;
while ((sline = sreader.ReadLine()) != null)
{
if (sline.StartsWith("<a href"))
{
slen = sline.IndexOf(">");
slen = slen + 1;
slen1 = sline.IndexOf("</a>");
sdata = sline.Substring(slen, slen1 - slen); //sdata: customer.csv
URIFullpath = URIPath + sdata; //URIFullpath: http://jcubefinance.com/customer/2012-01-01/customer.csv
mySSISWebClient.DownloadFile(URIFullpath, path + sdata); // It download data from this http://jcubefinance.com/customer/2012-01-01/customer.csv website and save into C:\Test\2012-01-01\customer.csv
}
}
sreader.Close(); //close the sfile2
}
}
reader.Close(); // close the sfile1
Dts.TaskResult = (int)ScriptResults.Success;
}
}
}
/*
We have customer details in CSV files on monthly basis in http site of jcubefinance.com.
We need to download CSV files on the basis of date and create same date folder and load the files.
Example of files in http:
http://jcubefinance.com/customer/2012-01-01/customer.csv,
http://jcubefinance.com/customer/2012-02-01/customer.csv,
http://jcubefinance.com/customer/2012-03-01/customer.csv
*/
using System;
using System.Data;
using Microsoft.SqlServer.Dts.Runtime;
using System.Windows.Forms;
using System.IO;
using System.Web;
using System.Net;
using System.Collections.Generic;
namespace ST_b30a48fe352a48ddbef16d2da49c11ae.csproj
{
[System.AddIn.AddIn("ScriptMain", Version = "1.0", Publisher = "", Description = "")]
public partial class ScriptMain : Microsoft.SqlServer.Dts.Tasks.ScriptTask.VSTARTScriptObjectModelBase
{
#region VSTA generated code
enum ScriptResults
{
Success = Microsoft.SqlServer.Dts.Runtime.DTSExecResult.Success,
Failure = Microsoft.SqlServer.Dts.Runtime.DTSExecResult.Failure
};
#endregion
public void Main()
{
WebClient mySSISWebClient = new WebClient();
mySSISWebClient.Credentials = new NetworkCredential("username", "password"); // Give Credentails username and password
String URI = @"http://jcubefinance.com/customer/"; // Give http site name.
string URIPath = String.Empty;
string URIFullpath = String.Empty;
string sfile1 = @"C:\Test\Test.html";
string sfile2 = @"C:\Test\NewTest.html";
string sfile3 = @"C:\Test\";
mySSISWebClient.DownloadFile(URI, sfile1); // It download data from this http://jcubefinance.com/customer/ website and save into C:\Test\Test.html
//Sample data <a href="2012-01-01/">2012-01-01/</a>
// <a href="2012-02-01/">2012-02-01/</a>
string data = string.Empty;
StreamReader reader = new StreamReader(sfile1); // read the data from sfile1
string line;
int len = 0; ;
int len1 = 0; ;
while ((line = reader.ReadLine()) != null) // read line by line
{
if (line.StartsWith("<a href"))
{
len = line.IndexOf(">");
len = len + 1;
len1 = line.IndexOf("</a>");
data = line.Substring(len, len1 - len); // data : 2012-01-01/
URIPath = URI + data; //URIPath : http://jcubefinance.com/customer/2012-01-01/
mySSISWebClient.DownloadFile(URIPath, sfile2); // It download data from this http://jcubefinance.com/customer/2012-01-01/ website and save into C:\Test\NewTest.html
string path = string.Empty;
path = sfile3 + data;
if (!Directory.Exists(path)) // Creating Date folder in local Server according to Date if directory is not exists. path: C:\Test\2012-01-01
{
Directory.CreateDirectory(path);
}
StreamReader sreader = new StreamReader(sfile2);
string sdata = string.Empty;
string sline;
int slen = 0; ;
int slen1 = 0; ;
while ((sline = sreader.ReadLine()) != null)
{
if (sline.StartsWith("<a href"))
{
slen = sline.IndexOf(">");
slen = slen + 1;
slen1 = sline.IndexOf("</a>");
sdata = sline.Substring(slen, slen1 - slen); //sdata: customer.csv
URIFullpath = URIPath + sdata; //URIFullpath: http://jcubefinance.com/customer/2012-01-01/customer.csv
mySSISWebClient.DownloadFile(URIFullpath, path + sdata); // It download data from this http://jcubefinance.com/customer/2012-01-01/customer.csv website and save into C:\Test\2012-01-01\customer.csv
}
}
sreader.Close(); //close the sfile2
}
}
reader.Close(); // close the sfile1
Dts.TaskResult = (int)ScriptResults.Success;
}
}
}
No comments:
Post a Comment