public class Program
{
private static string savePath = System.Environment.CurrentDirectory + Path.DirectorySeparatorChar + "down\\list";
private static string saveBookPath = System.Environment.CurrentDirectory + Path.DirectorySeparatorChar + "down\\book";
private static string saveCoverPath = Path.Combine(saveBookPath, @"cover");
private static int count = 0;
static void Main(string[] args)
{
//依次运行下面的三个方法。1 下载列表页，2 解析列表页 3 抓取书的信息
//DownloadList();
//ParseList();
ParseBook();
Console.Read();
}
static void ParseBook()
{
var minPage = 1;
var maxPage = 496;
for (var page = minPage; page <= maxPage; page++)
{
var bookPath = string.Format("{0}\\book_{1}.html", saveBookPath, page);
ParseBookPage(bookPath);
Thread.Sleep(3000*1);
}
Console.WriteLine("finishing all works!");
}
static void ParseBookPage(string bookPath)
{
var bookInfo = new BookInfo();
var doc = new HtmlDocument();
doc.Load(bookPath);
var rootNode = doc.GetElementbyId("book");
#region bookname
//book name
var bookNameNode = rootNode.SelectSingleNode("//div[@id='name']//h1[1]");
if (bookNameNode != null && bookNameNode.InnerText != null)
{
bookInfo.BookName = bookNameNode.InnerText.Trim();
}
#endregion
#region price
//price
var priceNode = rootNode.SelectSingleNode("//ul[@id='book-price']//li[1]");
var priceText = priceNode.InnerText.Replace("\r\n", "").Trim().Replace(" ","").Replace("定价：￥","");
bookInfo.Price = priceText;
#endregion
#region img
//img
var imgNode = rootNode.SelectSingleNode("//div[@class='jqzoom']//img[1]/@src");
var img = imgNode.Attributes["src"].Value;
var imgPath = DateTime.Now.ToShortDateString().Replace("/",@"\");
var cover = "cover/" + imgPath.Replace(@"\","/");
imgPath = Path.Combine(saveCoverPath, imgPath);
if (!Directory.Exists(imgPath))
{
Directory.CreateDirectory(imgPath);
}
var rand = DateTime.Now.Ticks.ToString() + new Random().Next(1000, 9999).ToString();
var randomName = Helper.GetMd5Hash(rand);
imgPath = Path.Combine(imgPath, randomName + ".jpg");
cover = cover + "/" + randomName + ".jpg";
DownloadPage(img, imgPath);
bookInfo.Cover = cover;
#endregion
#region basic items
var basicItemNodes = rootNode.SelectNodes("//ul[@id='summary']//li");
foreach(var item in basicItemNodes)
{
var text = item.InnerText;
text = text.Replace("\r\n\t\t\t\t", "").Replace(" ", "").Replace("\t\t\t\t", "").Trim();
var arr = text.Split('：');
var key = arr[0];
if (arr.Length > 1)
{
var value = arr[1];
if (key == "作 者")
{
var keyArr = value.Split('著');
bookInfo.Author = keyArr[0].Trim();
if (keyArr.Length > 1)
{
bookInfo.Translator = keyArr[1].Trim();
}
}
if (key == "出 版 社")
{
bookInfo.Press = value;
}
if (key == "ＩＳＢＮ")
{
bookInfo.Isbn = value;
}
if (key == "出版时间")
{
bookInfo.PressDate = value;
}
if (key == "开 本")
{
bookInfo.BookSize = value;
}
if(key == "装 帧")
{
bookInfo.Package = value;
}
if (key == "页 数")
{
var pages = 0;
Int32.TryParse(value,out pages);
bookInfo.Pages = Convert.ToInt32(pages);
}
if (key == "版 次")
{
var keyArr = value.Split('-');
if (keyArr.Length > 0)
{
bookInfo.EditionNum = keyArr[0];
if (keyArr.Length > 1)
{
bookInfo.PressNum = keyArr[1];
}
}
}
}
}
#endregion
#region content
var contentNodes = rootNode.SelectNodes("//div[@class='m m1']");
var contentText = string.Empty;
if (contentNodes != null)
{
foreach (var contentNode in contentNodes)
{
var removeNode = contentNode.SelectSingleNode("div[@class='mt']");
if(removeNode.InnerText != "" && removeNode.InnerText.IndexOf("该作者其它作品") != -1)
{
break;
}
var c = contentNode.InnerHtml.Replace("\r\n", "").Replace("\t", "");
c = c.Replace("<ul class=\"list-h\"> <li>·<a href='http://book.360buy.com/10797010.html' title=\"《人脉是设计出来的》知人性攻人心，最低价！\">《人脉是设计出来的》知人性攻人心，最低价！ >></a></li><li>·<a href='http://book.360buy.com/10901270.html' title=\"《那些年，我们一起追的女孩》65折！\">《那些年，我们一起追的女孩》65折！ >></a></li></ul>", "");
c = c.Replace("<div class=\"more\"><a href=\"javascript:void(0)\">·查看全部>></a></div>",
"");
c = c.Replace("<div class=\"more\" style=\"display: block;\"><a href=\"javascript:void(0)\">·查看全部>></a></div>", "");
contentText += c;
}
bookInfo.Content = contentText;
}
#endregion
count++;
SaveToDb(bookInfo);
Console.WriteLine("finishing parsing {0}...",count);
}
static void ParseList()
{
var minPage = 1;
var maxPage = 34;
for (var page = minPage; page <= maxPage; page++)
{
var pagePath = string.Format("{0}\\list_{1}.html", savePath, page);
ParsePage(pagePath);
}
Console.WriteLine("finishing download pages,total: {0}...", count);
}
static void ParsePage(string pagePath)
{
var doc = new HtmlDocument();
doc.Load(pagePath);
var rootNode = doc.GetElementbyId("plist");
var bookItemNodes = rootNode.SelectNodes("//dt[@class='p-name']/a");
foreach (var bookItem in bookItemNodes)
{
//var title = bookItem.SelectSingleNode("//a[@class='blue14a']");
count++;
var linkStr = bookItem.OuterHtml;
var linkHref = string.Empty;
Regex reg = new Regex(@"(?is)<a[^>]*?href=(['""\s]?)(?<href>[^'""\s]*)\1[^>]*?>");
MatchCollection match = reg.Matches(linkStr);
foreach (Match m in match)
{
linkHref = m.Groups["href"].Value;
}
var info = new BookTitleInfo { id = count, title = bookItem.InnerText, link = linkHref };
//SaveToDb(info);
DownloadPage(linkHref, string.Format("{0}\\book_{1}.html", saveBookPath, count));
Console.WriteLine("downloading book {0}...", count);
Thread.Sleep(1000);
}
}
static void SaveToDb(BookTitleInfo info)
{
var connString = "Server=127.0.0.1;Uid=root;Pwd=pwd;Database=db;CharSet=utf8";
MySqlConnection conn;
MySqlCommand cmd;
conn = new MySqlConnection { ConnectionString = connString };
conn.Open();
cmd = new MySqlCommand
{
Connection = conn,
CommandText =
"insert into `booktitle`(bid,title,link) values(@id,@title,@link)"
};
cmd.Prepare();
cmd.Parameters.Add("@id", MySqlDbType.Int32, 0);
cmd.Parameters.Add("@title", MySqlDbType.VarChar, 255);
cmd.Parameters.Add("@link", MySqlDbType.VarChar, 255);
cmd.Parameters["@id"].Value = info.id;
cmd.Parameters["@title"].Value = info.title;
cmd.Parameters["@link"].Value = info.link;
cmd.ExecuteNonQuery();
}
static void SaveToDb(BookInfo info)
{
if(string.IsNullOrEmpty(info.Author))
{
info.Author = "";
}
if(string.IsNullOrEmpty(info.BookName))
{
info.BookName = "";
}
if(string.IsNullOrEmpty(info.BookSize))
{
info.BookSize = "";
}
if(string.IsNullOrEmpty(info.Content))
{
info.Content = "";
}
if(string.IsNullOrEmpty(info.Cover))
{
info.Cover = "";
}
if(string.IsNullOrEmpty(info.EditionNum))
{
info.EditionNum = "";
}
if(string.IsNullOrEmpty(info.Isbn))
{
info.Isbn = "";
}
if(string.IsNullOrEmpty(info.Language))
{
info.Language = "";
}
if(string.IsNullOrEmpty(info.Package))
{
info.Package = "";
}
if(string.IsNullOrEmpty(info.Paper))
{
info.Paper = "";
}
if(string.IsNullOrEmpty(info.Press))
{
info.Press = "";
}
if(string.IsNullOrEmpty(info.PressDate))
{
info.PressDate = "";
}
if(string.IsNullOrEmpty(info.PressNum))
{
info.PressNum = "";
}
if(string.IsNullOrEmpty(info.Price))
{
info.Price = "";
}
if(string.IsNullOrEmpty(info.Translator))
{
info.Translator = "";
}
var connString = "Server=127.0.0.1;Uid=root;Pwd=pwd;Database=db;CharSet=utf8";
MySqlConnection conn;
MySqlCommand cmd;
conn = new MySqlConnection { ConnectionString = connString };
conn.Open();
cmd = new MySqlCommand
{
Connection = conn,
CommandText =
"insert into `ezdata_book_pendding`(isbn,bookName,author,translator,price,language,press,pressDate,editionNum,pressNum,package,paper,bookSize,pages,words,content,dateline,status,cover)" +
" values(@isbn,@bookName,@author,@translator,@price,@language,@press,@pressDate,@editionNum,@pressNum,@package,@paper,@bookSize,@pages,@words,@content,@dateline,@status,@cover)"
};
cmd.Prepare();
cmd.Parameters.Add("@isbn", MySqlDbType.VarChar, 255);
cmd.Parameters.Add("@bookName", MySqlDbType.VarChar, 255);
cmd.Parameters.Add("@author", MySqlDbType.VarChar, 255);
cmd.Parameters.Add("@translator", MySqlDbType.VarChar, 255);
cmd.Parameters.Add("@price", MySqlDbType.VarChar, 255);
cmd.Parameters.Add("@language", MySqlDbType.VarChar, 255);
cmd.Parameters.Add("@press", MySqlDbType.VarChar, 255);
cmd.Parameters.Add("@pressDate", MySqlDbType.VarChar, 255);
cmd.Parameters.Add("@editionNum", MySqlDbType.VarChar, 255);
cmd.Parameters.Add("@pressNum", MySqlDbType.VarChar, 255);
cmd.Parameters.Add("@package", MySqlDbType.VarChar, 255);
cmd.Parameters.Add("@paper", MySqlDbType.VarChar, 255);
cmd.Parameters.Add("@bookSize", MySqlDbType.VarChar, 255);
cmd.Parameters.Add("@pages", MySqlDbType.Int32, 0);
cmd.Parameters.Add("@words", MySqlDbType.Int32, 0);
cmd.Parameters.Add("@content", MySqlDbType.String, 999999);
cmd.Parameters.Add("@dateline", MySqlDbType.Int32, 0);
cmd.Parameters.Add("@status", MySqlDbType.Int32, 0);
cmd.Parameters.Add("@cover", MySqlDbType.VarChar, 255);
cmd.Parameters["@isbn"].Value = info.Isbn;
cmd.Parameters["@bookName"].Value = info.BookName;
cmd.Parameters["@author"].Value = info.Author;
cmd.Parameters["@translator"].Value = info.Translator;
cmd.Parameters["@price"].Value = info.Price;
cmd.Parameters["@language"].Value = info.Language;
cmd.Parameters["@press"].Value = info.Press;
cmd.Parameters["@pressDate"].Value = info.PressDate;
cmd.Parameters["@editionNum"].Value = info.EditionNum;
cmd.Parameters["@pressNum"].Value = info.PressNum;
cmd.Parameters["@package"].Value = info.Package;
cmd.Parameters["@paper"].Value = info.Paper;
cmd.Parameters["@bookSize"].Value = info.BookSize;
cmd.Parameters["@pages"].Value = info.Pages;
cmd.Parameters["@words"].Value = info.Words;
cmd.Parameters["@content"].Value = info.Content;
var timeStamp = new DateTime(1970, 1, 1);
var a = (DateTime.Now.Ticks - timeStamp.Ticks)/10000000;
cmd.Parameters["@dateline"].Value = a;
cmd.Parameters["@status"].Value = 1;
cmd.Parameters["@cover"].Value = info.Cover;
cmd.ExecuteNonQuery();
}
static void DownloadList()
{
var minPage = 1;
var maxPage = 34;
var urlTemplate = "http://www.360buy.com/products/1713-3278-3638-0-0-0-0-0-0-0-1-1-{0}.html";
var saveFile = string.Empty;
for (var page = minPage; page <= maxPage; page++)
{
saveFile = string.Format("{0}\\list_{1}.html", savePath, page);
DownloadPage(string.Format(urlTemplate, page), saveFile);
Thread.Sleep(3000);
Log(string.Format("downloading list-page {0}", page));
}
Console.WriteLine("finishing download list {0}...", count);
}
static void DownloadPage(string pageUrl, string saveFile)
{
var uri = new Uri(pageUrl);
using (var wc = new CGWebClient())
{
wc.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
try
{
wc.DownloadFile(pageUrl, saveFile);
}catch(Exception ex)
{
Log(ex.Message);
}
}
}
static void Log(string str)
{
Console.WriteLine(str);
using (var f = File.AppendText("log.txt"))
{
f.WriteLine(str);
}
}
}
