网络爬行者源代码介绍
网络爬行者源代码介绍
import java.awt.*;
import java.awt.event.*;
import java.io.*;
import java.net.*;
import java.util.*;
import java.util.regex.*;
import javax.swing.*;
import javax.swing.table.*;
//一个Web的爬行者(注：爬行在这里的意思与抓取，捕获相同)
public class SearchCrawler extends JFrame{
//最大URL保存值
private static final String[] MAX_URLS={"50","100","500","1000"};
//缓存robot禁止爬行列表
private HashMap disallowListCache=new HashMap();
//搜索GUI控件
private JTextField startTextField;
private JComboBox maxComboBox;
private JCheckBox limitCheckBox;
private JTextField logTextField;
private JTextField searchTextField;
private JCheckBox caseCheckBox;
private JButton searchButton;
//搜索状态GUI控件
private JLabel crawlingLabel2;
private JLabel crawledLabel2;
private JLabel toCrawlLabel2;
private JProgressBar progressBar;
private JLabel matchesLabel2;
//搜索匹配项表格列表
private JTable table;
//标记爬行机器是否正在爬行
private boolean crawling;
//写日志匹配文件的引用
private PrintWriter logFileWriter;
//网络爬行者的构造函数
public SearchCrawler(){
//设置应用程序标题栏
setTitle("搜索爬行者");
//设置窗体大小
setSize(600,600);
//处理窗体关闭事件
addWindowListener(new WindowAdapter(){
public void windowClosing(WindowEvent e){
actionExit();
}
});
//设置文件菜单
JMenuBar menuBar=new JMenuBar();
JMenu fileMenu=new JMenu("文件");
fileMenu.setMnemonic(KeyEvent.VK_F);
JMenuItem fileExitMenuItem=new JMenuItem("退出",KeyEvent.VK_X);
fileExitMenuItem.addActionListener(new ActionListener(){
public void actionPerformed(ActionEvent e){
actionExit();
}
});
fileMenu.add(fileExitMenuItem);
menuBar.add(fileMenu);
setJMenuBar(menuBar);
//设置搜索面板
JPanel searchPanel=new JPanel();
GridBagConstraints constraints;
GridBagLayout layout=new GridBagLayout();
searchPanel.setLayout(layout);
JLabel startLabel=new JLabel("开始URL：");
constraints=new GridBagConstraints();
constraints.anchor=GridBagConstraints.EAST;
constraints.insets=new Insets(5,5,0,0);
layout.setConstraints(startLabel,constraints);
searchPanel.add(startLabel);
startTextField=new JTextField();
constraints=new GridBagConstraints();
constraints.fill=GridBagConstraints.HORIZONTAL;
constraints.gridwidth=GridBagConstraints.REMAINDER;
constraints.insets=new Insets(5,5,0,5);
layout.setConstraints(startTextField,constraints);
searchPanel.add(startTextField);
JLabel maxLabel=new JLabel("最大抓取URL数(0表示不限制)：");
constraints=new GridBagConstraints();
constraints.anchor=GridBagConstraints.EAST;
constraints.insets=new Insets(5,5,0,0);
layout.setConstraints(maxLabel,constraints);
searchPanel.add(maxLabel);
maxComboBox=new JComboBox(MAX_URLS);
maxComboBox.setEditable(true);
constraints=new GridBagConstraints();
constraints.insets=new Insets(5,5,0,0);
layout.setConstraints(maxComboBox,constraints);
searchPanel.add(maxComboBox);
limitCheckBox=new JCheckBox("限制抓取开始URL站点");
constraints=new GridBagConstraints();
constraints.anchor=GridBagConstraints.WEST;
constraints.insets=new Insets(0,10,0,0);
layout.setConstraints(limitCheckBox,constraints);
searchPanel.add(limitCheckBox);
JLabel blankLabel=new JLabel();
constraints=new GridBagConstraints();
constraints.gridwidth=GridBagConstraints.REMAINDER;
layout.setConstraints(blankLabel,constraints);
searchPanel.add(blankLabel);
JLabel logLabel=new JLabel("匹配日志文件：");
constraints=new GridBagConstraints();
constraints.anchor=GridBagConstraints.EAST;
constraints.insets=new Insets(5,5,0,0);
layout.setConstraints(logLabel,constraints);
searchPanel.add(logLabel);
String file=System.getProperty("user.dir")+
System.getProperty("file.separator")+
"crawler.log";
logTextField=new JTextField(file);
constraints=new GridBagConstraints();
constraints.fill=GridBagConstraints.HORIZONTAL;
constraints.gridwidth=GridBagConstraints.REMAINDER;
constraints.insets=new Insets(5,5,0,5);
layout.setConstraints(logTextField,constraints);
searchPanel.add(logTextField);
JLabel searchLabel=new JLabel("搜索字符串：");
constraints=new GridBagConstraints();
constraints.anchor=GridBagConstraints.EAST;
constraints.insets=new Insets(5,5,0,0);
layout.setConstraints(searchLabel,constraints);
searchPanel.add(searchLabel);
searchTextField=new JTextField();
constraints=new GridBagConstraints();
constraints.fill=GridBagConstraints.HORIZONTAL;
constraints.insets=new Insets(5,5,0,0);
constraints.gridwidth=2;
constraints.weightx=1.0d;
layout.setConstraints(searchTextField,constraints);
searchPanel.add(searchTextField);
caseCheckBox=new JCheckBox("大小写敏感");
constraints=new GridBagConstraints();
constraints.insets=new Insets(5,5,0,5);
constraints.gridwidth=GridBagConstraints.REMAINDER;
layout.setConstraints(caseCheckBox,constraints);
searchPanel.add(caseCheckBox);
searchButton=new JButton("搜索");
searchButton.addActionListener(new ActionListener(){
public void actionPerformed(ActionEvent e){
actionSearch();
}
});
constraints=new GridBagConstraints();
constraints.gridwidth=GridBagConstraints.REMAINDER;
constraints.insets=new Insets(5,5,5,5);
layout.setConstraints(searchButton,constraints);
searchPanel.add(searchButton);
JSeparator separator=new JSeparator();
constraints=new GridBagConstraints();
constraints.fill=GridBagConstraints.HORIZONTAL;
constraints.gridwidth=GridBagConstraints.REMAINDER;
constraints.insets=new Insets(5,5,5,5);
layout.setConstraints(separator,constraints);
searchPanel.add(separator);
JLabel crawlingLabel1=new JLabel("爬行：");
constraints=new GridBagConstraints();
constraints.anchor=GridBagConstraints.EAST;
constraints.insets=new Insets(5,5,0,0);
layout.setConstraints(crawlingLabel1,constraints);
searchPanel.add(crawlingLabel1);
crawlingLabel2=new JLabel();
crawlingLabel2.setFont(crawlingLabel2.getFont().deriveFont(Font.PLAIN));
constraints=new GridBagConstraints();
constraints.fill=GridBagConstraints.HORIZONTAL;
constraints.gridwidth=GridBagConstraints.REMAINDER;
constraints.insets=new Insets(5,5,0,5);
layout.setConstraints(crawlingLabel2,constraints);
searchPanel.add(crawlingLabel2);
JLabel crawledLabel1=new JLabel("已抓取的URL数：");
constraints=new GridBagConstraints();
constraints.anchor=GridBagConstraints.EAST;
constraints.insets=new Insets(5,5,0,0);
layout.setConstraints(crawledLabel1,constraints);
searchPanel.add(crawledLabel1);
crawledLabel2=new JLabel();
crawledLabel2.setFont(crawledLabel2.getFont().deriveFont(Font.PLAIN));
constraints=new GridBagConstraints();
constraints.fill=GridBagConstraints.HORIZONTAL;
constraints.gridwidth=GridBagConstraints.REMAINDER;
constraints.insets=new Insets(5,5,0,5);
layout.setConstraints(crawledLabel2,constraints);
searchPanel.add(crawledLabel2);
JLabel toCrawlLabel1=new JLabel("爬行的URL数");
constraints=new GridBagConstraints();
constraints.anchor=GridBagConstraints.EAST;
constraints.insets=new Insets(5,5,0,0);
layout.setConstraints(toCrawlLabel1,constraints);
searchPanel.add(toCrawlLabel1);
toCrawlLabel2=new JLabel();
toCrawlLabel2.setFont(toCrawlLabel2.getFont().deriveFont(Font.PLAIN));
constraints=new GridBagConstraints();
constraints.fill=GridBagConstraints.HORIZONTAL;
constraints.gridwidth=GridBagConstraints.REMAINDER;
constraints.insets=new Insets(5,5,0,5);
layout.setConstraints(toCrawlLabel2,constraints);
searchPanel.add(toCrawlLabel2);
JLabel progressLabel=new JLabel("正在爬行进度：");
constraints=new GridBagConstraints();
constraints.anchor=GridBagConstraints.EAST;
constraints.insets=new Insets(5,5,0,0);
layout.setConstraints(progressLabel,constraints);
searchPanel.add(progressLabel);
progressBar=new JProgressBar();
progressBar.setMinimum(0);
progressBar.setStringPainted(true);
constraints=new GridBagConstraints();
constraints.gridwidth=GridBagConstraints.HORIZONTAL;
constraints.insets=new Insets(5,5,0,5);
layout.setConstraints(progressBar,constraints);
searchPanel.add(progressBar);
JLabel matchesLabel1=new JLabel("搜索匹配：");
constraints=new GridBagConstraints();
constraints.anchor=GridBagConstraints.EAST;
constraints.insets=new Insets(5,5,10,0);
layout.setConstraints(matchesLabel1,constraints);
searchPanel.add(matchesLabel1);
matchesLabel2=new JLabel();
matchesLabel2.setFont(matchesLabel2.getFont().deriveFont(Font.PLAIN));
constraints=new GridBagConstraints();
constraints.fill=GridBagConstraints.HORIZONTAL;
constraints.gridwidth=GridBagConstraints.REMAINDER;
constraints.insets=new Insets(5,5,10,5);
layout.setConstraints(matchesLabel2,constraints);
searchPanel.add(matchesLabel2);
//设置匹配表
table=new JTable(new DefaultTableModel(new Object[][]{},new String[]{"URL"}){
public boolean isCellEditable(int row,int column){
return false;
}
});
//设置匹配面板
JPanel matchesPanel=new JPanel();
matchesPanel.setBorder(BorderFactory.createTitledBorder("匹配"));
matchesPanel.setLayout(new BorderLayout());
matchesPanel.add(new JScrollPane(table),BorderLayout.CENTER);
//把面板添加到窗体上
getContentPane().setLayout(new BorderLayout());
getContentPane().add(searchPanel,BorderLayout.NORTH);
getContentPane().add(matchesPanel,BorderLayout.CENTER);
}
//处理搜索/停止按钮被点到
private void actionSearch(){
//如果停止按钮被点到，爬行标志关闭
if(crawling){
crawling=false;
return;
}
ArrayList errorList=new ArrayList();
//验证起始URL已经输入
String startUrl=startTextField.getText().trim();
if(startUrl.length()<1){
errorList.add("没有起始URL");
}else if(verifyUrl(startUrl)==null){//校验起始URL
errorList.add("非法的起始URL");
}
//校验最大URL数是否为空或者是一个数字
int maxUrls=0;
String max=((String)maxComboBox.getSelectedItem()).trim();
if(max.length()>0){
try{
maxUrls=Integer.parseInt(max);
}catch(NumberFormatException e){
}
if(maxUrls<1){
errorList.add("非法最大URL数值");
}
}
//验证匹配的日志文件已经键入
String logFile=logTextField.getText().trim();
if(logFile.length()<0){
errorList.add("未填写日志文件");
}
//验证搜索字符串已经被键入
String searchString=searchTextField.getText().trim();
if(searchString.length()<1){
errorList.add("未填写搜索字符串");
}
//如果有错，显示这些错误，然后返回
if(errorList.size()>0){
StringBuffer message=new StringBuffer();
//连接所有的错误到一个字符串中
for(int i=0;i<errorList.size();i++){
message.append(errorList.get(i));
if(i+1<errorList.size()){
message.append("/n");
}
}
showError(message.toString());
return;
}
//从起始URL移除"www"
startUrl=removeWwwFromUrl(startUrl);
//启动搜索爬行者
search(logFile,startUrl,maxUrls,searchString);
}
private void search(final String logFile,final String startUrl,
final int maxUrls,final String searchString){
//在一个新线程里开始搜索
Thread thread=new Thread(new Runnable(){
public void run(){
//当搜索正在进行时，换一个等待鼠标
setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
//禁用搜索控制面板
startTextField.setEnabled(false);
maxComboBox.setEnabled(false);
limitCheckBox.setEnabled(false);
logTextField.setEnabled(false);
searchTextField.setEnabled(false);
caseCheckBox.setEnabled(false);
//更改搜索按钮为"停止"
searchButton.setText("停止");
//重设状态
table.setModel(new DefaultTableModel(new Object[][]{},new String[]{"URL"}){
public boolean isCellEditable(int row,int column){
return false;
}
});
updateStats(startUrl,0,0,maxUrls);
//打开匹配日志文件
try{
logFileWriter=new PrintWriter(new FileWriter(logFile));
}catch(Exception e){
showError("不能打开匹配日志文件");
return;
}
//打开正在爬行标志
crawling=true;
//执行真正的爬行
crawl(startUrl,maxUrls,limitCheckBox.isSelected(),searchString,caseCheckBox.isSelected());
//关闭正在爬行标志
crawling=false;
//关闭匹配日志文件
try{
logFileWriter.close();
}catch(Exception e){
showError("不能关闭匹配日志文件");
}
//标记搜索结束
crawlingLabel2.setText("结束");
//重新使搜索面板可用
startTextField.setEnabled(true);
maxComboBox.setEnabled(true);
limitCheckBox.setEnabled(true);
logTextField.setEnabled(true);
searchTextField.setEnabled(true);
caseCheckBox.setEnabled(true);
//将搜索按钮改回"搜索"
searchButton.setText("搜索");
//改回默认的鼠标形状
setCursor(Cursor.getDefaultCursor());
//如果搜索字符串未被发现显示一个信息
if(table.getRowCount()==0){
JOptionPane.showMessageDialog(SearchCrawler.this,"你的搜索字符串未被发现，请尝试其它","搜索字符串未被发现",JOptionPane.WARNING_MESSAGE);
}
}
});
thread.start();
}
//退出程序
private void actionExit(){
System.exit(0);
}
//校验URL格式
private URL verifyUrl(String url ){
//只允许HTTP的URL
if(!url.toLowerCase().startsWith("http://")){
return null;
}
//校验URL的格式
URL verifiedUrl=null;
try{
verifiedUrl=new URL(url);
}catch(Exception e){
return null;
}
return verifiedUrl;
}
//添加匹配到匹配表和日志文件
private void addMatch(String url){
//添加URL到匹配表
DefaultTableModel model=(DefaultTableModel)table.getModel();
model.addRow(new Object[]{url});
//添加URL到日志文件
try{
logFileWriter.println(url);
}catch(Exception e){
showError("未成功的日志匹配");
}
}
//更新爬行中状态
private void updateStats(String crawling,int crawled,int toCrawl,int maxUrls){
crawlingLabel2.setText(crawling);
crawledLabel2.setText(""+crawled);
toCrawlLabel2.setText(""+toCrawl);
//更新进度条
if(maxUrls==-1){
progressBar.setMaximum(crawled+toCrawl);
}else{
progressBar.setMaximum(maxUrls);
}
progressBar.setValue(crawled);
matchesLabel2.setText(""+table.getRowCount());
}
//检查机器人是否允许访问获得的URL
private boolean isRobotAllowed(URL urlToCheck){
String host=urlToCheck.getHost().toLowerCase();
//从缓冲中找回服务器的不被允许列表
ArrayList disallowList=(ArrayList)disallowListCache.get(host);
//如果列表不在名单中，下载将它收入列表
if(disallowList==null){
disallowList=new ArrayList();
try{
URL robotsFileUrl=new URL("http://"+host+"/robots.txt");
//打开并读取robot文件
BufferedReader reader=new BufferedReader(new InputStreamReader(robotsFileUrl.openStream()));
//读robot文件，建立不被允许路径列表
String line;
while((line=reader.readLine())!=null){
if(line.indexOf("Disallow:")==0){
String disallowPath=line.substring("Disallow:".length());
//检查不被允许路径中如果含有注释则去除它
int commentIndex=disallowPath.indexOf("#");
if(commentIndex!=-1){
disallowPath=disallowPath.substring(0,commentIndex);
}
//移除不被允许路径前后空格
disallowPath=disallowPath.trim();
//添加不被允许路径到列表中
disallowList.add(disallowPath);
}
}
}catch(Exception e){
//假设当robot文件不存在时，所有的路径都将被允许爬行
return true;
}
}
//循环检查列表中是否包含给定的URL
String file=urlToCheck.getFile();
for(int i=0;i<disallowList.size();i++){
String disallow=(String)disallowList.get(i);
if(file.startsWith(disallow)){
return false;
}
}
return true;
}
//下载给定的URL页
private String downloadPage(URL pageUrl){
try{
//为读取打开一个到URL的连接
BufferedReader reader=new BufferedReader(new InputStreamReader(pageUrl.openStream()));
//读文件到缓冲中
String line;
StringBuffer pageBuffer=new StringBuffer();
while((line=reader.readLine())!=null){
pageBuffer.append(line);
}
return pageBuffer.toString();
}catch(Exception e){
}
return null;
}
//从一个URL中删除开头的"www"，如果它存在
private String removeWwwFromUrl(String url){
int index=url.indexOf("://www");
if(index!=-1){
return url.substring(0,index+3)+url.substring(index+7);
}
return url;
}
//解析所有的页面内容找到链接
private ArrayList retrieveLinks(URL pageUrl,String pageContents,HashSet crawledList,boolean limitHost){
//编译链接匹配模式
Pattern p=Pattern.compile("<a//s+href//s*=//s*/"?(.*?)[/"|>]",Pattern.CASE_INSENSITIVE);
Matcher m=p.matcher(pageContents);
//建立链接匹配列表
ArrayList linkList=new ArrayList();
while(m.find()){
String link=m.group(1).trim();
//跳过空链接
if(link.length()<1){
continue;
}
//跳过页面锚记链接
if(link.charAt(0)=='#'){
continue;
}
//跳过邮件链接
if(link.indexOf("mailto:")!=-1){
continue;
}
//跳过JavaScript链接
if(link.toLowerCase().indexOf("javascript")!=-1){
continue;
}
//如果需要，加上绝对与相对URL
if(link.indexOf("://")==-1){
//处理绝对URL
if(link.charAt(0)=='/'){
link="http://"+pageUrl.getHost()+link;
//处理相对URL
}else{
String file=pageUrl.getFile();
if(file.indexOf('/')==-1){
link="http://"+pageUrl.getHost()+"/"+link;
}else{
String path=file.substring(0,file.lastIndexOf('/')+1);
link="http://"+pageUrl.getHost()+path+link;
}
}
}
//从链接移除锚记
int index=link.indexOf('#');
if(index!=-1){
link=link.substring(0,index);
}
//去除开头的"www"
link=removeWwwFromUrl(link);
//校验链接，如果非法，则跳过
URL verifiedLink=verifyUrl(link);
if(verifiedLink==null){
continue;
}
//如果是特定的，那些与起始相同的服务器的链接，则跳过
if(limitHost && !pageUrl.getHost().toLowerCase().equals(
verifiedLink.getHost().toLowerCase())){
continue;
}
//如果它已经被捕获，则跳过
if(crawledList.contains(link)){
continue;
}
//添加链接到列表
linkList.add(link);
}
return linkList;
}
//决定获得的页面内容里是否有匹配的字符串
private boolean searchStringMatches(String pageContents,String searchString,boolean caseSensitive){
String searchContents=pageContents;
//如果是非大小写敏感，小写所有页面内容
if(!caseSensitive){
searchContents=pageContents.toLowerCase();
}
//从个别的队列中分隔字符串
Pattern p=Pattern.compile("[//s]+");
String[] terms=p.split(searchString);
//检查每一个队列是否匹配
for(int i=0;i<terms.length;i++){
if(caseSensitive){
if(searchContents.indexOf(terms[i])==-1){
return false;
}
}else{
if(searchContents.indexOf(terms[i].toLowerCase())==-1){
return false;
}
}
}
return false;
}
//执行真正的爬行，搜索搜索字符串
public void crawl(String startUrl,int maxUrls,boolean limitHost,String searchString,boolean caseSensitive){
//设置爬行列表
HashSet crawledList=new HashSet();
LinkedHashSet toCrawlList=new LinkedHashSet();
//添加开始URL到要爬行列表
toCrawlList.add(startUrl);
//循环整个要爬行列表，执行真正的爬行
while(crawling && toCrawlList.size()>0){
//如果指定过最大URL数，则检查是否达到了最大URL数
if(maxUrls!=-1){
if(crawledList.size()==maxUrls){
break;
}
}
//从底部的列表中获得URL
String url=(String)toCrawlList.iterator().next();
//从要爬行列表中移除URL
toCrawlList.remove(url);
//转换字符串URL为URL对象
URL verifiedUrl=verifyUrl(url);
//如果robots不允许访问这个URL，则跳过
if(!isRobotAllowed(verifiedUrl)){
continue;
}
//更新爬行状态
updateStats(url,crawledList.size(),toCrawlList.size(),maxUrls);
//添加页面到被爬行的列表
crawledList.add(url);
//从获得的URL下载页面
String pageContents=downloadPage(verifiedUrl);
//如果一个页面被下载成功，则找到所有的链接并比较是否包含搜索字符串
if(pageContents!=null&&pageContents.length()>0){
//从页面获得合法的链接
ArrayList links=retrieveLinks(verifiedUrl,pageContents,crawledList,limitHost);
//添加列表到被爬行列表
toCrawlList.addAll(links);
//检查搜索字符串是否存在，如果存在，则记录一个匹配
if(searchStringMatches(pageContents,searchString,caseSensitive)){
addMatch(url);
}
}
//更新爬行状态
updateStats(url,crawledList.size(),toCrawlList.size(),maxUrls);
}
}
//显示错误信息
private void showError(String message){
JOptionPane.showMessageDialog(this,message,"错误",JOptionPane.ERROR_MESSAGE);
}
public static void main(String[] args){
SearchCrawler crawler=new SearchCrawler();
crawler.show();
}
}