使用HtmlParser抓去网页内容

import org.htmlparser.Parser;

import org.htmlparser.beans.StringBean;

importorg.htmlparser.filters.NodeClassFilter;

importorg.htmlparser.parserapplications.StringExtractor;

import org.htmlparser.tags.BodyTag;

import org.htmlparser.util.NodeList;

import org.htmlparser.util.ParserException;

/**

 * 使用HtmlParser抓去网页内容: 要抓去页面的内容最方便的方法就是使用StringBean. 里面有几个控制页面内容的几个参数.

 * 在后面的代码中会有说明. Htmlparser包中还有一个示例StringExtractor 里面有个直接得到内容的方法,

 * 其中也是使用了StringBean . 另外直接解析Parser的每个标签也可以的.

 *

 *@author chenguoyong

 *

 */

public class GetContent {

       publicvoid getContentUsingStringBean(String url) {

              StringBeansb = new StringBean();

              sb.setLinks(true);// 是否显示web页面的连接(Links)

              //为了取得页面的整洁美观一般设置上面两项为true , 如果要保持页面的原有格式, 如代码页面的空格缩进 可以设置为false

              sb.setCollapse(true);// 如果是true的话把一系列空白字符用一个字符替代.

              sb.setReplaceNonBreakingSpaces(true);//If true regular space

              sb

                            .setURL("https://www.blogjava.net/51AOP/archive/2006/07/19/59064.html");

              System.out.println("TheContent is :/n" + sb.getStrings());

       }

       publicvoid getContentUsingStringExtractor(String url, boolean link) {

              //StringExtractor内部机制和上面的一样.做了一下包装

              StringExtractorse = new StringExtractor(url);

              Stringtext = null;

              try{

                     text= se.extractStrings(link);

                     System.out.println("Thecontent is :/n" + text);

              }catch (ParserException e) {

                     e.printStackTrace();

              }

       }

       publicvoid getContentUsingParser(String url) {

              NodeListnl;

              try{

                     Parserp = new Parser(url);

                     nl= p.parse(new NodeClassFilter(BodyTag.class));

                     BodyTagbt = (BodyTag) nl.elementAt(0);

                     System.out.println(bt.toPlainTextString());// 保留原来的内容格式. 包含js代码

              }catch (ParserException e) {

                     e.printStackTrace();

              }

       }

       /**

        * @param args

        */

       publicstatic void main(String[] args) {

              Stringurl = "https://www.blogjava.net/51AOP/archive/2006/07/19/59064.html";

              //newGetContent().getContentUsingParser(url);

              //————————————————–

              newGetContent().getContentUsingStringBean(url);

       }

 https://c.tieba.baidu.com/p/3476776824
https://c.tieba.baidu.com/p/3476808306
https://c.tieba.baidu.com/p/3476798710
https://c.tieba.baidu.com/p/3474281354
https://c.tieba.baidu.com/p/3474300101
https://c.tieba.baidu.com/p/3474294075
https://c.tieba.baidu.com/p/3474123295
https://c.tieba.baidu.com/p/3474314242
https://c.tieba.baidu.com/p/3474310411
https://c.tieba.baidu.com/p/3474304550
https://c.tieba.baidu.com/p/3475433945
https://c.tieba.baidu.com/p/3475430015
https://c.tieba.baidu.com/p/3475433348
https://c.tieba.baidu.com/p/3475431434
https://c.tieba.baidu.com/p/3474176863
https://c.tieba.baidu.com/p/3474159835
https://c.tieba.baidu.com/p/3474163941
https://c.tieba.baidu.com/p/3474156121
https://c.tieba.baidu.com/p/3474147660
https://c.tieba.baidu.com/p/3474151899
https://c.tieba.baidu.com/p/3474142287
https://c.tieba.baidu.com/p/3474136965
https://c.tieba.baidu.com/p/3474133165
https://c.tieba.baidu.com/p/3474128675
https://c.tieba.baidu.com/p/3474103896
https://c.tieba.baidu.com/p/3474099488
https://c.tieba.baidu.com/p/3474094120
https://c.tieba.baidu.com/p/3475431976
https://c.tieba.baidu.com/p/3474267991
https://c.tieba.baidu.com/p/3474259583
https://c.tieba.baidu.com/p/3474254990
https://c.tieba.baidu.com/p/3474228986
https://c.tieba.baidu.com/p/3474221626
https://c.tieba.baidu.com/p/3474215742
https://c.tieba.baidu.com/p/3474212122
https://c.tieba.baidu.com/p/3474188883
https://c.tieba.baidu.com/p/3474207722
https://c.tieba.baidu.com/p/3474184143
https://c.tieba.baidu.com/p/3474180522
https://c.tieba.baidu.com/p/3474171022
https://c.tieba.baidu.com/p/3474086627
https://c.tieba.baidu.com/p/3462847203
https://c.tieba.baidu.com/p/3462839334
https://c.tieba.baidu.com/p/3462834294
https://c.tieba.baidu.com/p/3462786130
https://c.tieba.baidu.com/p/3462782768
https://c.tieba.baidu.com/p/3461791753
https://c.tieba.baidu.com/p/3461784215
https://c.tieba.baidu.com/p/3461778008
https://c.tieba.baidu.com/p/3461772860
https://c.tieba.baidu.com/p/3461767442
https://c.tieba.baidu.com/p/3461736231
https://c.tieba.baidu.com/p/3461704953
https://c.tieba.baidu.com/p/3461692676
https://c.tieba.baidu.com/p/3461665341
https://c.tieba.baidu.com/p/3461656389
https://c.tieba.baidu.com/p/3461660595
https://c.tieba.baidu.com/p/3461566608
https://c.tieba.baidu.com/p/3461652243
https://c.tieba.baidu.com/p/3461561596
https://c.tieba.baidu.com/p/3461557067