7. 用JavaXPCOM来自动浏览网页
?? 在这一节里我们将解决一些自动浏览网页中的一些问题，尝试把一下通用的任务抽象成人类可读的方法，因此你可以轻易的阅读
代码并知道它的功能。我们将构建一个Web Browser来加载网页，点击按钮或者超链接，使用XPath来抽取一下有用的信息。在每一个
小节里面，我们都将在我们的浏览器里增加新的功能，因此在最后，我们将有一个Web Browser，它能够实现网页的自动
浏览。
7.1 使用我们的浏览器来加载网页
??? 在例子 SimpleBrowser里，我们使用方法 public boolean setUrl(String url) 来让浏览器加载一个url。这个方法的问题是它
可以让浏览器开始加载一个页面，但是不会等待浏览器加载页面完成。我们写了一个方法叫做go来实现这个功能，因此后面我们将使
用这种方法来安全的加载一个页面，阻塞住执行流程直到页面加载成功或者超时。
??? 注：实现的方法是在setUrl的时候定义一个CountDownLatch，然后监听browser.addProgressListener，等页面完成后把这个
CountDownLatch countDown一下。调用setUrl后使用CountDownLatch的await方法等待加载完成或者超时。注意调用setUrl时要启动
一个SWT线程??? ???
?????????????? display.syncExec(new Runnable() {
??? ??? ??? public void run() {
??? ??? ??? ??? browser.setUrl(url);
??? ??? ??? }
??? ??? });
??? 另外，我们可能还想获得加载后的页面内容，所有译者增加了一个变量content。增加的代码如下
??? ??? ??? ??? browser.addProgressListener(new ProgressListener() {
??? ??? ??? ??? ??? public void changed(ProgressEvent event) {
??? ??? ??? ??? ??? }
??? ??? ??? ??? ??? public void completed(ProgressEvent event) {
??? ??? ??? ??? ??? ??? // When a page is loaded, decrement the latch,
??? ??? ??? ??? ??? ??? // which count will be 0 after this call.
??? ??? ??? ??? ??? ??? latch.countDown();
??? ??? ??? ??? ??? ??? content=browser.getText();// added by LiLi
??? ??? ??? ??? ??? }
??? ??? ??? ??? });
package es.ladyr.javaxpcom.browser;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import org.eclipse.swt.SWT;
import org.eclipse.swt.browser.ProgressEvent;
import org.eclipse.swt.browser.ProgressListener;
import org.eclipse.swt.widgets.Display;
import org.eclipse.swt.widgets.Shell;
import org.eclipse.swt.browser.Browser;
import org.eclipse.swt.SWTError;
public class SimpleBrowserWithGo {
// We will need SWT display to execute methods
// into the SWT event thread.
Browser browser;
private Display display;
// Latch used to manage page loading
// Uses a count of 1, so when the browser starts loading
// a page, we create a new latch, which will be
// decremented when the page is loaded.
private CountDownLatch latch;
// Default timeout to 60 seconds
private long defaultTimeout = 60000;
/**
* Creates a web browser which is able to load pages waiting until
* the page is completely loaded.
*
*/
public SimpleBrowserWithGo (final String xulrunnerPath) {
// Use a latch to wait for the browser initialization.
final CountDownLatch initLatch = new CountDownLatch(1);
// MozillaBrowser needs a window manager to work. We are using SWT
// for the graphical interface, so we need to execute MozillaBrowser
// methods into the SWT event thread. If we were use another thread,
// that methods could not work properly and throw an exception,
// breaking the execution flow and crashing our application.
new Thread("SWT-Event-Thread") {
@Override
public void run() {
display = new Display();
Shell shell = new Shell(display);
shell.setSize(800, 600);
shell.open();
// If you have XULRunner installed, you can call the constructor without
// the last parameter:
//
// final MozillaBrowser browser = new MozillaBrowser(shell,SWT.BORDER);
//
// That last parameter is the path for XULRunner files
// (where you have uncompressed downloaded XULRunner package).
try {
browser = new Browser(shell, SWT.MOZILLA);
} catch (SWTError e) {
System.out.println("Could not instantiate Browser: " + e.getMessage
());
return;
}
// Adapt browser size to shell size
browser.setBounds(shell.getClientArea());
// Listens for page loading status.
browser.addProgressListener(new ProgressListener() {
public void changed(ProgressEvent event) {
}
public void completed(ProgressEvent event) {
// When a page is loaded, decrement the latch,
// which count will be 0 after this call.
latch.countDown();
}
});
// Realease the initialization latch, which has value 1,
// so after this call its value will be 0.
initLatch.countDown();
while (!shell.isDisposed()) {
if (!display.readAndDispatch()) {
display.sleep();
}
}
System.exit(0);
}
}.start();
try {
// Waits until the initialization latch is released.
initLatch.await();
} catch (InterruptedException e) {
Thread.interrupted();
}
}
/**
* Loads an URL into the browser and waits until the page is
* totally loaded.
* @param url
* @throws SimpleBrowserException
*/
public void go(final String url) throws SimpleBrowserException {
// Creates a latch with count 1
latch = new CountDownLatch(1);
// Uses the SWT event thread to execute the method to
// load an URL in the browser.
display.syncExec(new Runnable() {
public void run() {
browser.setUrl(url);
}
});
// Waits for the finish of the page loading, or for a given
// timeout in case that the loading doesn't finish in a
// reasonable time.
boolean timeout = waitLoad(defaultTimeout);
if (timeout) {
throw new SimpleBrowserException("Timeout waiting page loading.");
}
}
private boolean waitLoad(long millis) {
try {
// Uses the latch, created by 'go' method to wait for
// the finish of the page loading (it will occurs when
// our 'progressListener' receives a event for its method
// 'completed'), or for a given timeout in case that the
// loading doesn't finish in a reasonable time.
boolean timeout;
timeout = !latch.await(millis,TimeUnit.MILLISECONDS);
if (timeout) {
// If the timeout expired, then we will stop
// page loading.
display.syncExec(new Runnable() {
public void run() {
browser.stop();
}
});
// Waits for the loading is stopped
latch.await(millis,TimeUnit.MILLISECONDS);
}
return timeout;
} catch (InterruptedException e) {
throw new Error(e);
}
}
public static void main(String[] args) {
String xulrunnerPath = null;
if ( args.length > 0 ) {
xulrunnerPath = args[0];
}
// Instantiate our simple web browser
SimpleBrowserWithGo simpleBrowser = new SimpleBrowserWithGo(xulrunnerPath);
try{
// Use the new functionality to load some URLs
// with our browser.
simpleBrowser.go("http://www.google.com");
Thread.sleep(3000);
simpleBrowser.go("http://www.urjc.es");
Thread.sleep(3000);
simpleBrowser.go("http://www.mozilla.org");
Thread.sleep(3000);
} catch (SimpleBrowserException e) {
System.err.println("Problems calling go method.");
e.printStackTrace();
} catch (InterruptedException e) {
System.err.println("Problems calling sleep.");
e.printStackTrace();
Thread.interrupted();
}
ime().halt(0);
}
}
?
7.2 解析XPath来获得W3C Node
??? 一旦我们能够在浏览器里加载一个HTML页面，我们可能想访问DOM节点来抽取信息。前面我们花了一节来吧Mozilla Node转换成
W3C node。现在我们使用那个方法来用标准的方法操作W3C Node。我们实现了一些方法来创建XPath Evaluator和XPath resolver来
抽取节点。当Xpath evaluator返回一个结果，我们把每个返回的DOM node转换成响应的W3C DOM element，使用的方法是 static
Node getNodeInstance( nsIDOMNode node ) 。因此，使用我们的browser可以直接调用下面的方法：
...
import org.w3c.dom.Node;
...
/**
*
* @param xpath
* @return a list with the nodes corresponding to a given xpath.
* @throws SimpleBrowserException
*/
public List<Node> xpathNodes(String xpath) {
...
/**
*
* @param <T>
* @param xpath
* @param nodeClass
* @return a list of <code>nodeClass</code> nodes corresponding
* to a given xpath.
* @throws SimpleBrowserException
*/
public <T extends Node> List<T> xpathNodes(String xpath, Class<T> nodeClass) {
...
?
?
下面是完整的例子：
译注：核心代码就是下面这个方法。它有两个参数，xpath和nsIDOMNode,返回的是满足XPath的Node的list。
Node是W3C Node 。其实就是调用xpcom的接口来做xpath解析，然后把nsIDOMNode的List转成W3C Node的list。
不过我感觉其实没有必要，nsIDOMNode的属性更多，而且W3C Node没办法转回去成为nsIDOMNode。在使用XPath的时候，我们可能先
用绝对XPath找到某个table，然后根据相对XPath找tr，td等。第二次调用xPathNodes是的参数 nsIDOMNode context就是第一次返回
的结果里的nsIDOMNode。
private List<Node> xPathNodes(String xpath, nsIDOMNode context) {
// Obtain the Mozilla DOM HTML document
HTMLDocumentImpl documentImpl = (HTMLDocumentImpl) getW3CDocument();
nsIDOMHTMLDocument document = documentImpl.getInstance();
// Creates a name space resolver for the document
nsIDOMXPathNSResolver res = xpathEval.createNSResolver(document);
List<Node> resultNodes = null;
// Evaluates given XPath in a given context, using the resolver created
// for the current document as an ordered iterator
nsISupports obj = xpathEval.evaluate(xpath, context, res,
nsIDOMXPathResult.ORDERED_NODE_ITERATOR_TYPE, null);
// Obtain the interface corresponding to the XPath XPCOM results object
nsIDOMXPathResult result = (nsIDOMXPathResult) obj
.queryInterface(nsIDOMXPathResult.NS_IDOMXPATHRESULT_IID);
try {
// Extract result nodes for the XPath and add them
// to a standard List.
resultNodes = getNodes(result);
} catch (org.mozilla.xpcom.XPCOMException e) {
throw e;
}
return resultNodes;
}
package es.ladyr.javaxpcom.browser;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import org.eclipse.swt.SWT;
import org.eclipse.swt.SWTError;
import org.eclipse.swt.browser.Browser;
import org.eclipse.swt.browser.ProgressEvent;
import org.eclipse.swt.browser.ProgressListener;
import org.eclipse.swt.widgets.Display;
import org.eclipse.swt.widgets.Shell;
import org.mozilla.dom.NodeFactory;
import org.mozilla.dom.html.HTMLDocumentImpl;
import org.mozilla.interfaces.nsIComponentManager;
import org.mozilla.interfaces.nsIDOMDocument;
import org.mozilla.interfaces.nsIDOMHTMLDocument;
import org.mozilla.interfaces.nsIDOMNode;
import org.mozilla.interfaces.nsIDOMWindow;
import org.mozilla.interfaces.nsIDOMXPathEvaluator;
import org.mozilla.interfaces.nsIDOMXPathNSResolver;
import org.mozilla.interfaces.nsIDOMXPathResult;
import org.mozilla.interfaces.nsISupports;
import org.mozilla.interfaces.nsIWebBrowser;
import org.mozilla.xpcom.Mozilla;
import org.w3c.dom.Node;
import org.w3c.dom.html.HTMLAnchorElement;
import org.w3c.dom.html.HTMLDocument;
public class SimpleBrowserWithXPath {
private final static String NS_IDOMXPATHEVALUATOR_CONTRACTID = "@mozilla.org/dom/xpath-evaluator;1";
private Browser browser;
// We will need SWT display to execute methods
// into the SWT event thread.
private Display display;
// Latch used to manage page loading
// Uses a count of 1, so when the browser starts loading
// a page, we create a new latch, which will be
// decremented when the page is loaded.
private CountDownLatch latch;
// Default timeout to 60 seconds
private long defaultTimeout = 60000;
// XPath evaluator
private nsIDOMXPathEvaluator xpathEval;
/**
* Creates a web browser which is able to load pages waiting until
* the page is completely loaded and solve xpaths returning
* the corresponding nodes.
*
*/
public SimpleBrowserWithXPath (final String xulrunnerPath) {
// Use a latch to wait for the browser initialization.
final CountDownLatch initLatch = new CountDownLatch(1);
// MozillaBrowser needs a window manager to work. We are using SWT
// for the graphical interface, so we need to execute MozillaBrowser
// methods into the SWT event thread. If we were use another thread,
// that methods could not work properly and throw an exception,
// breaking the execution flow and crashing our application.
new Thread("SWT-Event-Thread") {
@Override
public void run() {
display = new Display();
Shell shell = new Shell(display);
shell.setSize(800, 600);
shell.open();
// If you have XULRunner installed, you can call the constructor without
// the last parameter:
//
// final MozillaBrowser browser = new MozillaBrowser(shell,SWT.BORDER);
//
// That last parameter is the path for XULRunner files
// (where you have uncompressed downloaded XULRunner package).
try {
browser = new Browser(shell, SWT.MOZILLA);
} catch (SWTError e) {
System.out.println("Could not instantiate Browser: " + e.getMessage
());
return;
}
// Adapt browser size to shell size
browser.setBounds(shell.getClientArea());
// Listens for page loading status.
browser.addProgressListener(new ProgressListener() {
public void changed(ProgressEvent event) {
}
public void completed(ProgressEvent event) {
// When a page is loaded, decrement the latch,
// which count will be 0 after this call.
latch.countDown();
}
});
// Realease the initialization latch, which has value 1,
// so after this call its value will be 0.
initLatch.countDown();
while (!shell.isDisposed()) {
if (!display.readAndDispatch()) {
display.sleep();
}
}
System.exit(0);
}
}.start();
try {
// Waits until the initialization latch is released.
initLatch.await();
} catch (InterruptedException e) {
Thread.interrupted();
}
// Creates the XPath evaluator XPCOM component
Mozilla moz = Mozilla.getInstance();
nsIComponentManager componentManager = moz.getComponentManager();
xpathEval = (nsIDOMXPathEvaluator) componentManager.createInstanceByContractID(
NS_IDOMXPATHEVALUATOR_CONTRACTID, null,
nsIDOMXPathEvaluator.NS_IDOMXPATHEVALUATOR_IID);
}
/**
* Loads an URL into the browser and waits until the page is
* totally loaded.
* @param url
* @throws SimpleBrowserException
*/
public void go(final String url) throws SimpleBrowserException {
// Creates a latch with count 1
latch = new CountDownLatch(1);
// Uses the SWT event thread to execute the method to
// load an URL in the browser.
display.syncExec(new Runnable() {
public void run() {
browser.setUrl(url);
}
});
// Waits for the finish of the page loading, or for a given
// timeout in case that the loading doesn't finish in a
// reasonable time.
boolean timeout = waitLoad(defaultTimeout);
if (timeout) {
throw new SimpleBrowserException("Timeout waiting page loading.");
}
}
/**
*
* @return an W3C HTML Document implementation corresponding to
* the Mozilla DOM HTML document currently loaded in the browser.
* @throws SimpleBrowserException
*/
public HTMLDocument getW3CDocument() {
// System.out.println("El browser es " + browser.toString());
class DocumentGetter implements Runnable {
private nsIDOMHTMLDocument htmldoc;
public void run(){
nsIWebBrowser webBrowser = (nsIWebBrowser)browser.getWebBrowser();
if (webBrowser == null) {
System.out.println("Could not get the nsIWebBrowser from the Browser widget");
}
nsIDOMWindow dw = webBrowser.getContentDOMWindow();
nsIDOMDocument nsDoc = dw.getDocument();
htmldoc = (nsIDOMHTMLDocument) nsDoc
.queryInterface
(nsIDOMHTMLDocument.NS_IDOMHTMLDOCUMENT_IID);
}
public nsIDOMHTMLDocument getHtmldoc() {
// TODO Auto-generated method stub
return htmldoc;
}}
DocumentGetter dg = new DocumentGetter();
display.syncExec(dg);
return HTMLDocumentImpl.getDOMInstance(dg.getHtmldoc());
}
/**
*
* @param xpath
* @return a list with the nodes corresponding to a given xpath.
* @throws SimpleBrowserException
*/
public List<Node> xpathNodes(String xpath) {
return xPathNodes(xpath,
((HTMLDocumentImpl) getW3CDocument()).getInstance());
}
/**
*
* @param <T>
* @param xpath
* @param nodeClass
* @return a list of <code>nodeClass</code> nodes corresponding
* to a given xpath.
* @throws SimpleBrowserException
*/
public <T extends Node> List<T> xpathNodes(String xpath, Class<T> nodeClass) {
return (List<T>)xPathNodes(xpath,
((HTMLDocumentImpl) getW3CDocument()).getInstance());
}
private boolean waitLoad(long millis) {
try {
// Uses the latch, created by 'go' method to wait for
// the finish of the page loading (it will occurs when
// our 'progressListener' receives a event for its method
// 'completed'), or for a given timeout in case that the
// loading doesn't finish in a reasonable time.
boolean timeout;
timeout = !latch.await(millis,TimeUnit.MILLISECONDS);
if (timeout) {
// If the timeout expired, then we will stop
// page loading.
display.syncExec(new Runnable() {
public void run() {
browser.stop();
}
});
// Waits for the loading is stopped
latch.await(millis,TimeUnit.MILLISECONDS);
}
return timeout;
} catch (InterruptedException e) {
throw new Error(e);
}
}
private List<Node> xPathNodes(String xpath, nsIDOMNode context) {
// Obtain the Mozilla DOM HTML document
HTMLDocumentImpl documentImpl = (HTMLDocumentImpl) getW3CDocument();
nsIDOMHTMLDocument document = documentImpl.getInstance();
// Creates a name space resolver for the document
nsIDOMXPathNSResolver res = xpathEval.createNSResolver(document);
List<Node> resultNodes = null;
// Evaluates given XPath in a given context, using the resolver created
// for the current document as an ordered iterator
nsISupports obj = xpathEval.evaluate(xpath, context, res,
nsIDOMXPathResult.ORDERED_NODE_ITERATOR_TYPE, null);
// Obtain the interface corresponding to the XPath XPCOM results object
nsIDOMXPathResult result = (nsIDOMXPathResult) obj.queryInterface(
nsIDOMXPathResult.NS_IDOMXPATHRESULT_IID);
try {
// Extract result nodes for the XPath and add them
// to a standard List.
resultNodes = getNodes(result);
} catch(org.mozilla.xpcom.XPCOMException e){
throw e;
}
return resultNodes;
}
private <T> List<T> getNodes(nsIDOMXPathResult result) {
List<T> nodes = new ArrayList<T>();
nsIDOMNode node;
while((node = result.iterateNext()) != null){
// Use the functionality provided by the mozdom4java
// (in our case, patched) library to obtain the corresponding
// W3C implementation of a node.
nodes.add((T)NodeFactory.getNodeInstance(node));
}
return nodes;
}
public static void main(String[] args) {
String xulrunnerPath = null;
if ( args.length > 0 ) {
xulrunnerPath = args[0];
}
// Instantiate our simple web browser
SimpleBrowserWithXPath simpleBrowser = new SimpleBrowserWithXPath(xulrunnerPath);
try{
// Load a page in the browser
simpleBrowser.go("http://www.google.com");
// Obtain a list of nodes, without a concrete class,
// because the XPath may return nodes of different
// types, so we work with them in a generic way.
List<Node> nodes = simpleBrowser.xpathNodes("//*");
for (Node node: nodes) {
System.out.println("Node Type: " + node.getNodeName()
+ " -- Content: " + node.getTextContent());
}
// Obtain a list of HTMLAnchorElements, because
// we can be sure about the result of our XPath,
// if it has any result, will be only of
// HTMLAnchorElement type.
for (HTMLAnchorElement a: simpleBrowser.xpathNodes(
"//a", HTMLAnchorElement.class)) {
System.out.println("Anchor: " + a.getHref());
}
} catch (SimpleBrowserException e) {
System.err.println("Problems calling go method.");
e.printStackTrace();
}
Runtime.getRuntime().halt(0);
}
}
?