搜索引擎原理与实践_源程序

9.6 源程序

9.6.1 FrontierSchedulerForBjfu类

package org.archive.crawler.postprocessor;

import org.archive.crawler.datamodel.CandidateURI;

public class FrontierSchedulerForBjfu extends FrontierScheduler { public FrontierSchedulerForBjfu(String name){

super(name);

}

protected void schedule(CandidateURI caUri) {

String uri = caUri.toString();

if(uri.indexOf("dns:") != -1){

getController().getFrontier().schedule(caUri);

}

else if(uri.indexOf("bjfu") != -1

&& (uri.indexOf(".html") != -1

|| uri.indexOf(".htm") != -1

|| uri.indexOf(".jsp") != -1

|| uri.indexOf(".asp") != -1

|| uri.indexOf(".aspx") != -1)){

System.out.println(uri);

getController().getFrontier().schedule(caUri);

}

}

}

9.6.2 Page类

package https://www.360docs.net/doc/c111806954.html,.bjfu.search.page;

public class Page {

private String url;

private String title;

private String summary;

private String context;

private int score;

public Page(){

url = null;

title = null;

summary = null;

context = null;

score = 10;

}

public String getUrl(){

return this.url;

}

public void setUrl(String url){

this.url = url;

}

public String getTitle(){

return this.title;

}

public void setTitle(String title){

this.title = title;

}

public String getSummary(){

return this.summary;

}

public void setSummary(String summary){ this.summary = summary;

}

public String getContext(){

return this.context;

}

public void SetContext(String context){ this.context = context;

}

public int getScore(){

return this.score;

}

public void setScore(int score){

this.score = score;

}

}

9.6.3 Extractor类

package https://www.360docs.net/doc/c111806954.html,.bjfu.search.extractor;

import org.htmlparser.*;

import org.htmlparser.util.*;

import org.htmlparser.visitors.*;

import org.htmlparser.nodes.*;

import org.htmlparser.tags.*;

import https://www.360docs.net/doc/c111806954.html,.bjfu.search.page.*;

import https://www.360docs.net/doc/c111806954.html,.bjfu.search.util.*;

public class Extractor implements Runnable{

private String filename;

private Parser parser;

private Page page;

private String encode;

public void setEncode(String encode){

this.encode = encode;

}

private String combineNodeText(Node[] nodes){

StringBuffer buffer = new StringBuffer();

for(int i = 0; i < nodes.length; i++){

Node anode = (Node)nodes[i];

String line = null;

if(anode instanceof TextNode){

TextNode textnode = (TextNode)anode;

line = textnode.getText();

}

else if (anode instanceof LinkTag){

LinkTag linknode = (LinkTag) anode;

line = linknode.getLinkText();

}

else if (anode instanceof Div){

if(anode.getChildren() != null){

line = combineNodeText(anode.getChildren().toNodeArray());

}

}

else if (anode instanceof ParagraphTag){

if(anode.getChildren() != null){

line = combineNodeText(anode.getChildren().toNodeArray());

}

}

else if (anode instanceof Span){

if(anode.getChildren() != null){

line = combineNodeText(anode.getChildren().toNodeArray());

}

}

else if (anode instanceof TableTag){

if(anode.getChildren() != null){

line = combineNodeText(anode.getChildren().toNodeArray());

}

}

else if (anode instanceof TableRow){

if(anode.getChildren() != null){

line = combineNodeText(anode.getChildren().toNodeArray());

}

}

else if (anode instanceof TableColumn){

if(anode.getChildren() != null){

line = combineNodeText(anode.getChildren().toNodeArray());

}

}

if(line != null){

buffer.append(line);

}

}

return buffer.toString();

}

private String getUrl(String filename){

String url = filename;

url = url.replace(ProperConfig.getValue("mirror.path"), "");

if(https://www.360docs.net/doc/c111806954.html,stIndexOf("/") == url.length() - 1){

url = url.substring(0, url.length() - 1);

}

url = url.substring(1);

return url;

}

private int getScore(String url, int score){

String[] subStr = url.split("/");

score = score - (subStr.length - 1);

return score;

}

private String getSummary(String context){

if(context == null){

context = "";

}

return MD5.MD5Encode(context);

}

public void extract(String filename){

System.out.println("Message: Now extracting " + filename);

this.filename = filename.replace("\\", "/");

run();

if(this.page != null){

PageLib.store(this.page);

}

}

public void run(){

try{

parser = new Parser(this.filename);

parser.setEncoding(encode);

HtmlPage visitor = new HtmlPage(parser);

parser.visitAllNodesWith(visitor);

page = new Page();

// 获取网页的URL

this.page.setUrl(getUrl(this.filename));

// 获取网页的标题

this.page.setTitle(visitor.getTitle());

// 验证网页标签内是否为空，如果是空则不用进行内容提取

if(visitor.getBody() == null){

this.page.SetContext(null);

}

else{

// 如果不为空，则提取内容

this.page.SetContext(combineNodeText(visitor.getBody().toNodeArray()));

}

// 计算网页的得分

this.page.setScore(getScore(this.page.getUrl(), this.page.getScore()));

// 计算网页的摘要

this.page.setSummary(getSummary(this.page.getContext()));

}

catch(ParserException pe){

this.page = null;

pe.printStackTrace();

System.out.println("Continue...");

}

}

}

9.6.4 PageLib类

package https://www.360docs.net/doc/c111806954.html,.bjfu.search.page;

import java.io.BufferedWriter;

import java.io.FileWriter;

import java.io.File;

import java.io.IOException;

import https://www.360docs.net/doc/c111806954.html,.bjfu.search.util.*;

public class PageLib {

public static void store(Page page){

String storepath = ProperConfig.getValue("files.path") + "/" + page.getSummary();

if(new File(storepath).exists() == true){

System.out.println("Message: " + storepath + " is existed!");

return;

}

try{

BufferedWriter writer = new BufferedWriter(new FileWriter(storepath));

// 第一行为URL

writer.append(page.getUrl());

writer.newLine();

// 第二行为标题

writer.append(page.getTitle());

writer.newLine();

// 第三行为得分

writer.append(String.valueOf(page.getScore()));

writer.newLine();

// 第四行为网页内容

writer.append(page.getContext());

// 关闭输出流

writer.close();

}

catch(IOException ioe){

System.out.println("Error: Processing " + page.getUrl() + " accurs error");

ioe.printStackTrace();

}

}

}

9.6.5 PropertyConfig类

package https://www.360docs.net/doc/c111806954.html,.bjfu.search.util;

import java.util.ResourceBundle;

import java.util.MissingResourceException;

public class ProperConfig {

private static String CONFIG_FILE = "config";

private static ResourceBundle bundle;

static {

try{

bundle = ResourceBundle.getBundle(CONFIG_FILE);

}

catch(MissingResourceException mre){

System.out.println("Cannot find config file "

+ CONFIG_FILE + ".properties.");

}

}

public static String getValue(String key){

return bundle.getString(key);

}

}

9.6.6 MD5类

package https://www.360docs.net/doc/c111806954.html,.bjfu.search.util;

import java.security.MessageDigest;

import java.security.NoSuchAlgorithmException;

public class MD5 {

private final static String[] hexDigits = {"0",

"1", "2", "3", "4", "5", "6", "7", "8",

"9", "a", "b", "c", "d", "e", "f"};

public static String byteArrayToHexString(byte[] b) {

StringBuffer resultSb = new StringBuffer();

for (int i = 0; i < b.length; i++) {

resultSb.append(byteToHexString(b[i]));

}

return resultSb.toString();

}

private static String byteToHexString(byte b) {

int n = b;

if (n < 0)

n = 256 + n;

int d1 = n / 16;

int d2 = n % 16;

return hexDigits[d1] + hexDigits[d2];

}

public static String MD5Encode(String origin) {

String resultString = null;

try {

resultString=new String(origin);

MessageDigest md = MessageDigest.getInstance("MD5");

resultString=byteArrayToHexString(md.digest(resultString.getBytes()));

}

catch (NoSuchAlgorithmException nsae) {

System.err.println("No such Algorithm called \"MD5\"!");

}

return resultString;

}

}

9.6.7 IndexBuilder类

package https://www.360docs.net/doc/c111806954.html,.bjfu.search.index;

import org.apache.lucene.document.*;

import org.apache.lucene.index.*;

import jeasy.analysis.*;

import java.io.IOException;

import java.io.FileReader;

import java.io.BufferedReader;

import java.io.File;

public class IndexBuilder {

// IndexWriter

IndexWriter writer;

public IndexBuilder(String path) throws IOException{

writer = new IndexWriter(path, new MMAnalyzer());

}

public void build(String path) throws IOException{

BufferedReader reader = null;

File[] files = new File(path).listFiles();

for(int i = 0; i < files.length; i++){

System.out.print(".");

reader = new BufferedReader(new FileReader(files[i]));

Document doc = new Document();

Field[] fields = new Field[5];

fields[0] = new Field("id", String.valueOf(i), Field.Store.YES, Field.Index.NO);

fields[1] = new Field("url", reader.readLine(), Field.Store.YES, Field.Index.NO);

fields[2] = new Field("title", reader.readLine(), Field.Store.YES, Field.Index.TOKENIZED);

fields[3] = new Field("score", reader.readLine(), Field.Store.YES, Field.Index.NO);

fields[4] = new Field("context", getBodyFile(files[i].getAbsolutePath(), reader), Field.Store.YES, Field.Index.TOKENIZED);

// 创建Document

for(int j = 0; j < fields.length; j++){

doc.add(fields[j]);

}

// 将Document添加至IndexWriter中

writer.addDocument(doc);

}

writer.optimize();

writer.close();

reader.close();

}

private String getBodyFile(String path, BufferedReader reader) throws IOException{ StringBuffer buffer = new StringBuffer();

String line = reader.readLine();

while(line != null){

buffer.append(line);

line = reader.readLine();

}

return buffer.toString();

}

}

9.6.8 index.html

北京林业大学搜索引擎 Search Engine of Beijing Forestry University

高级

9.6.9 soulin.css

.kw {

font-family:Verdana;font-size:16px;height:1.78em;padding-top:2px;

}

.sb {

height:2em;width:5.6em;font-size:14px;

}

.inf {

font-family:Verdana, Arial, Helvetica, sans-serif font:14px;

}

.result {

font-family:Verdana, Arial, Helvetica, sans-serif font:16px;

}

9.6.10 s.jsp

<%@ page language="java" pageEncoding="GB2312"%>

<%

String path = request.getContextPath();

String basePath = request.getScheme()+"://"+request.getServerName()+":"+request.getServerPort()+path+"/";

%>

北京林业大学搜索引擎 Search Engine of Beijing Forestry University

---

Loading...

<%

String param = request.getParameter("keys");

param = new String(param.getBytes("iso-8859-1"), "gb2312");

out.print("您提交的关键字：" + param);

%>

查找结果

<%

// 提取关键字

String keys = CharacterHandle.trans(param);

String[] results = new Query().getQueryResult(keys.split(" "));

// 结果总数

int length = results.length;

// 显示记录的启示序号

int start = 1;

// 显示记录的终止序号

int end = 0;

// 提取显示页数，如果为空，表示从第一页显示

if(request.getParameter("page") != null){

start = Integer.parseInt(request.getParameter("page"));

}

// 计算页数同序号的对应关系

start = (start - 1) * 10;

end = start + 10;

if(end >= length){

end = length;

}

// 从返回的查询结果中提取需要显示的部分

for(int i = start; i < end; i++){

int position = results[i].indexOf("|");

String title = results[i].substring(0, position);

String url = results[i].substring(position + 1, results[i].indexOf("|", position + 1));

position = results[i].indexOf("|", position + 1);

String context = results[i].substring(position + 1);

context = context.replace(" ", "");

context = context.replace(" ", " ");

String summary = "";

String[] k = keys.split(" ");

int maxChar = 100;

int l = k.length;

int summaryStart = 0;

int summaryEnd = 0;

// 单关键字

if(l == 1){

summaryStart = context.indexOf(k[0]) + k[0].length();

summaryEnd = summaryStart + maxChar;

if(summaryEnd > context.length()){

summaryEnd = context.length();

}

summary = context.substring(summaryStart, summaryEnd);

// 将关键字置位红色

summary = "" + k[0] + "" + summary;

}

// 多关键字

else{

int count = maxChar / l;

if(count < 5){

count = 5;

}

for(int j = 0; j < l; j++){

summaryStart = context.indexOf(k[j]) + k[0].length();

summaryEnd = summaryStart + count;

if(summaryEnd > context.length()){

summaryEnd = context.length();

}

if("".equals(summary)){

summary = "" + k[j] + "" + context.substring(summaryStart, summaryEnd);

}

else{

summary = summary + "..." + "" + k[j] + "" + context.substring(summaryStart, summaryEnd);

}

}

}

// 输出查询结果

out.print("" + title + "
");

out.print("" + "" + url + "" + "
");

out.print("" + summary + "


");

}

%>

<%

// 显示页数导航栏

for(int j = 0; j <= length; j += 10)

out.print("[" + (j / 10 + 1 )+ "] ");

%>

<%-- 显示结果--%>

9.6.11 Query类

package https://www.360docs.net/doc/c111806954.html,.bjfu.search.util;

import java.io.*;

import org.apache.lucene.search.*;

import org.apache.lucene.index.Term;

import org.apache.lucene.document.*;

public class Query {

private String INDEX_STORE_PATH = "C:/Program Files/eclipse/workspace/Preprocess/indexes";

private IndexSearcher searcher;

private BooleanQuery query;

public String[] getQueryResult(String[] keys) throws IOException{

searcher = new IndexSearcher(INDEX_STORE_PATH);

query = new BooleanQuery();

if(keys == null){

return null;

}

int length = keys.length;

TermQuery[] term = new TermQuery[length];

for(int i = 0; i < length; i++){

term[i] = new TermQuery(new Term("context", keys[i]));

query.add(term[i], BooleanClause.Occur.MUST);

}

// Sort sort = new Sort(new SortField("score"));

Hits hits = searcher.search(query);

length = hits.length();

String[] result = new String[length];

for(int i = 0; i < length; i++){

Document doc = hits.doc(i);

String tmp = doc.getField("title").stringValue();

tmp = tmp + "|" + doc.getField("url").stringValue();

tmp = tmp + "|" + doc.getField("context").stringValue();

result[i] = tmp;

}

return result;

}

}

9.6.12 CharacterHandle类

package https://www.360docs.net/doc/c111806954.html,.bjfu.search.util;

import java.util.HashMap;

public class CharacterHandle {

private static HashMap map = new HashMap();

public static String trans(String line){

map.put("", " ");

map.put("，", ",");

map.put("。", ".");

map.put("《", "<");

map.put("》", ">");

map.put("＜", "<");

map.put("＞", ">");

map.put("［", "[");

map.put("］", "]");

map.put("？", "?");

map.put("“", "\"");

map.put("”", "\"");

map.put("：", ":");

map.put("；", ";");

map.put("、", ",");

map.put("｛", "{");

map.put("｝", "}");

map.put("（", "(");

map.put("）", ")");

map.put("＋", "+");

map.put("－", "-");

map.put("！", "!");

map.put("１", "1");

map.put("２", "2");

map.put("３", "3");

map.put("４", "4");

map.put("５", "5");

map.put("６", "6");

map.put("７", "7");

map.put("８", "8");

map.put("９", "9");

map.put("０", "0");

for(int i = 0; i < line.length(); i++){

String charat = line.substring(i, i + 1);

if(map.get(charat) != null){

line = line.replace(charat, (String)map.get(charat));

}

}

line = line.toLowerCase();

return line;

}

}