hadoop和hbase分布式配置及整合eclipse开发

Hadoop和HBase分布式配置及整合My Eclipse开发Q:620734263说明：本文档主要侧重hadoop和hbase在windows下的开发.在linux上开发自行修改即可.一、下载安装包下载hadoop-0.20.2、hbase-0.90.3及cygwin软件二、安装cygwin和配置sshcygwin的安装google一下就行.三、在主机配置ssh无密码连接在这里找两台主机(仅供测试用),master(192.168.169.121)和slaver1(192.168.169.34)分别在两台主机C:\WINDOWS\system32\drivers\etc\hosts配置如下:至此在master上ssh slaver1就行，第一次要输入密码，系统将slaver1加入到know_hosts 中.以后就不用输入四、配置hadoop现在master上配置如下:在/hadoop-0.20.2/conf/masters、slavers文件中加入主机:配置完成后将此copy到slaver1上去.为了输入简单: 配置环境变量:HADOOP_HOME = e:/hadoop-0.20.2五、启动测试机器如有节点启动不起来的情况:可以先stop-all.sh再删除临时文件/tmp和日志文件logs.再从新格式化节点，重新启动(start-all.sh)主机.六、配置HBaseHBase是什么?官网WIKI,英文看不懂，google翻译…先配置环境变量吧.HBASE_HOME = e:/habse-0.90.3接着取消/cygwin/etc/pofile的注释修改<property><name>hbase.rootdir</name> //如果是在单机下只保留这一项即可<value>hdfs://master:9001/hbase</value> //与hadoop中的core-site.xml配置一直</property><property><name>hbase.cluster.distributed</name><value>true</value></property><property><name>hbase.master</name><value>hdfs://master:60000</value></property><property><name>hbase.zookeeper.quorum</name><value>slaver1</value> //这里配置的主机说是单数，参考wiki </property><property><name>hbase.zookeeper.property.dataDir</name><value>/data/work/zookeeper</value></property>七、启动Hbase八、整合eclipse做开发将hadoop-0.20.2的/hadoop-0.20.2/contrib/eclipse-plugin/hadoop-0.20.2-eclipse-plugin.jar插件以link的方式集成到my eclipise中，具体的操作自己查看.如果有问题，请将my eclipse升级到8.0以后的重启my eclipse将会在右上放工程模式中看到蓝色__ __/ \~~~/ \,----( .. )/ \__ __//| (\ |(^ \ /___\ /\ |新建工程新建map和reduce类，以及M/R驱动类:一个Map/Reduce例子import java.io.IOException;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.*;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;public class NewMaxTemperature {static class NewMaxTemperatureMapper extends Mapper<LongWritable, Text, Text, IntWritable> {// private static final int MISSING = 99;public void map(LongWritable key, Text value, Context context)throws IOException, InterruptedException {String line = value.toString();String year = line.substring(2, 6);int airTemperature;int startIndex = line.indexOf("+");int endIndex = line.indexOf(".");airTemperature = Integer.parseInt(line.substring(startIndex+1, endIndex));// String quality = line.substring(0, 2);// if (airTemperature !=MISSING && quality.matches("[00]")){context.write(new Text(year), new IntWritable(airTemperature));//}}}static class NewMaxTemperatureReducer extends Reducer<Text, IntWritable, Text, IntWritable> {public void reduce(Text key, Iterable<IntWritable> values, Context context)throws IOException, InterruptedException {int maxValue = Integer.MIN_V ALUE;for (IntWritable value : values) {maxValue = Math.max(maxValue, value.get());}context.write(key, new IntWritable(maxValue));}}public static void main(String[] args) throws Exception {if (args.length != 2) {System.err.println("Usage: NewMaxTemperature <input path><output path>");System.exit(-1);}Job job = new Job();job.setJarByClass(NewMaxTemperature.class);FileInputFormat.addInputPath(job, new Path("/user/Administrator/input"));FileOutputFormat.setOutputPath(job, new Path("/user/Administrator/output"));job.setMapperClass(NewMaxTemperatureMapper.class);job.setReducerClass(NewMaxTemperatureReducer.class);job.setOutputKeyClass(Text.class);job.setOutputValueClass(IntWritable.class);System.exit(job.waitForCompletion(true) ? 0 : 1);}}一个HBASE的例子:import java.io.IOException;import java.util.ArrayList;import java.util.List;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.hbase.HBaseConfiguration;import org.apache.hadoop.hbase.HColumnDescriptor;import org.apache.hadoop.hbase.HTableDescriptor;import org.apache.hadoop.hbase.KeyValue;import org.apache.hadoop.hbase.MasterNotRunningException;import org.apache.hadoop.hbase.ZooKeeperConnectionException;import org.apache.hadoop.hbase.client.Delete;import org.apache.hadoop.hbase.client.Get;import org.apache.hadoop.hbase.client.HBaseAdmin;import org.apache.hadoop.hbase.client.HTable;import org.apache.hadoop.hbase.client.Put;import org.apache.hadoop.hbase.client.Result;import org.apache.hadoop.hbase.client.ResultScanner;import org.apache.hadoop.hbase.client.Scan;import org.apache.hadoop.hbase.util.Bytes;public class TestHbase {private static Configuration conf = HBaseConfiguration.create();public void putData(){try {HTable table = new HTable(conf,"test");Put put = new Put(Bytes.toBytes("rows1"));put.add(Bytes.toBytes("id"), Bytes.toBytes("0"), Bytes.toBytes("feng"));table.put(put);} catch (IOException e) {e.printStackTrace();}}public void putData1(){try {HTable table = new HTable(conf,"test");String[] columns = new String[]{"id","age","name"};for(int i=0;i<columns.length;i++){Put put = new Put(Bytes.toBytes("rows"+(i+1)));for(int j=0;j<3;j++){put.add(Bytes.toBytes(columns[j]), Bytes.toBytes(String.valueOf(i)), Bytes.toBytes("n_"+i));table.put(put);}}} catch (IOException e) {e.printStackTrace();}}public void createTable() throws IOException{String tablename = "test";String[] columns = new String[]{"id","age","name"};HBaseAdmin admin = new HBaseAdmin(conf);if (admin.tableExists(tablename)) {System.out.println("表已存在");}else{HTableDescriptor tableDesc = new HTableDescriptor(tablename);for(int i=0;i<columns.length;i++){tableDesc.addFamily(new HColumnDescriptor(columns[i]));}admin.createTable(tableDesc);System.out.println("创建表成功！.");}}public void delete() throws IOException{try {HBaseAdmin admin = new HBaseAdmin(conf);admin.disableTable("scores");admin.deleteTable("scores");System.out.println("表删除成功!");} catch (MasterNotRunningException e) {e.printStackTrace();} catch (ZooKeeperConnectionException e) {e.printStackTrace();}}public void scaner(){try {HTable table = new HTable(conf, "test");Scan s = new Scan();ResultScanner rs = table.getScanner(s);for(Result r:rs){// System.out.println(new String(r.getRow()));// byte[] b = r.getValue(Bytes.toBytes("cf"), Bytes.toBytes("c")); // if(b!=null){// System.out.println(new String(b));// }KeyValue[] kv = r.raw();for(int i=0;i<kv.length;i++){System.out.print(new String(kv[i].getRow())+" ");System.out.print(new String(kv[i].getFamily())+":");System.out.print(new String(kv[i].getQualifier())+" ");System.out.print(kv[i].getTimestamp()+" ");System.out.println(new String(kv[i].getValue()));}}} catch (IOException e) {e.printStackTrace();}}//scan 'test', {COLUMNS => ['id'], CACHE_BLOCKS => false}public void getFamily(){try {HTable table = new HTable(conf, "test");Get get = new Get(Bytes.toBytes("rows1"));Result r = table.get(get);byte[] b = r.getValue(Bytes.toBytes("id"), Bytes.toBytes(1));if(b==null){System.out.println("值不存在!检查簇是不是存在");return;}else{System.out.println(new String(b));}} catch (IOException e) {e.printStackTrace();}}/** 删除某一行数据* */public void deleteRow() throws IOException{HTable table = new HTable(conf, "test");List<Delete> list = new ArrayList<Delete>();Delete d1 = new Delete("rows2".getBytes());list.add(d1);table.delete(list);控制台输出:11/06/26 15:30:16 DEBUG catalog.CatalogTracker: Stopping catalog trackerorg.apache.hadoop.hbase.catalog.CatalogTracker@178460d11/06/26 15:30:16 DEBUG client.HConnectionManager$HConnectionImplementation: Lookedup root region location,connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImpl ementation@2bd3a; hsa=slaver1:6002011/06/26 15:30:16 DEBUG client.HConnectionManager$HConnectionImplementation: Cached location for .META.,,1.1028785192 is slaver1:6002011/06/26 15:30:16 DEBUG client.MetaScanner: Scanning .META. starting atrow=test,,00000000000000 for max=10 rows11/06/26 15:30:16 DEBUG client.HConnectionManager$HConnectionImplementation: Cached location for test,,1309073416281.9e47124da6d9845efd91385ba73c18ee. is slaver1:60020创建表成功！11/06/26 15:30:16 INFO zookeeper.ZooKeeper: Initiating client connection, connectString=192.168.169.34:2181 sessionTimeout=180000 watcher=hconnection11/06/26 15:30:16 INFO zookeeper.ClientCnxn: Opening socket connection to server /192.168.169.34:218111/06/26 15:30:16 INFO zookeeper.ClientCnxn: Socket connection established to slaver1/192.168.169.34:2181, initiating session11/06/26 15:30:16 INFO zookeeper.ClientCnxn: Session establishment complete on server slaver1/192.168.169.34:2181, sessionid = 0x1314670b4d70005, negotiated timeout = 18000011/06/26 15:30:16 DEBUG client.HConnectionManager$HConnectionImplementation: Lookedup root region location,connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImpl ementation@b1aebf; hsa=slaver1:6002011/06/26 15:30:16 DEBUG client.HConnectionManager$HConnectionImplementation: Cached location for .META.,,1.1028785192 is slaver1:6002011/06/26 15:30:16 DEBUG client.MetaScanner: Scanning .META. starting at。