采用多种方式实现词频统计

/** * Created by howard on 2017/8/27. */
object AKKAUtils { def getConf(ip: String, port: String): HashMap[String, Object] = { val conf = new HashMap[String, Object]() val list = new ArrayList[String]() list.add("ty.tcp" "ty.tcp") conf.put("akka.remote.enabled-transports" "akka.remote.enabled-transports", list) conf.put("akka.actor.provider" "akka.actor.provider", "akka.remote.RemoteActorRefProvider") "akka.remote.RemoteActorRefProvider" conf.put("ty.tcp.hostname" "ty.tcp.hostname", ip) conf.put("ty.tcp.port" "ty.tcp.port", port) return conf } }
采用多种方式实现词频统计
采用多种方式实现词频统计 1、利用Java程序实现词频统计 2、利用Scala程序实现词频统计 3、利用Akka和Scala实现词频统计 4、利用MapReduce实现词频统计 5、利用Hive实现词频统计 6、利用Storm实现词频统计 零、要读取的文本文件
一、利用Java程序实现词频统计
public class WordCount { public static void main(String[] args) throws Exception { BufferedReader br = new BufferedReader(new new FileReader("test.txt" "test.txt")); Map<String, Integer> map = new HashMap<>(); String nextLine = ""; "" while ((nextLine = br.readLine()) != null) null { String[] data = nextLine.split(" " "); " for (String word : data) {
WcMapper接收字符串data，调用mapFunc进行处理，返回的是tuple数组arr: Array[(String, Int)，于是，WcReducer接收的参数就是arr: Array[(String, Int)。 （3）创建WcInfo2，封装从WcDriver发往WcMapper的数据
object WcReducer { class ReducerActor extends Actor {
def receive = { case wif2: WcInfo2 => { val arr = wif2.arr val reduceFunc = wif2.reduceFunc val map = reduceFunc(arr)
（6）创建WcReducer
package net.hw.akka.wc import akka.actor.{Actor, ActorSystem, Props} import com.typesafe.config.ConfigFactory
/** * Created by howard on 2017/8/27. */
二、利用scala程序实现词频统计
package net.hw.scala.wc import scala.io.Source
/** * Created by howard on 2017/8/27. */
object WordCount {
def main(args: Array[String]): Unit = { val rx = Source.fromFile("test.txt" "test.txt") .getLines() .toList.mkString(" " ").split(" " " ") " .map((_, 1)) .groupBy(_._1) .mapValues(_.map(_._2)) .mapValues(_.reduce(_ + _)) rx.foreach(println) } }
context.actorSelection("akka.tcp://myAkkaServerSys@127.0.0.1:44442/user/reducerActor" "akka.tcp://myAkkaServerSys@127.0.0.1:44442/user/reducerActor")
! new WcInfo2(arr, reduceFunc) } } } def main(args: Array[String]): Unit = { val sys = ActorSystem("myAkkaServerSys" "myAkkaServerSys", ConfigFactory.parseMap(AKKAUtils.getConf("127.0.0.1" "127.0.0.1", "44443"))); "44443" sys.actorOf(Props[MapperActor], "mapperActor") "mapperActor" } }
（2）创建WcInfo1，封装从WcDriver发往WcMapper的数据
package net.hw.akka.wc
/** * Created by howard on 2017/8/27. */
case class WcInfo1(data: String, mapFunc: String => Array[(String, Int)], reduceFunc: Array[(String, Int)] => Map[String, Int]) { val datax = data val mapFuncx = mapFunc val reduceFuncx = reduceFp.containsKey(word) ? map.get(word) + 1 : 1); } } for (String key : map.keySet()) { System.out .println(key + ": " + map.get(key)); } } }
object WcDriver { def main(args: Array[String]): Unit = { val sys = ActorSystem("myAkkaClientSys" "myAkkaClientSys", ConfigFactory.parseMap(AKKAUtils.getConf("127.0.0.1" "127.0.0.1", "44444"))) "44444" val scan = new Scanner(System.in) while (true true) { val data = scan.nextLine(); val mapFunc = (line: String) => { val arr = line.split(" " ") " arr.map((_, 1)) } val reduceFunc = (arr: Array[(String, Int)]) => { arr.groupBy(_._1).mapValues(_.map(_._2)).mapValues(_.reduce(_ + _)) } sys.actorSelection("akka.tcp://myAkkaServerSys@127.0.0.1:44443/user/mapperActor" "akka.tcp://myAkkaServerSys@127.0.0.1:44443/user/mapperActor") ! new WcInfo1(data, mapFunc, reduceFunc); } } }
package net.hw.akka.wc
/** * Created by howard on 2017/8/27.
*/
case class WcInfo2(arr: Array[(String, Int)], reduceFunc: Array[(String, Int)] => Map[String, Int]) { val arrx = arr val reduceFuncx = reduceFunc }
/** * Created by howard on 2017/8/27. */
object WcMapper { class MapperActor extends Actor { def receive = { case wif1: WcInfo1 => { val line = wif1.data val mapFunc = wif1.mapFunc val reduceFunc = wif1.reduceFunc val arr = mapFunc(line)