spark中使用redis

代码

代码如下，仅供参考

import org.apache.spark.{SparkConf, SparkContext}
import redis.clients.jedis.Jedis

object UseRedis {
  def main(args: Array[String]): Unit = {
    val sparkConf = new SparkConf().setAppName("intention recognition")
    sparkConf.set("spark.driver.maxResultSize", "4g")
    val sparkContext = new SparkContext(sparkConf)

    val outerIP = "*"
    val globalJedis = new Jedis(outerIP, 6379) //redis服务器的ip，端口默认6379
    globalJedis.flushAll()
    sparkContext
      .textFile(args(0))
      .mapPartitions {
        item => {
          val jedis = new Jedis(outerIP, 6379)
          item.toArray.foreach {
            it =>
              val splits = it.split("\t")
              val app_id = splits(0)
              val app_category2 = splits(2)
              jedis.set(app_id, app_category2)
          }
          item
        }
      }.saveAsTextFile(args(1))
  }
}

参考

参考：

https://www.cnblogs.com/Laymen/p/6122108.html