本期內(nèi)容：

1、在線動態(tài)計算分類最熱門商品案例回顧與演示
2、基于案例貫通Spark Streaming的運行源碼

第一部分案例：

package com.dt.spark.sparkstreaming
import com.robinspark.utils.ConnectionPool
import org.apache.spark.SparkConf
import org.apache.spark.sql.Row
import org.apache.spark.sql.hive.HiveContext
import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType}
import org.apache.spark.streaming.{Seconds, StreamingContext}

/**
  * 使用Spark Streaming+Spark SQL來在線動態(tài)計算電商中不同類別中最熱門的商品排名，例如手機這個類別下面最熱門的三種手機、電視這個類別
  * 下最熱門的三種電視，該實例在實際生產(chǎn)環(huán)境下具有非常重大的意義；
  *
  *
  *
  *   實現(xiàn)技術(shù)：Spark Streaming+Spark SQL，之所以Spark Streaming能夠使用ML、sql、graphx等功能是因為有foreachRDD和Transform
  * 等接口，這些接口中其實是基于RDD進行操作，所以以RDD為基石，就可以直接使用Spark其它所有的功能，就像直接調(diào)用API一樣簡單。
  *  假設(shè)說這里的數(shù)據(jù)的格式：user item category，例如Rocky Samsung Android
  */
object OnlineTheTop3ItemForEachCategory2DB {
  def main(args: Array[String]){
    /**
      * 第1步：創(chuàng)建Spark的配置對象SparkConf，設(shè)置Spark程序的運行時的配置信息，
      * 例如說通過setMaster來設(shè)置程序要鏈接的Spark集群的Master的URL,如果設(shè)置
      * 為local，則代表Spark程序在本地運行，特別適合于機器配置條件非常差（例如
      * 只有1G的內(nèi)存）的初學(xué)者       *
      */
    val conf = new SparkConf() //創(chuàng)建SparkConf對象
    conf.setAppName("OnlineTheTop3ItemForEachCategory2DB") //設(shè)置應(yīng)用程序的名稱，在程序運行的監(jiān)控界面可以看到名稱
    conf.setMaster("spark://Master:7077") //此時，程序在Spark集群
    //conf.setMaster("local[2]")
    //設(shè)置batchDuration時間間隔來控制Job生成的頻率并且創(chuàng)建Spark Streaming執(zhí)行的入口
    val ssc = new StreamingContext(conf, Seconds(5))

    ssc.checkpoint("/root/Documents/SparkApps/checkpoint")

    val userClickLogsDStream = ssc.socketTextStream("Master", 9999)

    val formattedUserClickLogsDStream = userClickLogsDStream.map(clickLog =>
        (clickLog.split(" ")(2) + "_" + clickLog.split(" ")(1), 1))

//    val categoryUserClickLogsDStream = formattedUserClickLogsDStream.reduceByKeyAndWindow((v1:Int, v2: Int) => v1 + v2,
//      (v1:Int, v2: Int) => v1 - v2, Seconds(60), Seconds(20))

    val categoryUserClickLogsDStream = formattedUserClickLogsDStream.reduceByKeyAndWindow(_+_,
      _-_, Seconds(60), Seconds(20))

    categoryUserClickLogsDStream.foreachRDD { rdd => {
      if (rdd.isEmpty()) {
        println("No data inputted!!!")
      } else {
        val categoryItemRow = rdd.map(reducedItem => {
          val category = reducedItem._1.split("_")(0)
          val item = reducedItem._1.split("_")(1)
          val click_count = reducedItem._2
          Row(category, item, click_count)
        })

        val structType = StructType(Array(
          StructField("category", StringType, true),
          StructField("item", StringType, true),
          StructField("click_count", IntegerType, true)
        ))

        val hiveContext = new HiveContext(rdd.context)
        val categoryItemDF = hiveContext.createDataFrame(categoryItemRow, structType)

        categoryItemDF.registerTempTable("categoryItemTable")

        val reseltDataFram = hiveContext.sql("SELECT category,item,click_count FROM (SELECT category,item,click_count,row_number()" +
          " OVER (PARTITION BY category ORDER BY click_count DESC) rank FROM categoryItemTable) subquery " +
          " WHERE rank <= 3")
        reseltDataFram.show()

        val resultRowRDD = reseltDataFram.rdd

        resultRowRDD.foreachPartition { partitionOfRecords => {

          if (partitionOfRecords.isEmpty){
            println("This RDD is not null but partition is null")
          } else {
            // ConnectionPool is a static, lazily initialized pool of connections
            val connection = ConnectionPool.getConnection()
            partitionOfRecords.foreach(record => {
              val sql = "insert into categorytop3(category,item,client_count) values('" + record.getAs("category") + "','" +
                record.getAs("item") + "'," + record.getAs("click_count") + ")"
              val stmt = connection.createStatement();
              stmt.executeUpdate(sql);

            })
            ConnectionPool.returnConnection(connection) // return to the pool for future reuse

          }
        }
        }
      }
    }
    }
    /**
      * 在StreamingContext調(diào)用start方法的內(nèi)部其實是會啟動JobScheduler的Start方法，進行消息循環(huán)，在JobScheduler
      * 的start內(nèi)部會構(gòu)造JobGenerator和ReceiverTacker，并且調(diào)用JobGenerator和ReceiverTacker的start方法：
      *   1，JobGenerator啟動后會不斷的根據(jù)batchDuration生成一個個的Job
      *   2，ReceiverTracker啟動后首先在Spark Cluster中啟動Receiver（其實是在Executor中先啟動ReceiverSupervisor），在Receiver收到
      *   數(shù)據(jù)后會通過ReceiverSupervisor存儲到Executor并且把數(shù)據(jù)的Metadata信息發(fā)送給Driver中的ReceiverTracker，在ReceiverTracker
      *   內(nèi)部會通過ReceivedBlockTracker來管理接受到的元數(shù)據(jù)信息
      * 每個BatchInterval會產(chǎn)生一個具體的Job，其實這里的Job不是Spark Core中所指的Job，它只是基于DStreamGraph而生成的RDD
      * 的DAG而已，從Java角度講，相當(dāng)于Runnable接口實例，此時要想運行Job需要提交給JobScheduler，在JobScheduler中通過線程池的方式找到一個
      * 單獨的線程來提交Job到集群運行（其實是在線程中基于RDD的Action觸發(fā)真正的作業(yè)的運行），為什么使用線程池呢？
      *   1，作業(yè)不斷生成，所以為了提升效率，我們需要線程池；這和在Executor中通過線程池執(zhí)行Task有異曲同工之妙；
      *   2，有可能設(shè)置了Job的FAIR公平調(diào)度的方式，這個時候也需要多線程的支持；
      *
      */
    ssc.start()
    ssc.awaitTermination()
  }
}

第二部分源碼解析：

1. 構(gòu)建StreamingContext時傳遞SparkConf參數(shù)（或者自己Configuration）在內(nèi)部創(chuàng)建SparkContext

def this(conf: SparkConf, batchDuration: Duration) = {
  this(StreamingContext.createNewSparkContext(conf), null, batchDuration)
}

2. 事實說明SparkStreaming就是SparkCore上的一個應(yīng)用程序

private[streaming] def createNewSparkContext(conf: SparkConf): SparkContext = {
  new SparkContext(conf)
}

3. 創(chuàng)建Socket輸入流

def socketTextStream(
    hostname: String,
    port: Int,
    storageLevel: StorageLevel = StorageLevel.MEMORY_AND_DISK_SER_2
  ): ReceiverInputDStream[String] = withNamedScope("socket text stream") {
  socketStream[String](hostname, port, SocketReceiver.bytesToLines, storageLevel)
}

4. 創(chuàng)建SocketInputDStream

def socketStream[T: ClassTag](
    hostname: String,
    port: Int,
    converter: (InputStream) => Iterator[T],
    storageLevel: StorageLevel
  ): ReceiverInputDStream[T] = {
  new SocketInputDStream[T](this, hostname, port, converter, storageLevel)
}

5. SocketInputDstream繼承ReceiverInputDStream，通過構(gòu)建Receiver來接收數(shù)據(jù)

private[streaming]
class SocketInputDStream[T: ClassTag](
    ssc_ : StreamingContext,
    host: String,
    port: Int,
    bytesToObjects: InputStream => Iterator[T],
    storageLevel: StorageLevel
  ) extends ReceiverInputDStream[T](ssc_) {

  def getReceiver(): Receiver[T] = {
    new SocketReceiver(host, port, bytesToObjects, storageLevel)
  }
}

5.1

  ReceiverInputDStream
  abstract class ReceiverInputDStream[T: ClassTag](ssc_ :   StreamingContext)
    extends InputDStream[T](ssc_) {

  abstract class InputDStream[T: ClassTag] (ssc_ : StreamingContext)
    extends DStream[T](ssc_) {

5.2. DStream

    - 依賴于其他DStream
    -  什么時候依據(jù)DStream，依賴關(guān)系的模板，構(gòu)成RDD之間的依賴
    -  基于DStream它有一個Function，F(xiàn)unction 基于Batch     Interval（time Interval）生成RDD，這個和定時器有關(guān)系

  abstract class DStream[T: ClassTag] (
     @transient private[streaming] var ssc: StreamingContext
) extends Serializable with Logging {

6. SocketReceiver對象在onStart中創(chuàng)建Thread啟動run方法調(diào)用執(zhí)行receive接收數(shù)據(jù)。
   def onStart() {
   // Start the thread that receives data over a connection
   new Thread("Socket Receiver") {
   setDaemon(true)
   override def run() { receive() }
   }.start()
   }

7. 創(chuàng)建一個Socket connection連接接收數(shù)據(jù)

/** Create a socket connection and receive data until receiver is stopped */
  def receive() {
    var socket: Socket = null
    try {
      logInfo("Connecting to " + host + ":" + port)
      socket = new Socket(host, port)
      logInfo("Connected to " + host + ":" + port)
      val iterator = bytesToObjects(socket.getInputStream())
      while(!isStopped && iterator.hasNext) {
        store(iterator.next)
      }
      if (!isStopped()) {
        restart("Socket data stream had no more data")
      } else {
        logInfo("Stopped receiving")
      }

8. 總體流程：在StreamingContext調(diào)用start方法的內(nèi)部其實是會啟動JobScheduler的Start方法，進行消息循環(huán)，在JobScheduler的start內(nèi)部會構(gòu)造JobGenerator和ReceiverTacker，并且調(diào)用JobGenerator和ReceiverTacker的start方法：
   1）、JobGenerator啟動后會不斷的根據(jù)batchDuration生成一個個的Job
   2）、ReceiverTracker啟動后首先在Spark Cluster中啟動Receiver（其實是在Executor中先啟動ReceiverSupervisor），
   在Receiver收到 數(shù)據(jù)后會通過ReceiverSupervisor存儲到Executor并且把數(shù)據(jù)的Metadata信息發(fā)送給Driver中的ReceiverTracker，
   在ReceiverTracker 內(nèi)部會通過ReceivedBlockTracker來管理接受到的元數(shù)據(jù)信息 每個BatchInterval會產(chǎn)生一個具體的Job（這里的Job主要是封裝了業(yè)務(wù)邏輯例如上面實例中的代碼），其實這里的Job不是Spark Core中所指的Job，它只是基于DStreamGraph而生成的RDD 的DAG而已，
   從Java角度講，相當(dāng)于Runnable接口實例，此時要想運行Job需要提交給JobScheduler，在JobScheduler中通過線程池的方式找到一個 單獨的線程來提交Job到集群運行（其實是在線程中基于RDD的Action觸發(fā)真正的作業(yè)的運行）,
   為什么使用線程池呢？
   a）、作業(yè)不斷生成，所以為了提升效率，我們需要線程池；這和在Executor中通過線程池執(zhí)行Task有異曲同工之妙；
   b）、有可能設(shè)置了Job的FAIR公平調(diào)度的方式，這個時候也需要多線程的支持；

8.1、StreamingContext.start

// Start the streaming scheduler in a new thread, so that thread local properties
// like call sites and job groups can be reset without affecting those of the
// current thread.
ThreadUtils.runInNewThread("streaming-start") {
  sparkContext.setCallSite(startSite.get)
  sparkContext.clearJobGroup()
  sparkContext.setLocalProperty(SparkContext.SPARK_JOB_INTERRUPT_ON_CANCEL, "false")
  scheduler.start()
}

補充：線程本地存儲，線程ThreadLocal每個線程有自己的私有屬性，設(shè)置線程的私有屬性不會影響當(dāng)前線程或其他線程

9. JobScheduler.start 創(chuàng)建EventLoop消息線程并啟動

def start(): Unit = synchronized {
  if (eventLoop != null) return // scheduler has already been started

  logDebug("Starting JobScheduler")
  eventLoop = new EventLoop[JobSchedulerEvent]("JobScheduler") {
    override protected def onReceive(event: JobSchedulerEvent): Unit = processEvent(event)

    override protected def onError(e: Throwable): Unit = reportError("Error in job scheduler", e)
  }
  eventLoop.start()

9.1 EventLoop中創(chuàng)建Thread線程接收和發(fā)送消息，調(diào)用JobScheduler中的processEvent方法

private[spark] abstract class EventLoop[E](name: String) extends Logging {

  private val eventQueue: BlockingQueue[E] = new LinkedBlockingDeque[E]()

  private val stopped = new AtomicBoolean(false)

  private val eventThread = new Thread(name) {
    setDaemon(true)

    override def run(): Unit = {
      try {
        while (!stopped.get) {
          val event = eventQueue.take()
          try {
            onReceive(event)
          } catch {

9.2 會接受不同的任務(wù)，JobScheduler是整個Job的調(diào)度器，它本身用了一個線程循環(huán)，去監(jiān)聽不同的Job啟動、Job完成、Job失敗等任務(wù)（消息驅(qū)動系統(tǒng)）
private def processEvent(event: JobSchedulerEvent) {
try {
event match {
case JobStarted(job, startTime) => handleJobStart(job, startTime)
case JobCompleted(job, completedTime) => handleJobCompletion(job, completedTime)
case ErrorReported(m, e) => handleError(m, e)
}
} catch {

10. JobScheduler.start

// attach rate controllers of input streams to receive batch completion updates
for {
  inputDStream <- ssc.graph.getInputStreams
  rateController <- inputDStream.rateController
} ssc.addStreamingListener(rateController)

10.1 多個InputStream
``
inputDStream <- ssc.graph.getInputStreams

  10.2 RateController控制輸入的速度
// Keep track of the freshest rate for this stream using the rateEstimator
protected[streaming] val rateController: Option[RateController] = None

11. JobScheduler.start

listenerBus.start(ssc.sparkContext)
receiverTracker = new ReceiverTracker(ssc)
inputInfoTracker = new InputInfoTracker(ssc)
receiverTracker.start()
jobGenerator.start()

11.1 StreamingListenerBus

override def onPostEvent(listener: StreamingListener, event: StreamingListenerEvent): Unit = {
event match {
case receiverStarted: StreamingListenerReceiverStarted =>
listener.onReceiverStarted(receiverStarted)
case receiverError: StreamingListenerReceiverError =>
listener.onReceiverError(receiverError)
case receiverStopped: StreamingListenerReceiverStopped =>
listener.onReceiverStopped(receiverStopped)
case batchSubmitted: StreamingListenerBatchSubmitted =>
listener.onBatchSubmitted(batchSubmitted)
case batchStarted: StreamingListenerBatchStarted =>
listener.onBatchStarted(batchStarted)
case batchCompleted: StreamingListenerBatchCompleted =>
listener.onBatchCompleted(batchCompleted)
case outputOperationStarted: StreamingListenerOutputOperationStarted =>
listener.onOutputOperationStarted(outputOperationStarted)
case outputOperationCompleted: StreamingListenerOutputOperationCompleted =>
listener.onOutputOperationCompleted(outputOperationCompleted)
case _ =>
}
}

  11.2 receiverTracker.start()，ReceiveTracker是通過發(fā)Job的方式到集群的Executor上啟動Receiver

/** Start the endpoint and receiver execution thread. */
def start(): Unit = synchronized {
if (isTrackerStarted) {
throw new SparkException("ReceiverTracker already started")
}

if (!receiverInputStreams.isEmpty) {
endpoint = ssc.env.rpcEnv.setupEndpoint(
"ReceiverTracker", new ReceiverTrackerEndpoint(ssc.env.rpcEnv))
if (!skipReceiverLaunch) launchReceivers()
logInfo("ReceiverTracker started")
trackerState = Started
}
}

11.2.1、創(chuàng)建一個ReceiverTrackerEndpoint消息通信體

override def receive: PartialFunction[Any, Unit] = {
// Local messages
case StartAllReceivers(receivers) =>
val scheduledLocations = schedulingPolicy.scheduleReceivers(receivers, getExecutors)
for (receiver <- receivers) {
val executors = scheduledLocations(receiver.streamId)
updateReceiverScheduledExecutors(receiver.streamId, executors)
receiverPreferredLocations(receiver.streamId) = receiver.preferredLocation
startReceiver(receiver, executors)
}
case RestartReceiver(receiver) =>
// Old scheduled executors minus the ones that are not active any more
val oldScheduledExecutors = getStoredScheduledExecutors(receiver.streamId)
val scheduledLocations = if (oldScheduledExecutors.nonEmpty) {
// Try global scheduling again
oldScheduledExecutors
} else {
val oldReceiverInfo = receiverTrackingInfos(receiver.streamId)
// Clear "scheduledLocations" to indicate we are going to do local scheduling
val newReceiverInfo = oldReceiverInfo.copy(
state = ReceiverState.INACTIVE, scheduledLocations = None)
receiverTrackingInfos(receiver.streamId) = newReceiverInfo
schedulingPolicy.rescheduleReceiver(
receiver.streamId,
receiver.preferredLocation,
receiverTrackingInfos,
getExecutors)
}
// Assume there is one receiver restarting at one time, so we don't need to update
// receiverTrackingInfos
startReceiver(receiver, scheduledLocations)

11.2.1.1、ReceiverSchedulingPolicy.scheduleReceivers，從下面的代碼中可以看出來在那些Executor上啟動Receiver,以及怎么具體在Executor上啟動Receiver

// Firstly, we need to respect "preferredLocation". So if a receiver has "preferredLocation",
// we need to make sure the "preferredLocation" is in the candidate scheduled executor list.
for (i <- 0 until receivers.length) {
// Note: preferredLocation is host but executors are host_executorId
receivers(i).preferredLocation.foreach { host =>
hostToExecutors.get(host) match {
case Some(executorsOnHost) =>
// preferredLocation is a known host. Select an executor that has the least receivers in
// this host
val leastScheduledExecutor =
executorsOnHost.minBy(executor => numReceiversOnExecutor(executor))
scheduledLocations(i) += leastScheduledExecutor
numReceiversOnExecutor(leastScheduledExecutor) =
numReceiversOnExecutor(leastScheduledExecutor) + 1
case None =>
// preferredLocation is an unknown host.
// Note: There are two cases:
// 1. This executor is not up. But it may be up later.
// 2. This executor is dead, or it's not a host in the cluster.
// Currently, simply add host to the scheduled executors.

    // Note: host could be `HDFSCacheTaskLocation`, so use `TaskLocation.apply` to handle
    // this case
    scheduledLocations(i) += TaskLocation(host)
}

}
}

補充：ReceiverTracker本身不直接監(jiān)管Receiver，它是Driver級別的可間接地，用ReceiverSupervisor監(jiān)控那臺機器上Executor中的Receiver。

11.2.2、if (!skipReceiverLaunch) launchReceivers()

/**

• Get the receivers from the ReceiverInputDStreams, distributes them to the
• worker nodes as a parallel collection, and runs them.
  */
  private def launchReceivers(): Unit = {
  val receivers = receiverInputStreams.map(nis => {
  val rcvr = nis.getReceiver()
  rcvr.setReceiverId(nis.id)
  rcvr
  })

runDummySparkJob()

logInfo("Starting " + receivers.length + " receivers")
endpoint.send(StartAllReceivers(receivers))
}

11.2.2.1運行了一個Dummy的作業(yè)，確保所有的Slaves正常工作，保證所有的Receiver都在一臺機器上

/**

• Run the dummy Spark job to ensure that all slaves have registered. This avoids all the
• receivers to be scheduled on the same node.
• TODO Should poll the executor number and wait for executors according to
• "spark.scheduler.minRegisteredResourcesRatio" and
• "spark.scheduler.maxRegisteredResourcesWaitingTime" rather than running a dummy job.
  */
  private def runDummySparkJob(): Unit = {
  if (!ssc.sparkContext.isLocal) {
  ssc.sparkContext.makeRDD(1 to 50, 50).map(x => (x, 1)).reduceByKey(_ + _, 20).collect()
  }
  assert(getExecutors.nonEmpty)
  }

11.2.2.2、endpoint.send(StartAllReceivers(receivers)

// endpoint is created when generator starts.
// This not being null means the tracker has been started and not stopped
private var endpoint: RpcEndpointRef = null

endpoint = ssc.env.rpcEnv.setupEndpoint(
"ReceiverTracker", new ReceiverTrackerEndpoint(ssc.env.rpcEnv))

ReceiverTrackerEndpoint
override def receive: PartialFunction[Any, Unit] = {
// Local messages
case StartAllReceivers(receivers) =>
val scheduledLocations = schedulingPolicy.scheduleReceivers(receivers, getExecutors)
for (receiver <- receivers) {
val executors = scheduledLocations(receiver.streamId)
updateReceiverScheduledExecutors(receiver.streamId, executors)
receiverPreferredLocations(receiver.streamId) = receiver.preferredLocation
startReceiver(receiver, executors)
}

startReceiver
// Function to start the receiver on the worker node
val startReceiverFunc: Iterator[Receiver[]] => Unit =
(iterator: Iterator[Receiver[]]) => {
if (!iterator.hasNext) {
throw new SparkException(
"Could not start receiver as object not found.")
}
if (TaskContext.get().attemptNumber() == 0) {
val receiver = iterator.next()
assert(iterator.hasNext == false)
val supervisor = new ReceiverSupervisorImpl(
receiver, SparkEnv.get, serializableHadoopConf.value, checkpointDirOption)
supervisor.start()
supervisor.awaitTermination()
} else {
// It's restarted by TaskScheduler, but we want to reschedule it again. So exit it.
}
}

逆天的設(shè)計啊

// Create the RDD using the scheduledLocations to run the receiver in a Spark job
val receiverRDD: RDD[Receiver[]] =
if (scheduledLocations.isEmpty) {
ssc.sc.makeRDD(Seq(receiver), 1)
} else {
val preferredLocations = scheduledLocations.map(.toString).distinct
ssc.sc.makeRDD(Seq(receiver -> preferredLocations))
}
receiverRDD.setName(s"Receiver $receiverId")
ssc.sparkContext.setJobDescription(s"Streaming job running receiver $receiverId")
ssc.sparkContext.setCallSite(Option(ssc.getStartSite()).getOrElse(Utils.getCallSite()))

val future = ssc.sparkContext.submitJob[Receiver[], Unit, Unit](
receiverRDD, startReceiverFunc, Seq(0), (, ) => Unit, ())
// We will keep restarting the receiver job until ReceiverTracker is stopped
future.onComplete {
case Success() =>
if (!shouldStartReceiver) {
onReceiverJobFinish(receiverId)
} else {
logInfo(s"Restarting Receiver $receiverId")
self.send(RestartReceiver(receiver))
}
case Failure(e) =>
if (!shouldStartReceiver) {
onReceiverJobFinish(receiverId)
} else {
logError("Receiver has been stopped. Try to restart it.", e)
logInfo(s"Restarting Receiver $receiverId")
self.send(RestartReceiver(receiver))
}
}(submitJobThreadPool)
logInfo(s"Receiver ${receiver.streamId} started")
}

ReceiverSupervisorImpl.startReceiver
/** Start receiver */
def startReceiver(): Unit = synchronized {
try {
if (onReceiverStart()) {
logInfo("Starting receiver")
receiverState = Started
receiver.onStart()
logInfo("Called receiver onStart")
} else {
// The driver refused us
stop("Registered unsuccessfully because Driver refused to start receiver " + streamId, None)
}

override protected def onReceiverStart(): Boolean = {
val msg = RegisterReceiver(
streamId, receiver.getClass.getSimpleName, host, executorId, endpoint)
trackerEndpoint.askWithRetryBoolean
}

11.3、JobScheduler.start  jobGenerator.start()

/** Start generation of jobs */
def start(): Unit = synchronized {
if (eventLoop != null) return // generator has already been started

// Call checkpointWriter here to initialize it before eventLoop uses it to avoid a deadlock.
// See SPARK-10125
checkpointWriter

eventLoop = new EventLoopJobGeneratorEvent {
override protected def onReceive(event: JobGeneratorEvent): Unit = processEvent(event)

override protected def onError(e: Throwable): Unit = {
  jobScheduler.reportError("Error in job generator", e)
}

}
eventLoop.start()

if (ssc.isCheckpointPresent) {
restart()
} else {
startFirstTime()
}
}

根據(jù)時間間隔不斷發(fā)送消息

/** Processes all events */
private def processEvent(event: JobGeneratorEvent) {
logDebug("Got event " + event)
event match {
case GenerateJobs(time) => generateJobs(time)
case ClearMetadata(time) => clearMetadata(time)
case DoCheckpoint(time, clearCheckpointDataLater) =>
doCheckpoint(time, clearCheckpointDataLater)
case ClearCheckpointData(time) => clearCheckpointData(time)
}
}

/** Generate jobs and perform checkpoint for the given time. */
private def generateJobs(time: Time) {
// Set the SparkEnv in this thread, so that job generation code can access the environment
// Example: BlockRDDs are created in this thread, and it needs to access BlockManager
// Update: This is probably redundant after threadlocal stuff in SparkEnv has been removed.
SparkEnv.set(ssc.env)
Try {
jobScheduler.receiverTracker.allocateBlocksToBatch(time) // allocate received blocks to batch
graph.generateJobs(time) // generate jobs using allocated block
} match {
case Success(jobs) =>
val streamIdToInputInfos = jobScheduler.inputInfoTracker.getInfo(time)
jobScheduler.submitJobSet(JobSet(time, jobs, streamIdToInputInfos))
case Failure(e) =>
jobScheduler.reportError("Error generating jobs for time " + time, e)
}
eventLoop.post(DoCheckpoint(time, clearCheckpointDataLater = false))
}

def submitJobSet(jobSet: JobSet) {
if (jobSet.jobs.isEmpty) {
logInfo("No jobs added for time " + jobSet.time)
} else {
listenerBus.post(StreamingListenerBatchSubmitted(jobSet.toBatchInfo))
jobSets.put(jobSet.time, jobSet)
jobSet.jobs.foreach(job => jobExecutor.execute(new JobHandler(job)))
logInfo("Added jobs for time " + jobSet.time)
}
}

/**

• Executes the given task sometime in the future. The task
• may execute in a new thread or in an existing pooled thread.
• If the task cannot be submitted for execution, either because this
• executor has been shutdown or because its capacity has been reached,
• the task is handled by the current {@code RejectedExecutionHandler}.
• @param command the task to execute
• @throws RejectedExecutionException at discretion of

•     {@code RejectedExecutionHandler}, if the task
  

•     cannot be accepted for execution
  

• @throws NullPointerException if {@code command} is null
  /
  public void execute(Runnable command) {
  if (command == null)
  throw new NullPointerException();
  /
  • Proceed in 3 steps:
  • 1. If fewer than corePoolSize threads are running, try to
  • start a new thread with the given command as its first
  • task. The call to addWorker atomically checks runState and
  • workerCount, and so prevents false alarms that would add
  • threads when it shouldn't, by returning false.
  • 2. If a task can be successfully queued, then we still need
  • to double-check whether we should have added a thread
  • (because existing ones died since last checking) or that
  • the pool shut down since entry into this method. So we
  • recheck state and if necessary roll back the enqueuing if
  • stopped, or start a new thread if there are none.
  • 3. If we cannot queue task, then we try to add a new
  • thread. If it fails, we know we are shut down or saturated
  • and so reject the task.
    */
    int c = ctl.get();
    if (workerCountOf(c) < corePoolSize) {
    if (addWorker(command, true))
    return;
    c = ctl.get();
    }
    if (isRunning(c) && workQueue.offer(command)) {
    int recheck = ctl.get();
    if (! isRunning(recheck) && remove(command))
    reject(command);
    else if (workerCountOf(recheck) == 0)
    addWorker(null, false);
    }
    else if (!addWorker(command, false))
    reject(command);
    }

private class JobHandler(job: Job) extends Runnable with Logging {
import JobScheduler._

def run() {
  try {
    val formattedTime = UIUtils.formatBatchTime(
      job.time.milliseconds, ssc.graph.batchDuration.milliseconds, showYYYYMMSS = false)
    val batchUrl = s"/streaming/batch/?id=${job.time.milliseconds}"
    val batchLinkText = s"[output operation ${job.outputOpId}, batch time ${formattedTime}]"

    ssc.sc.setJobDescription(
      s"""Streaming job from <a href="$batchUrl">$batchLinkText</a>""")
    ssc.sc.setLocalProperty(BATCH_TIME_PROPERTY_KEY, job.time.milliseconds.toString)
    ssc.sc.setLocalProperty(OUTPUT_OP_ID_PROPERTY_KEY, job.outputOpId.toString)

    // We need to assign `eventLoop` to a temp variable. Otherwise, because
    // `JobScheduler.stop(false)` may set `eventLoop` to null when this method is running, then
    // it's possible that when `post` is called, `eventLoop` happens to null.
    var _eventLoop = eventLoop
    if (_eventLoop != null) {
      _eventLoop.post(JobStarted(job, clock.getTimeMillis()))
      // Disable checks for existing output directories in jobs launched by the streaming
      // scheduler, since we may need to write output to an existing directory during checkpoint
      // recovery; see SPARK-4835 for more details.
      PairRDDFunctions.disableOutputSpecValidation.withValue(true) {
        job.run()
      }
      _eventLoop = eventLoop
      if (_eventLoop != null) {
        _eventLoop.post(JobCompleted(job, clock.getTimeMillis()))
      }
    } else {
      // JobScheduler has been stopped.
    }
  } finally {
    ssc.sc.setLocalProperty(JobScheduler.BATCH_TIME_PROPERTY_KEY, null)
    ssc.sc.setLocalProperty(JobScheduler.OUTPUT_OP_ID_PROPERTY_KEY, null)
  }
}

}
}

private def processEvent(event: JobSchedulerEvent) {
try {
event match {
case JobStarted(job, startTime) => handleJobStart(job, startTime)
case JobCompleted(job, completedTime) => handleJobCompletion(job, completedTime)
case ErrorReported(m, e) => handleError(m, e)
}
} catch {
case e: Throwable =>
reportError("Error in job scheduler", e)
}
}

private def handleJobStart(job: Job, startTime: Long) {
val jobSet = jobSets.get(job.time)
val isFirstJobOfJobSet = !jobSet.hasStarted
jobSet.handleJobStart(job)
if (isFirstJobOfJobSet) {
// "StreamingListenerBatchStarted" should be posted after calling "handleJobStart" to get the
// correct "jobSet.processingStartTime".
listenerBus.post(StreamingListenerBatchStarted(jobSet.toBatchInfo))
}
job.setStartTime(startTime)
listenerBus.post(StreamingListenerOutputOperationStarted(job.toOutputOperationInfo))
logInfo("Starting job " + job.id + " from job set of time " + jobSet.time)
}

private class JobHandler(job: Job) extends Runnable with Logging {
import JobScheduler._

def run() {
  try {
    val formattedTime = UIUtils.formatBatchTime(
      job.time.milliseconds, ssc.graph.batchDuration.milliseconds, showYYYYMMSS = false)
    val batchUrl = s"/streaming/batch/?id=${job.time.milliseconds}"
    val batchLinkText = s"[output operation ${job.outputOpId}, batch time ${formattedTime}]"

    ssc.sc.setJobDescription(
      s"""Streaming job from <a href="$batchUrl">$batchLinkText</a>""")
    ssc.sc.setLocalProperty(BATCH_TIME_PROPERTY_KEY, job.time.milliseconds.toString)
    ssc.sc.setLocalProperty(OUTPUT_OP_ID_PROPERTY_KEY, job.outputOpId.toString)

    // We need to assign `eventLoop` to a temp variable. Otherwise, because
    // `JobScheduler.stop(false)` may set `eventLoop` to null when this method is running, then
    // it's possible that when `post` is called, `eventLoop` happens to null.
    var _eventLoop = eventLoop
    if (_eventLoop != null) {
      _eventLoop.post(JobStarted(job, clock.getTimeMillis()))
      // Disable checks for existing output directories in jobs launched by the streaming
      // scheduler, since we may need to write output to an existing directory during checkpoint
      // recovery; see SPARK-4835 for more details.
      PairRDDFunctions.disableOutputSpecValidation.withValue(true) {
        job.run()
      }
      _eventLoop = eventLoop
      if (_eventLoop != null) {
        _eventLoop.post(JobCompleted(job, clock.getTimeMillis()))
      }
    } else {
      // JobScheduler has been stopped.
    }
  } finally {
    ssc.sc.setLocalProperty(JobScheduler.BATCH_TIME_PROPERTY_KEY, null)
    ssc.sc.setLocalProperty(JobScheduler.OUTPUT_OP_ID_PROPERTY_KEY, null)
  }
}

}
}