開發(fā)

mesos擴展

代碼位置: /include/mesos/module

1. 認證
   見authenticatee.hpp, authorizer.hpp
   用于擴展第三方鑒權(quán)機制

2. 資源分配
   用于擴展資源分配機制，可以擴展資源分配算法和為內(nèi)建層級分配器實現(xiàn)新的排序算法(用戶和框架)

   代碼接口詳見/include/mesos/master/allocator.hpp(hirerarchical.hpp實現(xiàn)默認分配算法)

class Allocator
{
public:
  // Attempts either to create a built-in DRF allocator or to load an
  // allocator instance from a module using the given name. If Try
  // does not report an error, the wrapped Allocator* is not null.
  static Try<Allocator*> create(const std::string& name);

  Allocator() {}

  virtual ~Allocator() {}

  //初始化分配器
  virtual void initialize(
      const Duration& allocationInterval,
      const lambda::function<
          void(const FrameworkID&,
               const hashmap<SlaveID, Resources>&)>& offerCallback,
      const lambda::function<
          void(const FrameworkID&,
               const hashmap<SlaveID, UnavailableResources>&)>&
        inverseOfferCallback,
      const hashmap<std::string, RoleInfo>& roles) = 0;

  //在分配機制中添加框架
  virtual void addFramework(
      const FrameworkID& frameworkId,
      const FrameworkInfo& frameworkInfo,
      const hashmap<SlaveID, Resources>& used) = 0;

  //從分配機制中移除框架
  virtual void removeFramework(
      const FrameworkID& frameworkId) = 0;

  //激活指定框架
  // Offers are sent only to activated frameworks.
  virtual void activateFramework(
      const FrameworkID& frameworkId) = 0;

  //暫停指定框架
  virtual void deactivateFramework(
      const FrameworkID& frameworkId) = 0;

  //更新框架信息
  virtual void updateFramework(
      const FrameworkID& frameworkId,
      const FrameworkInfo& frameworkInfo) = 0;

  // Note that the 'total' resources are passed explicitly because it
  // includes resources that are dynamically "checkpointed" on the
  // slave (e.g. persistent volumes, dynamic reservations, etc). The
  // slaveInfo resources, on the other hand, correspond directly to
  // the static --resources flag value on the slave.

  //在分配機制中添加slave
  virtual void addSlave(
      const SlaveID& slaveId,
      const SlaveInfo& slaveInfo,
      const Option<Unavailability>& unavailability,
      const Resources& total,
      const hashmap<FrameworkID, Resources>& used) = 0;

  //從分配機制中移除slave
  virtual void removeSlave(
      const SlaveID& slaveId) = 0;

  // Note that 'oversubscribed' resources include the total amount of
  // oversubscribed resources that are allocated and available.
  // TODO(vinod): Instead of just oversubscribed resources have this
  // method take total resources. We can then reuse this method to
  // update slave's total resources in the future.

  //更新slave信息
  virtual void updateSlave(
      const SlaveID& slave,
      const Resources& oversubscribed) = 0;

  //激活指定的slave
  // Offers are sent only for activated slaves.
  virtual void activateSlave(
      const SlaveID& slaveId) = 0;

  //暫停指定的slave
  virtual void deactivateSlave(
      const SlaveID& slaveId) = 0;

  //更新slave白名單列表
  virtual void updateWhitelist(
      const Option<hashset<std::string>>& whitelist) = 0;

  //當框架接收指定資源請時觸發(fā)的動作
  virtual void requestResources(
      const FrameworkID& frameworkId,
      const std::vector<Request>& requests) = 0;

  //更新指定框架在指定slave上的資源分配
  virtual void updateAllocation(
      const FrameworkID& frameworkId,
      const SlaveID& slaveId,
      const std::vector<Offer::Operation>& operations) = 0;

  //
  virtual process::Future<Nothing> updateAvailable(
      const SlaveID& slaveId,
      const std::vector<Offer::Operation>& operations) = 0;

  // We currently support storing the next unavailability, if there is one, per
  // slave. If `unavailability` is not set then there is no known upcoming
  // unavailability. This might require the implementation of the function to
  // remove any inverse offers that are outstanding.
  virtual void updateUnavailability(
      const SlaveID& slaveId,
      const Option<Unavailability>& unavailability) = 0;

  // Informs the allocator that the inverse offer has been responded to or
  // revoked. If `status` is not set then the inverse offer was not responded
  // to, possibly because the offer timed out or was rescinded. This might
  // require the implementation of the function to remove any inverse offers
  // that are outstanding. The `unavailableResources` can be used by the
  // allocator to distinguish between different inverse offers sent to the same
  // framework for the same slave.
  virtual void updateInverseOffer(
      const SlaveID& slaveId,
      const FrameworkID& frameworkId,
      const Option<UnavailableResources>& unavailableResources,
      const Option<InverseOfferStatus>& status,
      const Option<Filters>& filters = None()) = 0;

  // Retrieves the status of all inverse offers maintained by the allocator.
  virtual process::Future<
      hashmap<SlaveID, hashmap<FrameworkID, mesos::master::InverseOfferStatus>>>
    getInverseOfferStatuses() = 0;

  //從指定框架回收資源觸發(fā)
  // Informs the Allocator to recover resources that are considered
  // used by the framework.
  virtual void recoverResources(
      const FrameworkID& frameworkId,
      const SlaveID& slaveId,
      const Resources& resources,
      const Option<Filters>& filters) = 0;

  //框架想要重新獲得之前所排查的資源offer
  // Whenever a framework that has filtered resources wants to revive
  // offers for those resources the master invokes this callback.
  virtual void reviveOffers(
      const FrameworkID& frameworkId) = 0;
  
  // 資源offer分配超時
  // Informs the allocator to stop sending resources for the framework
  virtual void suppressOffers(
      const FrameworkID& frameworkId) = 0;
};

使用: --allocator指定資源分配模塊

代碼詳見: /src/master/allocator/sorter/sorter.hpp(drf/sorter.hpp中實現(xiàn)公平分配并支持帶權(quán)值優(yōu)先級)

class Sorter
{
public:
  virtual ~Sorter() {}

  //從分配算法中增加某個client
  // Adds a client to allocate resources to. A client
  // may be a user or a framework.
  virtual void add(const std::string& client, double weight = 1) = 0;

  //從分配算法刪除某個client
  // Removes a client.
  virtual void remove(const std::string& client) = 0;

  //從排序器增加某個client
  // Readds a client to the sort after deactivate.
  virtual void activate(const std::string& client) = 0;

  //從排序器移除某個client
  // Removes a client from the sort, so it won't get allocated to.
  virtual void deactivate(const std::string& client) = 0;

  // Specify that resources have been allocated to the given client.
  virtual void allocated(
      const std::string& client,
      const SlaveID& slaveId,
      const Resources& resources) = 0;

  // Updates a portion of the allocation for the client, in order to
  // augment the resources with additional metadata (e.g., volumes)
  // This means that the new allocation must not affect the static
  // roles, or the overall quantities of resources!
  virtual void update(
      const std::string& client,
      const SlaveID& slaveId,
      const Resources& oldAllocation,
      const Resources& newAllocation) = 0;

  // Specify that resources have been unallocated from the given client.
  virtual void unallocated(
      const std::string& client,
      const SlaveID& slaveId,
      const Resources& resources) = 0;

  // Returns the resources that have been allocated to this client.
  virtual hashmap<SlaveID, Resources> allocation(const std::string& client) = 0;

  // Returns the clients that have allocations on this slave.
  virtual hashmap<std::string, Resources> allocation(
      const SlaveID& slaveId) = 0;

  //返回分配給某個client的資源
  // Returns the given slave's resources that have been allocated to
  // this client.
  virtual Resources allocation(
      const std::string& client,
      const SlaveID& slaveId) = 0;

  //在資源池中添加slave資源
  // Add resources to the total pool of resources this
  // Sorter should consider.
  virtual void add(const SlaveID& slaveId, const Resources& resources) = 0;

  //在資源池中移除slave資源
  // Remove resources from the total pool.
  virtual void remove(const SlaveID& slaveId, const Resources& resources) = 0;

  ////更新資源池中slave資源
  // Updates the total pool of resources.
  virtual void update(const SlaveID& slaveId, const Resources& resources) = 0;

  //按排序算法對client進行排序
  // Returns a list of all clients, in the order that they
  // should be allocated to, according to this Sorter's policy.
  virtual std::list<std::string> sort() = 0;

  // Returns true if this Sorter contains the specified client,
  // either active or deactivated.
  virtual bool contains(const std::string& client) = 0;

  // Returns the number of clients this Sorter contains,
  // either active or deactivated.
  virtual int count() = 0;
};

3. 隔離
   見isolator.hpp
   提供新的隔離方式和監(jiān)控機制

4. 匿名
   見anonymous.hpp
   與master和slave啟動時被加載, 與父進程共同存在, 不會擴展或替代mesos已有的功能

5. hook
   見/include/mesos/hook.hpp
   通過hook擴展組件的功能

   使用--hooks選項設(shè)置hook列表

模塊通過在master和slave時通過參數(shù)--modules指定json文件來設(shè)置模塊加載及配置, json文件格式:

{
    "libraries" : [
        {
            "file" : "",
            "name" : "",
            "modules" : [
                {
                    "name" : "",
                    "parameters" : [
                        {
                            "key" : "",
                            "value" : ""
                        }
                    ]
                }
            ]
        }
    ]
}

mesos開發(fā)

消息
mesos各組件之間使用protocol buffer定義發(fā)送的消息，所有消息格式見代碼/include/mesos/mesos.proto

框架
+ 調(diào)度器 & 調(diào)度驅(qū)動
負責(zé)管理框架所獲得的資源, 代碼見/include/mesos/scheduler/scheduler.hpp

class Scheduler
{
public:
  // Empty virtual destructor (necessary to instantiate subclasses).
  virtual ~Scheduler() {}

  //mesos進行注冊時被回調(diào)
  // Invoked when the scheduler successfully registers with a Mesos
  // master. A unique ID (generated by the master) used for
  // distinguishing this framework from others and MasterInfo with the
  // ip and port of the current master are provided as arguments.
  virtual void registered(
      SchedulerDriver* driver,
      const FrameworkID& frameworkId,
      const MasterInfo& masterInfo) = 0;

  //當重新選舉mesos master后，被回調(diào)重新進行注冊
  // Invoked when the scheduler re-registers with a newly elected
  // Mesos master. This is only called when the scheduler has
  // previously been registered. MasterInfo containing the updated
  // information about the elected master is provided as an argument.
  virtual void reregistered(
      SchedulerDriver* driver,
      const MasterInfo& masterInfo) = 0;

  //在調(diào)度器和master被斷開時被回調(diào)
  // Invoked when the scheduler becomes "disconnected" from the master
  // (e.g., the master fails and another is taking over).
  virtual void disconnected(SchedulerDriver* driver) = 0;

  //在master向framework提供資源offer時調(diào)用
  // Invoked when resources have been offered to this framework. A
  // single offer will only contain resources from a single slave.
  // Resources associated with an offer will not be re-offered to
  // _this_ framework until either (a) this framework has rejected
  // those resources (see SchedulerDriver::launchTasks) or (b) those
  // resources have been rescinded (see Scheduler::offerRescinded).
  // Note that resources may be concurrently offered to more than one
  // framework at a time (depending on the allocator being used). In
  // that case, the first framework to launch tasks using those
  // resources will be able to use them while the other frameworks
  // will have those resources rescinded (or if a framework has
  // already launched tasks with those resources then those tasks will
  // fail with a TASK_LOST status and a message saying as much).
  virtual void resourceOffers(
      SchedulerDriver* driver,
      const std::vector<Offer>& offers) = 0;

  /根據(jù)不同的分配器，可能將一個資源分配給多個框架，但是第一個響應(yīng)master的framework會得到資源，其他framework會被回調(diào)表示master撤銷某資源offer, 若framework在收到該請求之前已經(jīng)返回taskinfo，則在task狀態(tài)中將受到taskstatus為lost狀態(tài)
  // Invoked when an offer is no longer valid (e.g., the slave was
  // lost or another framework used resources in the offer). If for
  // whatever reason an offer is never rescinded (e.g., dropped
  // message, failing over framework, etc.), a framework that attempts
  // to launch tasks using an invalid offer will receive TASK_LOST
  // status updates for those tasks (see Scheduler::resourceOffers).
  virtual void offerRescinded(
      SchedulerDriver* driver,
      const OfferID& offerId) = 0;

  //任務(wù)狀態(tài)發(fā)生變化回調(diào)
  // Invoked when the status of a task has changed (e.g., a slave is
  // lost and so the task is lost, a task finishes and an executor
  // sends a status update saying so, etc). If implicit
  // acknowledgements are being used, then returning from this
  // callback _acknowledges_ receipt of this status update! If for
  // whatever reason the scheduler aborts during this callback (or
  // the process exits) another status update will be delivered (note,
  // however, that this is currently not true if the slave sending the
  // status update is lost/fails during that time). If explicit
  // acknowledgements are in use, the scheduler must acknowledge this
  // status on the driver.
  virtual void statusUpdate(
      SchedulerDriver* driver,
      const TaskStatus& status) = 0;

  //向調(diào)度器傳遞執(zhí)行器發(fā)送的消息，調(diào)度器可以訪問執(zhí)行器和slavede Id, 以及調(diào)度器所發(fā)送的數(shù)據(jù)
  // Invoked when an executor sends a message. These messages are best
  // effort; do not expect a framework message to be retransmitted in
  // any reliable fashion.
  virtual void frameworkMessage(
      SchedulerDriver* driver,
      const ExecutorID& executorId,
      const SlaveID& slaveId,
      const std::string& data) = 0;

  //當slave丟失時回調(diào)
  // Invoked when a slave has been determined unreachable (e.g.,
  // machine failure, network partition). Most frameworks will need to
  // reschedule any tasks launched on this slave on a new slave.
  virtual void slaveLost(
      SchedulerDriver* driver,
      const SlaveID& slaveId) = 0;

  //執(zhí)行器丟失是回調(diào)
  // Invoked when an executor has exited/terminated. Note that any
  // tasks running will have TASK_LOST status updates automagically
  // generated.
  virtual void executorLost(
      SchedulerDriver* driver,
      const ExecutorID& executorId,
      const SlaveID& slaveId,
      int status) = 0;

  //當發(fā)送錯誤時調(diào)用，常用于清理工作
  // Invoked when there is an unrecoverable error in the scheduler or
  // scheduler driver. The driver will be aborted BEFORE invoking this
  // callback.
  virtual void error(
      SchedulerDriver* driver,
      const std::string& message) = 0;
};


// Abstract interface for connecting a scheduler to Mesos. This
// interface is used both to manage the scheduler's lifecycle (start
// it, stop it, or wait for it to finish) and to interact with Mesos
// (e.g., launch tasks, kill tasks, etc.). See MesosSchedulerDriver
// below for a concrete example of a SchedulerDriver.
class SchedulerDriver
{
public:
  // Empty virtual destructor (necessary to instantiate subclasses).
  // It is expected that 'stop()' is called before this is called.
  virtual ~SchedulerDriver() {}

  //啟動調(diào)度器
  // Starts the scheduler driver. This needs to be called before any
  // other driver calls are made.
  virtual Status start() = 0;

  //停止驅(qū)動
  // Stops the scheduler driver. If the 'failover' flag is set to
  // false then it is expected that this framework will never
  // reconnect to Mesos. So Mesos will unregister the framework and
  // shutdown all its tasks and executors. If 'failover' is true, all
  // executors and tasks will remain running (for some framework
  // specific failover timeout) allowing the scheduler to reconnect
  // (possibly in the same process, or from a different process, for
  // example, on a different machine).
  virtual Status stop(bool failover = false) = 0;

  // Aborts the driver so that no more callbacks can be made to the
  // scheduler. The semantics of abort and stop have deliberately been
  // separated so that code can detect an aborted driver (i.e., via
  // the return status of SchedulerDriver::join, see below), and
  // instantiate and start another driver if desired (from within the
  // same process). Note that 'stop()' is not automatically called
  // inside 'abort()'.
  virtual Status abort() = 0;

  //等待驅(qū)動退出發(fā)送abort和stop動作
  // Waits for the driver to be stopped or aborted, possibly
  // _blocking_ the current thread indefinitely. The return status of
  // this function can be used to determine if the driver was aborted
  // (see mesos.proto for a description of Status).
  virtual Status join() = 0;

  //依次執(zhí)行start和join
  // Starts and immediately joins (i.e., blocks on) the driver.
  virtual Status run() = 0;

  //向mesos請求資源并將資源提供給調(diào)度器
  // Requests resources from Mesos (see mesos.proto for a description
  // of Request and how, for example, to request resources from
  // specific slaves). Any resources available are offered to the
  // framework via Scheduler::resourceOffers callback, asynchronously.
  virtual Status requestResources(const std::vector<Request>& requests) = 0;

  //在offer上啟動一組任務(wù)
  // Launches the given set of tasks. Any resources remaining (i.e.,
  // not used by the tasks or their executors) will be considered
  // declined. The specified filters are applied on all unused
  // resources (see mesos.proto for a description of Filters).
  // Available resources are aggregated when multiple offers are
  // provided. Note that all offers must belong to the same slave.
  // Invoking this function with an empty collection of tasks declines
  // offers in their entirety (see Scheduler::declineOffer).
  virtual Status launchTasks(
      const std::vector<OfferID>& offerIds,
      const std::vector<TaskInfo>& tasks,
      const Filters& filters = Filters()) = 0;

  
  // DEPRECATED: Use launchTasks(offerIds, tasks, filters) instead.
  virtual Status launchTasks(
      const OfferID& offerId,
      const std::vector<TaskInfo>& tasks,
      const Filters& filters = Filters()) = 0;

  //kill任務(wù)
  // Kills the specified task. Note that attempting to kill a task is
  // currently not reliable. If, for example, a scheduler fails over
  // while it was attempting to kill a task it will need to retry in
  // the future. Likewise, if unregistered / disconnected, the request
  // will be dropped (these semantics may be changed in the future).
  virtual Status killTask(const TaskID& taskId) = 0;

  //接受資源offer
  // Accepts the given offers and performs a sequence of operations on
  // those accepted offers. See Offer.Operation in mesos.proto for the
  // set of available operations. Available resources are aggregated
  // when multiple offers are provided. Note that all offers must
  // belong to the same slave. Any unused resources will be considered
  // declined. The specified filters are applied on all unused
  // resources (see mesos.proto for a description of Filters).
  virtual Status acceptOffers(
      const std::vector<OfferID>& offerIds,
      const std::vector<Offer::Operation>& operations,
      const Filters& filters = Filters()) = 0;

  //拒絕資源offer
  // Declines an offer in its entirety and applies the specified
  // filters on the resources (see mesos.proto for a description of
  // Filters). Note that this can be done at any time, it is not
  // necessary to do this within the Scheduler::resourceOffers
  // callback.
  virtual Status declineOffer(
      const OfferID& offerId,
      const Filters& filters = Filters()) = 0;

  //刪除所有過濾器
  // Removes all filters previously set by the framework (via
  // launchTasks()). This enables the framework to receive offers from
  // those filtered slaves.
  virtual Status reviveOffers() = 0;

  // Inform Mesos master to stop sending offers to the framework. The
  // scheduler should call reviveOffers() to resume getting offers.
  virtual Status suppressOffers() = 0;

  // Acknowledges the status update. This should only be called
  // once the status update is processed durably by the scheduler.
  // Not that explicit acknowledgements must be requested via the
  // constructor argument, otherwise a call to this method will
  // cause the driver to crash.
  virtual Status acknowledgeStatusUpdate(
      const TaskStatus& status) = 0;

  //從框架向執(zhí)行器發(fā)送消息
  // Sends a message from the framework to one of its executors. These
  // messages are best effort; do not expect a framework message to be
  // retransmitted in any reliable fashion.
  virtual Status sendFrameworkMessage(
      const ExecutorID& executorId,
      const SlaveID& slaveId,
      const std::string& data) = 0;

  //獲取任務(wù)狀態(tài)
  // Allows the framework to query the status for non-terminal tasks.
  // This causes the master to send back the latest task status for
  // each task in 'statuses', if possible. Tasks that are no longer
  // known will result in a TASK_LOST update. If statuses is empty,
  // then the master will send the latest status for each task
  // currently known.
  virtual Status reconcileTasks(
      const std::vector<TaskStatus>& statuses) = 0;
};


// Concrete implementation of a SchedulerDriver that connects a
// Scheduler with a Mesos master. The MesosSchedulerDriver is
// thread-safe.
//
// Note that scheduler failover is supported in Mesos. After a
// scheduler is registered with Mesos it may failover (to a new
// process on the same machine or across multiple machines) by
// creating a new driver with the ID given to it in
// Scheduler::registered.
//
// The driver is responsible for invoking the Scheduler callbacks as
// it communicates with the Mesos master.
//
// Note that blocking on the MesosSchedulerDriver (e.g., via
// MesosSchedulerDriver::join) doesn't affect the scheduler callbacks
// in anyway because they are handled by a different thread.
//
// Note that the driver uses GLOG to do its own logging. GLOG flags
// can be set via environment variables, prefixing the flag name with
// "GLOG_", e.g., "GLOG_v=1". For Mesos specific logging flags see
// src/logging/flags.hpp. Mesos flags can also be set via environment
// variables, prefixing the flag name with "MESOS_", e.g.,
// "MESOS_QUIET=1".
//
// See src/examples/test_framework.cpp for an example of using the
// MesosSchedulerDriver.
class MesosSchedulerDriver : public SchedulerDriver
{
public:
  // Creates a new driver for the specified scheduler. The master
  // should be one of:
  //
  //     host:port
  //     zk://host1:port1,host2:port2,.../path
  //     zk://username:password@host1:port1,host2:port2,.../path
  //     file:///path/to/file (where file contains one of the above)
  //
  // The driver will attempt to "failover" if the specified
  // FrameworkInfo includes a valid FrameworkID.
  //
  // Any Mesos configuration options are read from environment
  // variables, as well as any configuration files found through the
  // environment variables.
  //
  // TODO(vinod): Deprecate this once 'MesosSchedulerDriver' can take
  // 'Option<Credential>' as parameter. Currently it cannot because
  // 'stout' is not visible from here.
  MesosSchedulerDriver(
      Scheduler* scheduler,
      const FrameworkInfo& framework,
      const std::string& master);

  // Same as the above constructor but takes 'credential' as argument.
  // The credential will be used for authenticating with the master.
  MesosSchedulerDriver(
      Scheduler* scheduler,
      const FrameworkInfo& framework,
      const std::string& master,
      const Credential& credential);

  // These constructors are the same as the above two, but allow
  // the framework to specify whether implicit or explicit
  // acknowledgements are desired. See statusUpdate() for the
  // details about explicit acknowledgements.
  //
  // TODO(bmahler): Deprecate the above two constructors. In 0.22.0
  // these new constructors are exposed.
  MesosSchedulerDriver(
      Scheduler* scheduler,
      const FrameworkInfo& framework,
      const std::string& master,
      bool implicitAcknowledgements);

  MesosSchedulerDriver(
      Scheduler* scheduler,
      const FrameworkInfo& framework,
      const std::string& master,
      bool implicitAcknowlegements,
      const Credential& credential);

  // This destructor will block indefinitely if
  // MesosSchedulerDriver::start was invoked successfully (possibly
  // via MesosSchedulerDriver::run) and MesosSchedulerDriver::stop has
  // not been invoked.
  virtual ~MesosSchedulerDriver();

  // See SchedulerDriver for descriptions of these.
  virtual Status start();
  virtual Status stop(bool failover = false);
  virtual Status abort();
  virtual Status join();
  virtual Status run();

  virtual Status requestResources(
      const std::vector<Request>& requests);

  // TODO(nnielsen): launchTasks using single offer is deprecated.
  // Use launchTasks with offer list instead.
  virtual Status launchTasks(
      const OfferID& offerId,
      const std::vector<TaskInfo>& tasks,
      const Filters& filters = Filters());

  virtual Status launchTasks(
      const std::vector<OfferID>& offerIds,
      const std::vector<TaskInfo>& tasks,
      const Filters& filters = Filters());

  virtual Status killTask(const TaskID& taskId);

  virtual Status acceptOffers(
      const std::vector<OfferID>& offerIds,
      const std::vector<Offer::Operation>& operations,
      const Filters& filters = Filters());

  virtual Status declineOffer(
      const OfferID& offerId,
      const Filters& filters = Filters());

  virtual Status reviveOffers();

  virtual Status suppressOffers();

  virtual Status acknowledgeStatusUpdate(
      const TaskStatus& status);

  virtual Status sendFrameworkMessage(
      const ExecutorID& executorId,
      const SlaveID& slaveId,
      const std::string& data);

  virtual Status reconcileTasks(
      const std::vector<TaskStatus>& statuses);

protected:
  // Used to detect (i.e., choose) the master.
  internal::MasterDetector* detector;

private:
  void initialize();

  Scheduler* scheduler;
  FrameworkInfo framework;
  std::string master;

  // Used for communicating with the master.
  internal::SchedulerProcess* process;

  // URL for the master (e.g., zk://, file://, etc).
  std::string url;

  // Mutex for enforcing serial execution of all non-callbacks.
  std::recursive_mutex mutex;

  // Latch for waiting until driver terminates.
  process::Latch* latch;

  // Current status of the driver.
  Status status;

  const bool implicitAcknowlegements;

  const Credential* credential;

  // Scheduler process ID.
  std::string schedulerId;
};

+ 執(zhí)行器 & 執(zhí)行驅(qū)動
    負責(zé)啟動任務(wù)并執(zhí)行調(diào)取分配的任務(wù), 代碼見/include/mesos/scheduler/executor.hpp

class Executor
{
public:
  // Empty virtual destructor (necessary to instantiate subclasses).
  virtual ~Executor() {}

  //在執(zhí)行驅(qū)動器執(zhí)行成功后和slave連接后調(diào)用
  // Invoked once the executor driver has been able to successfully
  // connect with Mesos. In particular, a scheduler can pass some
  // data to its executors through the FrameworkInfo.ExecutorInfo's
  // data field.
  virtual void registered(
      ExecutorDriver* driver,
      const ExecutorInfo& executorInfo,
      const FrameworkInfo& frameworkInfo,
      const SlaveInfo& slaveInfo) = 0;

  //向重啟的slave重新注冊
  // Invoked when the executor re-registers with a restarted slave.
  virtual void reregistered(
      ExecutorDriver* driver,
      const SlaveInfo& slaveInfo) = 0;

  //在執(zhí)行器與slave斷開連接時調(diào)用
  // Invoked when the executor becomes "disconnected" from the slave
  // (e.g., the slave is being restarted due to an upgrade).
  virtual void disconnected(ExecutorDriver* driver) = 0;

  //在任務(wù)在當前執(zhí)行器上啟動時被調(diào)用
  // Invoked when a task has been launched on this executor (initiated
  // via Scheduler::launchTasks). Note that this task can be realized
  // with a thread, a process, or some simple computation, however, no
  // other callbacks will be invoked on this executor until this
  // callback has returned.
  virtual void launchTask(
      ExecutorDriver* driver,
      const TaskInfo& task) = 0;

  //當任務(wù)被kill時調(diào)用
  // Invoked when a task running within this executor has been killed
  // (via SchedulerDriver::killTask). Note that no status update will
  // be sent on behalf of the executor, the executor is responsible
  // for creating a new TaskStatus (i.e., with TASK_KILLED) and
  // invoking ExecutorDriver::sendStatusUpdate.
  virtual void killTask(
      ExecutorDriver* driver,
      const TaskID& taskId) = 0;

  //當接到框架執(zhí)行器發(fā)送的消息到達時被調(diào)用
  // Invoked when a framework message has arrived for this executor.
  // These messages are best effort; do not expect a framework message
  // to be retransmitted in any reliable fashion.
  virtual void frameworkMessage(
      ExecutorDriver* driver,
      const std::string& data) = 0;

  //通知執(zhí)行器結(jié)束所有運行中的任務(wù)
  // Invoked when the executor should terminate all of its currently
  // running tasks. Note that after a Mesos has determined that an
  // executor has terminated any tasks that the executor did not send
  // terminal status updates for (e.g., TASK_KILLED, TASK_FINISHED,
  // TASK_FAILED, etc) a TASK_LOST status update will be created.
  virtual void shutdown(ExecutorDriver* driver) = 0;

  //當制執(zhí)行或執(zhí)行器驅(qū)動發(fā)送錯誤時被調(diào)用
  // Invoked when a fatal error has occured with the executor and/or
  // executor driver. The driver will be aborted BEFORE invoking this
  // callback.
  virtual void error(
      ExecutorDriver* driver,
      const std::string& message) = 0;
};


// Abstract interface for connecting an executor to Mesos. This
// interface is used both to manage the executor's lifecycle (start
// it, stop it, or wait for it to finish) and to interact with Mesos
// (e.g., send status updates, send framework messages, etc.). See
// MesosExecutorDriver below for a concrete example of an
// ExecutorDriver.
class ExecutorDriver
{
public:
  // Empty virtual destructor (necessary to instantiate subclasses).
  virtual ~ExecutorDriver() {}

  //對驅(qū)動進行初始化
  // Starts the executor driver. This needs to be called before any
  // other driver calls are made.
  virtual Status start() = 0;

  //對驅(qū)動進行清理
  // Stops the executor driver.
  virtual Status stop() = 0;

  //在驅(qū)動異常退出時調(diào)用
  // Aborts the driver so that no more callbacks can be made to the
  // executor. The semantics of abort and stop have deliberately been
  // separated so that code can detect an aborted driver (i.e., via
  // the return status of ExecutorDriver::join, see below), and
  // instantiate and start another driver if desired (from within the
  // same process ... although this functionality is currently not
  // supported for executors).
  virtual Status abort() = 0;

  //等待驅(qū)動停止或異常停止
  // Waits for the driver to be stopped or aborted, possibly
  // _blocking_ the current thread indefinitely. The return status of
  // this function can be used to determine if the driver was aborted
  // (see mesos.proto for a description of Status).
  virtual Status join() = 0;

  //啟動驅(qū)動并阻塞后調(diào)用join操作
  // Starts and immediately joins (i.e., blocks on) the driver.
  virtual Status run() = 0;

  //想調(diào)度器發(fā)送任務(wù)狀態(tài)更新
  // Sends a status update to the framework scheduler, retrying as
  // necessary until an acknowledgement has been received or the
  // executor is terminated (in which case, a TASK_LOST status update
  // will be sent). See Scheduler::statusUpdate for more information
  // about status update acknowledgements.
  virtual Status sendStatusUpdate(const TaskStatus& status) = 0;

  //發(fā)送消息給framework
  // Sends a message to the framework scheduler. These messages are
  // best effort; do not expect a framework message to be
  // retransmitted in any reliable fashion.
  virtual Status sendFrameworkMessage(const std::string& data) = 0;
};


// Concrete implementation of an ExecutorDriver that connects an
// Executor with a Mesos slave. The MesosExecutorDriver is
// thread-safe.
//
// The driver is responsible for invoking the Executor callbacks as it
// communicates with the Mesos slave.
//
// Note that blocking on the MesosExecutorDriver (e.g., via
// MesosExecutorDriver::join) doesn't affect the executor callbacks in
// anyway because they are handled by a different thread.
//
// Note that the driver uses GLOG to do its own logging. GLOG flags
// can be set via environment variables, prefixing the flag name with
// "GLOG_", e.g., "GLOG_v=1". For Mesos specific logging flags see
// src/logging/flags.hpp. Mesos flags can also be set via environment
// variables, prefixing the flag name with "MESOS_", e.g.,
// "MESOS_QUIET=1".
//
// See src/examples/test_executor.cpp for an example of using the
// MesosExecutorDriver.
class MesosExecutorDriver : public ExecutorDriver
{
public:
  // Creates a new driver that uses the specified Executor. Note, the
  // executor pointer must outlive the driver.
  explicit MesosExecutorDriver(Executor* executor);

  // This destructor will block indefinitely if
  // MesosExecutorDriver::start was invoked successfully (possibly via
  // MesosExecutorDriver::run) and MesosExecutorDriver::stop has not
  // been invoked.
  virtual ~MesosExecutorDriver();

  // See ExecutorDriver for descriptions of these.
  virtual Status start();
  virtual Status stop();
  virtual Status abort();
  virtual Status join();
  virtual Status run();
  virtual Status sendStatusUpdate(const TaskStatus& status);
  virtual Status sendFrameworkMessage(const std::string& data);

private:
  friend class internal::ExecutorProcess;

  Executor* executor;

  // Libprocess process for communicating with slave.
  internal::ExecutorProcess* process;

  // Mutex for enforcing serial execution of all non-callbacks.
  std::recursive_mutex mutex;

  // Latch for waiting until driver terminates.
  process::Latch* latch;

  // Current status of the driver.
  Status status;
};

+ 啟動器
   用于啟動調(diào)度器驅(qū)動

framework調(diào)度器示例：

#!/usr/bin/env python
#encoding: utf-8

import Queue
import logging
import threading
import time


from pesos.scheduler import PesosSchedulerDriver
from pesos.vendor.mesos import mesos_pb2

from mesos.interface import Scheduler


_logger = logging.getLogger(__name__)

class TestScheduler(Scheduler):

    TASK_CPU = 0.1
    TASK_MEM = 2

    def __init__(self, queue):
        self.tasks = queue
        self.terminal = 0
        self.total_tasks = queue.qsize()

    def registered(self, driver, frameworkId, masterInfo):
        _logger.info('Registered framework %s', frameworkId)

    def reregistered(self, driver, masterInfo):
        _logger.info('Connected with master %s', masterInfo.ip)

    def disconnected(self, driver):
        _logger.info('Disconnected from master')

    def resourceOffers(self, driver, offers):
        _logger.info('Recived %s offers', len(offers))

        def handle_offers():
            declined = []
            
            for offer in offers:
                offer_cpu = 0
                offer_mem = 0

                if self.tasks.empty():
                    declined.append(offer.id.value)
                    continue

                for resource in offer.resources:
                    if resource.name == 'cpus':
                        offer_cpu = resource.scalar.value
                    if resource.name == 'mem':
                        offer_mem = resource.scalar.value

                _logger.info('offer:%s, cpu:%s, mem:%s', offer.id.value, offer_cpu, offer_mem)
                tasks = []

                while offer_mem >= self.TASK_MEM and offer_cpu >= self.TASK_CPU\
                    and not self.tasks.empty():
                    offer_cpu -= self.TASK_CPU
                    offer_mem -= self.TASK_MEM

                    executor_id, task_id, args = self.tasks.get()
                    self.tasks.task_done()
                    _logger.info('Queue task %s:%s', executor_id, task_id)
                    tasks.append(self._build_task(offer, executor_id, task_id, args))
                if tasks:
                    driver.launch_tasks([offer.id.value], tasks)
            for offerid in declined:
                driver.decline_offer(offerid)

        th = threading.Thread(target=handle_offers)
        th.start()

    def _build_task(self, offer, executor_id, task_id, args):
        task = mesos_pb2.TaskInfo()
        task.name = "Test Task of Silence"

        cpus = task.resources.add()
        cpus.name = "cpus"
        cpus.type = mesos_pb2.Value.SCALAR
        cpus.scalar.value = self.TASK_CPU

        mem = task.resources.add()
        mem.name = "mem"
        mem.type = mesos_pb2.Value.SCALAR
        mem.scalar.value = self.TASK_MEM

        task.executor.command.value = "ping %s -c 20" % args[0]

        '''
        task.executor.command.user.value = 'root'

        # TODO LIST
        environment = mesos_pb2.Environment()
        variable = environment.variables.add()
        variable.name = key
        variable.value = value

        uri = task.executor.uris.add()
        uri.value = p_uri
        uri.executable = False
        uri.extract = True
        '''

        '''
        cpus = task.executor.resources.add()
        cpus.name = "cpus"
        cpus.type = mesos_pb2.Value.SCALAR
        cpus.scalar.value = self.TASK_CPU

        mem = task.executor.resources.add()
        mem.name = "mem"
        mem.type = mesos_pb2.Value.SCALAR
        mem.scalar.value = self.TASK_MEM
        task.executor.source = None
        task.executor.data = None

        '''

        task.task_id.value = "%d:%d" % (executor_id, task_id)
        task.slave_id.MergeFrom(offer.slave_id)

        task.executor.executor_id.value = str(executor_id)
        task.executor.framework_id.value = offer.framework_id.value
        
        return task

    def offerRescinded(self, driver, offerId):
        _logger.info('Offer rescinded %s', offerId.value)

    def statusUpdate(self, driver, taskStatus):
        statuses = {
            mesos_pb2.TASK_STAGING: "STAGING",
            mesos_pb2.TASK_STARTING: "STARTING",
            mesos_pb2.TASK_RUNNING: "RUNNING",
            mesos_pb2.TASK_FINISHED: "FINISHED",
            mesos_pb2.TASK_FAILED: "FAILED",
            mesos_pb2.TASK_KILLED: "KILLED",
            mesos_pb2.TASK_LOST: "LOST",
        }

        _logger.info("Received status update for task %s (%s)", taskStatus.task_id.value, statuses[taskStatus.state])

        if taskStatus.state == mesos_pb2.TASK_FINISHED or taskStatus.state == mesos_pb2.TASK_FAILED or \
            taskStatus.state == mesos_pb2.TASK_KILLED or taskStatus.state == mesos_pb2.TASK_LOST:
            self.terminal += 1

        if self.terminal == self.total_tasks:
            driver.stop()

    def frameworkMessage(self, driver, executorId, slaveId, data):
        _logger.info('Message from executor %s and slave %s : %s', executorId.value, slaveId.value, data)

    def slaveLost(self, driver, slaveId):
        _logger.info('Slave %s has been lost', slaveId.value)

    def executorLost(self, dirver, executorId, slaveId, exitCode):
        _logger.info('Executor %s has been lost on slave %s with exit code %s', executorId.value, slaveId.value, exitCode)

    def error(self, driver, message):
        _logger.info('There was an error:%s', message)



if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)

    num_tasks = 50
    num_executors = 5

    tasks = Queue.Queue()
    for task in xrange(num_tasks):
        for executor in xrange(num_executors):
            tasks.put((executor, task, ["www.360.cn"]))

    master_uri = 'master@192.168.56.101:5050'

    framework = mesos_pb2.FrameworkInfo()
    framework.name = "Test Python Framework of Silence"
    framework.user = "root"

    driver = PesosSchedulerDriver(
        TestScheduler(tasks),
        framework,
        master_uri
    )

    _logger.info('Starting driver')
    driver.start()

    _logger.info('Joining driver')
    driver.join()